Merge branch 'NOAA-EMC:develop' into IASI-NG

.github/workflows/pw_aws_ci.yaml

@@ -72,9 +72,9 @@ jobs:
       id: get-branch
       run: |
         pr_number=${{ github.event.inputs.pr_number }}
+        repo=${{ github.repository }}
         if [ "$pr_number" -eq "0" ]; then
           branch=${{ github.event.inputs.ref }}
-          repo=${{ github.repository }}
         else
           branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName')
           repo_owner=$(gh pr view $pr_number --repo $repo --json headRepositoryOwner --jq '.headRepositoryOwner.login')

.gitignore

@@ -69,6 +69,7 @@ parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
 parm/post/gfs
 parm/post/gefs
+parm/post/sfs
 parm/post/ocean.csv
 parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2

ci/Jenkinsfile

@@ -222,6 +222,7 @@ pipeline {
                                                 def build_system = yaml_case.experiment.system
                                                 try {
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -21,4 +21,4 @@ skip_ci_on_hosts:
   - gaea
   - orion
   - hercules
-
+  - wcoss2

ci/cases/sfs/C96mx100_S2S.yaml

@@ -0,0 +1,19 @@
+experiment:
+  system: gefs
+  mode: forecast-only
+
+arguments:
+  idate: 1994050100 
+  edate: 1994050100
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 96
+  resensatmos: 96
+  resdetocean: 1
+  start: 'cold'
+  nens: 10
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'TOPICDIR' | getenv }}/HR4/C96mx100
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/sfs_defaults.yaml
+

ci/cases/yamls/gefs_defaults_ci.yaml

@@ -2,3 +2,11 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  SFS_POST: "NO"
+  FHOUT_GFS: 6
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "NO"
+  USE_ATM_ENS_PERTURB_FILES: "NO"
+ocn:
+  MOM6_INTERP_ICS: "NO"
+

ci/cases/yamls/gefs_replay_ci.yaml

@@ -11,4 +11,8 @@ base:
   FHOUT_OCN_GFS: 24
   FHOUT_ICE_GFS: 24
   HOMEDIR: {{ 'RUNTESTS' | getenv }}/GLOBAL
+  SFS_POST: "NO"
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
 

ci/cases/yamls/sfs_defaults.yaml

@@ -0,0 +1,33 @@
+base:
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "NO"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  KEEPDATA: "YES"
+  DO_EXTRACTVARS: "NO"
+  FHMAX_GFS: 2976
+  FHMAX_HF_GFS: 0
+  FHOUT_HF_GFS: 1
+  FHOUT_GFS: 24
+  FHOUT_OCN_GFS: 24
+  FHOUT_ICE_GFS: 24
+  FCST_BREAKPOINTS: ""
+  REPLAY_ICS: "NO"
+  HPSSARCH: "NO"
+  LOCALARCH: "NO"
+  SFS_POST: "YES"
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+fcst:
+  TYPE: "hydro"
+  MONO: "mono"
+  reforecast: "YES"
+  FHZER: 24
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
+ocn:
+  MOM6_INTERP_ICS: "YES"

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -17,4 +17,3 @@ nsst:
   NST_MODEL: "1"
 sfcanl:
   DONST: "NO"
-

ci/scripts/check_ci.sh

@@ -168,8 +168,7 @@ for pr in ${pr_list}; do
     fi
     if [[ "${rocoto_state}" == "DONE" ]]; then
       #Remove Experment cases that completed successfully
-      rm -Rf "${pslot_dir}"
-      rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
+      "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}"
       rm -f "${output_ci_single}"
       # echo "\`\`\`" > "${output_ci_single}"
       DATE=$(date +'%D %r')

ci/scripts/utils/ci_utils.sh

@@ -122,7 +122,9 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  # Remove RUNDIRS dir incase this is a retry
+  # Remove RUNDIRS dir incase this is a retry (STMP now in host file)
+  STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s)
+  echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry"
   rm -Rf "${STMP}/RUNDIRS/${pslot}"
 
   "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}"
@@ -137,7 +139,6 @@ function publish_logs() {
     local PR_header="$1"
     local dir_path="$2"
     local file="$3"
-
     local full_paths=""
     while IFS= read -r line; do
         full_path="${dir_path}/${line}"
@@ -155,3 +156,24 @@ function publish_logs() {
     fi
     echo "${URL}"
 }
+
+function cleanup_experiment() {
+
+    local EXPDIR="$1"
+    local pslot
+    local ARCDIR
+    local ATARDIR
+    local COMROOT
+
+    EXPDIR="$1"
+    pslot=$(basename "${EXPDIR}")
+
+    # Use the Python utility to get the required variables
+    read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true
+
+    rm -Rf "${ARCDIR:?}"
+    rm -Rf "${ATARDIR:?}"
+    rm -Rf "${COMROOT}/${pslot:?}"
+    rm -Rf "${EXPDIR}/${pslot:?}"
+    rm -Rf "${STMP}/RUNDIRS/${pslot:?}"
+}

ci/scripts/utils/get_config_var.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+from wxflow import Configuration
+
+
+def get_config_vars(var_names, config_path):
+    """
+    GET_CONFIG_VARS Get configuration variables from a config file or directory.
+    Parameters:
+    var_names (list of str): The names of the configuration variables to retrieve.
+    config_path (str): The path to the configuration file or directory.
+    Returns:
+    list of str: The values of the specified configuration variables.
+    """
+    if os.path.isfile(config_path):
+        config_dir = os.path.dirname(config_path)
+        config_file = os.path.basename(config_path)
+    elif os.path.isdir(config_path):
+        config_dir = config_path
+        config_file = 'config.base'
+    config = Configuration(config_dir)
+    config_data = config.parse_config(config_file)
+    return [config_data[var_name] for var_name in var_names]
+
+
+if __name__ == "__main__":
+    """
+    Main entry point for the script.
+    Parses command-line arguments and retrieves the specified configuration variables.
+    """
+    parser = argparse.ArgumentParser(description="Get configuration variables from a config file or directory.")
+    parser.add_argument("var_names", nargs='+', help="The names of the configuration variables to retrieve.")
+    parser.add_argument("config_path", help="The path to the configuration file or directory.")
+
+    args = parser.parse_args()
+
+    var_names = args.var_names
+    config_path = args.config_path
+
+    values = get_config_vars(var_names, config_path)
+    print(" ".join(values))

env/HERCULES.env

@@ -145,14 +145,19 @@ case ${step} in
     export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}}
     [[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task}
     export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
-;;
+ ;;
  "marineanlchkpt")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_OCNANAL=${NTHREADSmax}
     export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
  ;;
+ "marineanlletkf")
+
+    export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
+    export APRUN_MARINEANLLETKF="${APRUN_default}"
+ ;;
  "anal" | "analcalc")
 
     export MKL_NUM_THREADS=4

env/WCOSS2.env

@@ -107,17 +107,15 @@ elif [[ "${step}" = "marinebmat" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
     export APRUN_MARINEBMAT="${APRUN_default}"
 
-elif [[ "${step}" = "ocnanalrun" ]]; then
+elif [[ "${step}" = "marineanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEANLVAR="${APRUN_default}"
 
-    export APRUN_OCNANAL="${APRUN_default}"
-
-elif [[ "${step}" = "ocnanalchkpt" ]]; then
+elif [[ "${step}" = "marineanlchkpt" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
-
-    export APRUN_OCNANAL="${APRUN_default}"
+    export APRUN_MARINEANLCHKPT="${APRUN_default}"
 
 elif [[ "${step}" = "ocnanalecen" ]]; then
 

modulefiles/module_gwsetup.wcoss2.lua

@@ -4,5 +4,8 @@ Load environment to run GFS workflow ci scripts on WCOSS2
 
 prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core")
 load(pathJoin("rocoto","1.3.5"))
+load(pathJoin("PrgEnv-intel"))
+load(pathJoin("intel","19.1.3.304"))
+load(pathJoin("python", "3.8.6"))
 
 whatis("Description: GFS run setup environment")

parm/archive/gfs_downstream.yaml.j2

@@ -6,7 +6,7 @@ gfs_downstream:
         - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc"
         - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.snd"
         {% for i in range(1, NUM_SND_COLLECTIVES) %}
-        - "{{ COMIN_ATMOS_WMO | relpath(ROTDIR) }}/gfs_collective{{ i }}.postsnd_{{ cycle_HH }}"
+        - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs_collective{{ i }}.fil"
         {% endfor %}
-        - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.t{{ cycle_HH }}z"
+        - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.??????.{{ cycle_YMDH }}"
         - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.bufrsnd.tar.gz"

parm/config/gefs/config.atmos_products

@@ -25,9 +25,14 @@ fi
 export FLXGF="NO"  # Create interpolated sflux.1p00 file
 
 # paramlist files for the different forecast hours and downsets
-export paramlista="${PARMgfs}/product/gefs.0p25.fFFF.paramlist.a.txt"
-export paramlista_anl="${PARMgfs}/product/gefs.0p25.anl.paramlist.a.txt"
-export paramlista_f000="${PARMgfs}/product/gefs.0p25.f000.paramlist.a.txt"
-export paramlistb="${PARMgfs}/product/gefs.0p25.fFFF.paramlist.b.txt"
+if [[ ${SFS_POST} == "YES" ]]; then
+    export post_prefix='sfs'
+else
+    export post_prefix='gefs'
+fi
+export paramlista="${PARMgfs}/product/${post_prefix}.0p25.fFFF.paramlist.a.txt"
+export paramlista_anl="${PARMgfs}/product/${post_prefix}.0p25.anl.paramlist.a.txt"
+export paramlista_f000="${PARMgfs}/product/${post_prefix}.0p25.f000.paramlist.a.txt"
+export paramlistb="${PARMgfs}/product/${post_prefix}.0p25.fFFF.paramlist.b.txt"
 
 echo "END: config.atmos_products"

parm/config/gefs/config.base

@@ -64,6 +64,7 @@ export REALTIME="YES"
 
 # Experiment mode (cycled or forecast-only)
 export MODE="@MODE@" # cycled/forecast-only
+export SFS_POST="@SFS_POST@" # TODO, place holder until RUN=SFS is developed
 
 ####################################################
 # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE
@@ -210,12 +211,11 @@ case "${APP}" in
     if [[ "${APP}" =~ ^S2SW ]]; then
         export DO_WAVE="YES"
         export WAVE_RUN="both"
-        export cplwav2atm=".true."
     fi
     ;;
   *)
-    echo "Unrecognized APP: ${1}"
-    exit 1
+    echo "FATAL ERROR: Unrecognized APP: '${APP}'"
+    exit 2
     ;;
 esac
 
@@ -238,15 +238,14 @@ else
     export OFFSET_START_HOUR=0
 fi
 
-
 # GFS output and frequency
 export FHMIN_GFS=0
 export FHMAX_GFS="@FHMAX_GFS@"
 # Intermediate times to stop forecast when running in segments
 breakpnts="@FCST_BREAKPOINTS@"
 export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}"
 
-export FHOUT_GFS=6
+export FHOUT_GFS=@FHOUT_GFS@
 export FHMAX_HF_GFS=@FHMAX_HF_GFS@
 export FHOUT_HF_GFS=@FHOUT_HF_GFS@
 export FHOUT_OCN_GFS=@FHOUT_OCN_GFS@
@@ -296,6 +295,7 @@ export ENSMEM=${ENSMEM:-"000"}
 export MEMDIR="mem${ENSMEM}"
 
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
+
 # Check if cycle is warm starting with IAU
 if [[ "${EXP_WARM_START}" = ".true." ]]; then
   if [[ "${DOIAU}" = "YES" ]]; then
@@ -327,7 +327,7 @@ export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive
 export LOCALARCH="@LOCALARCH@"        # save data to local archive
 if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
    echo "Both HPSS and local archiving selected.  Please choose one or the other."
-   exit 2
+   exit 3
 fi
 export ARCH_CYC=00           # Archive data at this cycle for warm_start capability
 export ARCH_WARMICFREQ=4     # Archive frequency in days for warm_start capability
@@ -338,10 +338,4 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES"   # NO=retain ROTDIR.  YES default in arc
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 
-# The tracker, genesis, and METplus jobs are not supported on CSPs yet
-# TODO: we should place these in workflow/hosts/[aws|azure|google]pw.yaml as part of CSP's setup, not for general.
-if [[ "${machine}" =~ "PW" ]]; then
-   export DO_WAVE="NO"
-fi
-
 echo "END: config.base"

parm/config/gefs/config.efcs

@@ -27,7 +27,11 @@ source "${EXPDIR}/config.ufs" ${string}
 source "${EXPDIR}/config.resources" efcs
 
 # Stochastic physics parameters (only for ensemble forecasts)
-export DO_SKEB="YES"
+if [[ "${CASE}" == "C96" ]] ; then
+    export DO_SKEB="NO" # SKEB turned off for C96
+else
+    export DO_SKEB="YES" # SKEB turned on for all other resolutions
+fi
 export SKEB="0.8,-999,-999,-999,-999"
 export SKEB_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
 export SKEB_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"