Merge branch 'develop' into feature/cmake_build

Externals.cfg

@@ -57,7 +57,7 @@ protocol = git
 required = False
 
 [GDASApp]
-hash = 7e3d694
+hash = 7966501
 local_path = sorc/gdas.cd
 repo_url = https://github.com/NOAA-EMC/GDASApp.git
 protocol = git

ci/cases/C48_ATM.yaml

@@ -0,0 +1,13 @@
+experiment:
+  type: gfs
+  mode: forecast-only
+
+arguments:
+  pslot: ${pslot}
+  app: ATM
+  resdet: 48
+  comrot: ${RUNTESTS}/COMROT
+  expdir: ${RUNTESTS}/EXPDIR
+  idate: 2021032312
+  edate: 2021032312
+  yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml

ci/scripts/create_experiment.py

@@ -2,108 +2,89 @@
 
 """
 Basic python script to create an experiment directory on the fly from a given
-
 yaml file for the arguments to the two scripts below in ${HOMEgfs}/workflow
-
 where ${HOMEgfs} is specified within the input yaml file.
 
  ${HOMEgfs}/workflow/setup_expt.py
  ${HOMEgfs}/workflow/setup_xml.py
 
 The yaml file are simply the arguments for these two scripts.
-After this scripts runs these two the use will have an experiment ready for launching
+After this scripts runs the experiment is ready for launch.
 
 Output
 ------
-
 Functionally an experiment is setup as a result running the two scripts described above
 with an error code of 0 upon success.
 """
 
 import os
+import sys
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
 
-from wxflow import YAMLFile, Logger, logit, Executable
+from wxflow import YAMLFile, Logger, logit
 
 
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..'))
 logger = Logger(level='DEBUG', colored_log=True)
 
 
+# TODO: move create_experiment.py to workflow/ and remove this sys.path.insert business
+sys.path.insert(0, os.path.join(_top, 'workflow'))
+import setup_expt
+import setup_xml
+
+
 @logit(logger)
 def input_args():
     """
-    Method to collect user arguments for `create_experiment.py`
-
-    Input
-    -----
-
-    A single key valued argument: --yaml <full path to YAML file>
-
     Description
     -----------
 
-    A full path to a YAML file with the following format with required sections: experiment, arguments
+    Method to collect user arguments for `create_experiment.py`
 
-    experiment:
-        mode: <cycled> <forecast-only>
-            used to hold the only required positional argument to setup_expt.py
+    Parameters
+    ----------
 
-    arguments:
-        holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py
-        Note: the argument pslot is derived from the basename of the yamlfile itself
+    None
 
     Returns
     -------
 
-    args: Namespace
-
-        Namespace with the value of the file path to a yaml file from the key yaml
+    argparse.Namespace:
+        argparse.Namespace with the value of the file path to a yaml file from the key yaml
     """
 
-    description = """Single argument as a yaml file containing the
-    key value pairs as arguments to setup_expt.py
-    """
+    description = """Create a global-workflow experiment"""
 
     parser = ArgumentParser(description=description,
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True)
-    parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True)
+    parser.add_argument('--yaml', help='full path to yaml file describing the experiment configuration', type=str, required=True)
+    parser.add_argument('--dir', help='full path to global-workflow build', type=str, required=True)
 
-    args = parser.parse_args()
-    return args
+    return parser.parse_args()
 
 
 if __name__ == '__main__':
 
     user_inputs = input_args()
-    setup_expt_args = YAMLFile(path=user_inputs.yaml)
-
     HOMEgfs = Path.absolute(Path(user_inputs.dir))
-    type = setup_expt_args.experiment.type
-    mode = setup_expt_args.experiment.mode
-
-    setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py'))
-
-    setup_expt_cmd.add_default_arg(type)
-    setup_expt_cmd.add_default_arg(mode)
+    testconf = YAMLFile(path=user_inputs.yaml)
+    experiment_dir = Path.absolute(Path.joinpath(Path(testconf.arguments.expdir), Path(testconf.arguments.pslot)))
 
-    for conf, value in setup_expt_args.arguments.items():
-        setup_expt_cmd.add_default_arg(f'--{conf}')
-        setup_expt_cmd.add_default_arg(str(value))
+    # Create a list of arguments to setup_expt.py
+    setup_expt_args = [testconf.experiment.type, testconf.experiment.mode]  # TODO: rename 'type' as 'system' in case.yaml
+    for kk, vv in testconf.arguments.items():
+        setup_expt_args.append(f"--{kk}")
+        setup_expt_args.append(str(vv))
 
-    logger.info(f'Run command: {setup_expt_cmd.command}')
-    setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr'))
-    setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout'))
-    print(setup_expt_stderr)
-    setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr)
+    logger.info(f'Call: setup_expt.main()')
+    setup_expt.main(setup_expt_args)
 
-    setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py'))
-    expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(setup_expt_args.arguments.pslot)))
-    setup_xml_cmd.add_default_arg(str(expdir))
+    # Create a list of arguments to setup_xml.py
+    setup_xml_args = [str(experiment_dir)]
 
-    logger.info(f'Run command: {setup_xml_cmd.command}')
-    setup_xml_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stderr'))
-    setup_xml_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stdout'))
-    setup_xml_cmd(output=setup_xml_stdout, error=setup_xml_stderr)
+    logger.info(f"Call: setup_xml.main()")
+    setup_xml.main(setup_xml_args)

ci/scripts/driver.sh

@@ -13,7 +13,6 @@ set -eux
 # PR number and calls clone-build_ci.sh to perform a clone and full build from $(HOMEgfs)/sorc
 # of the PR. It then is ready to run a suite of regression tests with various
 # configurations with run_tests.py.
-# No-op for test
 #######################################################################################
 
 #################################################################
@@ -156,7 +155,7 @@ for pr in ${pr_list}; do
       pslot="${case}_${pr_sha}"
       export pslot
       set +e
-      "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}"
+      "${HOMEgfs_PR}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}"
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY

@@ -28,6 +28,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS
 
 # Add UFSDA to PYTHONPATH
 export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/:${HOMEgfs}/sorc/gdas.cd/ush/eva:${PYTHONPATH}
+export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/soca:${PYTHONPATH}
 
 ###############################################################
 # Run relevant script

jobs/rocoto/ocnpost.sh

@@ -44,7 +44,7 @@ done
 fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g')
 
 export OMP_NUM_THREADS=1
-export ENSMEM=${ENSMEM:-01}
+export ENSMEM=${ENSMEM:-000}
 
 export IDATE=${PDY}${cyc}
 

jobs/rocoto/preplandobs.sh

@@ -14,7 +14,7 @@ export jobid="${job}.$$"
 ###############################################################
 # setup python path for workflow utilities and tasks
 wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-gdasappPATH="${HOMEgfs}/sorc/gdas.cd/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.7/pyioda"
+gdasappPATH="${HOMEgfs}/sorc/gdas.cd/iodaconv/src:${HOMEgfs}/sorc/gdas.cd/build/lib/python3.7"
 PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}"
 export PYTHONPATH
 

parm/config/gefs/config.aero

@@ -0,0 +1 @@
+../gfs/config.aero

parm/config/gefs/config.base.emc.dyn

@@ -276,6 +276,11 @@ export DO_MERGENSST="NO"
 # Hybrid related
 export NMEM_ENS=@NMEM_ENS@
 
+# set default member number memdir for control
+# this will be overwritten for the perturbed members
+export ENSMEM="000"
+export MEMDIR="mem${ENSMEM}"
+
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then
     export IAU_FHROT=0

parm/config/gefs/config.defaults.s2sw

@@ -0,0 +1 @@
+../gfs/config.defaults.s2sw

parm/config/gefs/config.efcs

@@ -5,12 +5,6 @@
 
 echo "BEGIN: config.efcs"
 
-# TODO: the _ENKF counterparts need to be defined in config.base
-export DO_AERO=${DO_AERO_ENKF:-"NO"}
-export DO_OCN=${DO_OCN_ENKF:-"NO"}
-export DO_ICE=${DO_ICE_ENKF:-"NO"}
-export DO_WAVE=${DO_WAVE_ENKF:-"NO"}
-
 # TODO: Possibly need OCNRES_ENKF, ICERES_ENKF, WAVRES_ENKF too
 if [[ ${DO_OCN} == "YES" ]]; then
   case "${CASE_ENS}" in
@@ -66,7 +60,6 @@ export SPPT_LSCALE=500000.
 export SPPT_LOGIT=".true."
 export SPPT_SFCLIMIT=".true."
 
-# FV3 model namelist parameters to over-ride
 export restart_interval=${restart_interval:-6}
 
 echo "END: config.efcs"

parm/config/gefs/config.ice

@@ -0,0 +1 @@
+../gfs/config.ice

parm/config/gefs/config.ocn

@@ -0,0 +1 @@
+../gfs/config.ocn

parm/config/gefs/config.wave

@@ -0,0 +1 @@
+../gfs/config.wave

parm/config/gfs/config.aero

@@ -2,11 +2,14 @@
 
 # UFS-Aerosols settings
 
+# Turn off warnings about unused variables
+# shellcheck disable=SC2034
+
 # Directory containing GOCART configuration files. Defaults to parm/chem if unset.
-AERO_CONFIG_DIR=$HOMEgfs/parm/chem
+AERO_CONFIG_DIR=${HOMEgfs}/parm/chem
 
 # Path to the input data tree
-case $machine in
+case ${machine} in
   "HERA")
     AERO_INPUTS_DIR="/scratch1/NCEPDEV/global/glopara/data/gocart_emissions"
     ;;
@@ -23,7 +26,7 @@ case $machine in
     AERO_INPUTS_DIR="/lfs4/HFIP/hfv3gfs/glopara/data/gocart_emissions"
     ;;
   *)
-    echo "FATAL ERROR: Machine $machine unsupported for aerosols"
+    echo "FATAL ERROR: Machine ${machine} unsupported for aerosols"
     exit 2
     ;;
 esac

parm/config/gfs/config.defaults.s2sw

@@ -2,6 +2,9 @@
 
 # Empty variables must include a space otherwise they will be overwritten
 
+# Turn off warnings about unused variables
+# shellcheck disable=SC2034
+
 # config.base
 FHMAX_GFS_00=48
 FHMAX_GFS_06=48

parm/config/gfs/config.landanl

@@ -21,11 +21,7 @@ export JEDIEXE="${HOMEgfs}/exec/fv3jedi_letkf.x"
 export JEDIYAML="${HOMEgfs}/sorc/gdas.cd/parm/land/letkfoi/letkfoi.yaml"
 
 # Ensemble member properties
-if [[ "${FRAC_GRID}" = ".true." ]]; then
-  export SNOWDEPTHVAR="snodl"
-elif [[ "${FRAC_GRID}" = ".false." ]]; then
-  export SNOWDEPTHVAR="snwdph"
-fi
+export SNOWDEPTHVAR="snodl"
 export BESTDDEV="30."  # Background Error Std. Dev. for LETKFOI
 
 # Name of the executable that applies increment to bkg and its namelist template

parm/config/gfs/config.preplandobs

@@ -8,6 +8,9 @@ echo "BEGIN: config.preplandobs"
 # Get task specific resources
 . "${EXPDIR}/config.resources" preplandobs
 
+export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_gts.yaml"
+export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x"
+export BUFR2IODAYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml"
 export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/fims.nml.j2"
 export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_ims.yaml"
 export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe"

parm/config/gfs/config.resources

@@ -250,12 +250,12 @@ elif [[ "${step}" = "aeroanlinit" ]]; then
    # below lines are for creating JEDI YAML
    case ${CASE} in
      C768)
-        layout_x=6
-        layout_y=6
+        layout_x=8
+        layout_y=8
         ;;
       C384)
-        layout_x=5
-        layout_y=5
+        layout_x=8
+        layout_y=8
         ;;
      C192 | C96 | C48)
         layout_x=8
@@ -280,12 +280,12 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
 
    case ${CASE} in
      C768)
-        layout_x=6
-        layout_y=6
+        layout_x=8
+        layout_y=8
         ;;
       C384)
-        layout_x=5
-        layout_y=5
+        layout_x=8
+        layout_y=8
         ;;
      C192 | C96 | C48)
         layout_x=8
@@ -470,6 +470,7 @@ elif [[ ${step} = "analcalc" ]]; then
     npe_node_analcalc=$(echo "${npe_node_max} / ${nth_analcalc}" | bc)
     export npe_node_analcalc
     export is_exclusive=True
+    export memory_analcalc="48GB"
 
 elif [[ ${step} = "analdiag" ]]; then
 
@@ -869,7 +870,7 @@ elif [[ ${step} = "ecen" ]]; then
 
 elif [[ ${step} = "esfc" ]]; then
 
-    export wtime_esfc="00:06:00"
+    export wtime_esfc="00:08:00"
     export npe_esfc=80
     export nth_esfc=1
     npe_node_esfc=$(echo "${npe_node_max} / ${nth_esfc}" | bc)
@@ -883,10 +884,7 @@ elif [[ ${step} = "epos" ]]; then
 
     export wtime_epos="00:15:00"
     export npe_epos=80
-    export nth_epos=4
-    if [[ "${machine}" == "HERA" ]]; then
-      export nth_epos=6
-    fi
+    export nth_epos=1
     npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc)
     export npe_node_epos
     export is_exclusive=True

parm/config/gfs/config.ufs

@@ -151,7 +151,7 @@ case "${fv3_res}" in
         export layout_y=8
         export layout_x_gfs=8
         export layout_y_gfs=8
-        export nthreads_fv3=1
+        export nthreads_fv3=2
         export nthreads_fv3_gfs=2
         export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=2

parm/parm_fv3diag/field_table_thompson_noaero_tke_progsigma

@@ -67,4 +67,4 @@
  "TRACER", "atmos_mod", "sigmab"
            "longname",     "sigma fraction"
            "units",        "fraction"
-       "profile_type", "fixed", "surface_value=0.0" /
+       "profile_type", "fixed", "surface_value=0.0" /