From 6d34932674fafa02e34e98a667f72700c098f74c Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Thu, 7 Nov 2024 11:00:23 -0600 Subject: [PATCH 01/13] addeing no up for ci develop branch self test --- ci/scripts/driver.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index acf54381b8..b2a5799f5a 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -12,6 +12,7 @@ set -eux # development branch for the global-workflow repo. It then stages tests directories per # PR number and calls clone-build_ci.sh to perform a clone and full build from the PR. # It then is ready to run a suite of regression tests with various configurations +# ci devleop self test no-op ####################################################################################### export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} From 7e47fbaa09798cb46f2131532b61c8e39601de01 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 11:14:57 -0600 Subject: [PATCH 02/13] added hotfix to clear RUNDIRS before experments are ran to conver for restarts --- ci/Jenkinsfile | 4 +++ ci/scripts/check_ci.sh | 3 +-- ci/scripts/utils/ci_utils.sh | 23 +++++++++++++++- ci/scripts/utils/get_config_var.py | 43 ++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) create mode 100755 ci/scripts/utils/get_config_var.py diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 6a2e064be0..e78486f5be 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -221,7 +221,11 @@ pipeline { def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" def build_system = yaml_case.experiment.system try { + // Remove any data in that might still be in the DATAROOT directory from previous runs + STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim() + sh(script: "rm -Rf ${STMP}/RUNDIR/${pslot}") sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") ws(CUSTOM_WORKSPACE) { diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 825d8f5e8b..a89a661042 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -168,8 +168,7 @@ for pr in ${pr_list}; do fi if [[ "${rocoto_state}" == "DONE" ]]; then #Remove Experment cases that completed successfully - rm -Rf "${pslot_dir}" - rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}" + "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}" rm -f "${output_ci_single}" # echo "\`\`\`" > "${output_ci_single}" DATE=$(date +'%D %r') diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 2a51467d38..03a71f90ec 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -137,7 +137,6 @@ function publish_logs() { local PR_header="$1" local dir_path="$2" local file="$3" - local full_paths="" while IFS= read -r line; do full_path="${dir_path}/${line}" @@ -155,3 +154,25 @@ function publish_logs() { fi echo "${URL}" } + +function cleanup_experiment() { + + local EXPDIR="$1" + local pslot + local ARCDIR + local ATARDIR + local DATAROOT + local COMROOT + + EXPDIR="$1" + pslot=$(basename "${EXPDIR}") + + # Use the Python utility to get the required variables + read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true + + rm -Rf "${STMP}/RUNDIRS/${pslot}" + rm -Rf "${ARCDIR:?}" + rm -Rf "${ATARDIR:?}" + rm -Rf "${COMROOT:?}" + rm -Rf "${EXPDIR}" +} diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py new file mode 100755 index 0000000000..3e63fcf376 --- /dev/null +++ b/ci/scripts/utils/get_config_var.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import os +import argparse +from wxflow import Configuration + + +def get_config_vars(var_names, config_path): + """ + GET_CONFIG_VARS Get configuration variables from a config file or directory. + Parameters: + var_names (list of str): The names of the configuration variables to retrieve. + config_path (str): The path to the configuration file or directory. + Returns: + list of str: The values of the specified configuration variables. + """ + if os.path.isfile(config_path): + config_dir = os.path.dirname(config_path) + config_file = os.path.basename(config_path) + elif os.path.isdir(config_path): + config_dir = config_path + config_file = 'config.base' + config = Configuration(config_dir) + config_data = config.parse_config(config_file) + return [config_data[var_name] for var_name in var_names] + + +if __name__ == "__main__": + """ + Main entry point for the script. + Parses command-line arguments and retrieves the specified configuration variables. + """ + parser = argparse.ArgumentParser(description="Get configuration variables from a config file or directory.") + parser.add_argument("var_names", nargs='+', help="The names of the configuration variables to retrieve.") + parser.add_argument("config_path", help="The path to the configuration file or directory.") + + args = parser.parse_args() + + var_names = args.var_names + config_path = args.config_path + + values = get_config_vars(var_names, config_path) + print(" ".join(values)) From 4abf7ebff64d74a8819a415df5da12e65a448186 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 11:18:19 -0600 Subject: [PATCH 03/13] removed no-op and local DATAROOT variable (there is non its a kludge variable) --- ci/scripts/driver.sh | 1 - ci/scripts/utils/ci_utils.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index b2a5799f5a..acf54381b8 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -12,7 +12,6 @@ set -eux # development branch for the global-workflow repo. It then stages tests directories per # PR number and calls clone-build_ci.sh to perform a clone and full build from the PR. # It then is ready to run a suite of regression tests with various configurations -# ci devleop self test no-op ####################################################################################### export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 03a71f90ec..64988986fe 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -161,7 +161,6 @@ function cleanup_experiment() { local pslot local ARCDIR local ATARDIR - local DATAROOT local COMROOT EXPDIR="$1" From bcf571cf1b81d14b26ae80941f0dad1962740b4f Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 11:23:36 -0600 Subject: [PATCH 04/13] ITS RUNDIRS NOT RUNDIR arrrggg (bug fix) in Jeknkinsfile --- ci/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index e78486f5be..8b987a16fa 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -223,7 +223,7 @@ pipeline { try { // Remove any data in that might still be in the DATAROOT directory from previous runs STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim() - sh(script: "rm -Rf ${STMP}/RUNDIR/${pslot}") + sh(script: "rm -Rf ${STMP}/RUNDIRS/${pslot}") sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") } catch (Exception error_experment) { From 82c76bc29f7f4fb29f5b99b622982262a8ae05f2 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 11:28:59 -0600 Subject: [PATCH 05/13] adding echo output in pipeline for sanity check to check bug fix for removing the correct path to RUNDIRS --- ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 8b987a16fa..fc7c847322 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -223,6 +223,7 @@ pipeline { try { // Remove any data in that might still be in the DATAROOT directory from previous runs STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim() + echo "Cleaning RUNDIRS if previously ran in: ${STMP}/RUNDIRS/${pslot}" sh(script: "rm -Rf ${STMP}/RUNDIRS/${pslot}") sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") From f611ba6b03a0d83e63db7e79bf1a82b1139390f5 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 16:36:30 -0600 Subject: [PATCH 06/13] removed cleaning of RUNDIRS in cleanup experment because it failed in the tests and also noticed that we had convered this corner case for clean RUNDIRS on restart in the bash util for create_experement but broke it when we moved STMP to the host files. --- ci/scripts/utils/ci_utils.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 64988986fe..4b85eb99b7 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -123,6 +123,7 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry + STMP=$(grep STMP "workflow/hosts/${MACHIND_ID}.yaml" | cut -d: -f2 | sed 's/[ "'\''"]*//g') rm -Rf "${STMP}/RUNDIRS/${pslot}" "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" @@ -169,7 +170,7 @@ function cleanup_experiment() { # Use the Python utility to get the required variables read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true - rm -Rf "${STMP}/RUNDIRS/${pslot}" + #rm -Rf "${STMP}/RUNDIRS/${pslot}" rm -Rf "${ARCDIR:?}" rm -Rf "${ATARDIR:?}" rm -Rf "${COMROOT:?}" From d9d63af5ccda467b9c9c0d1ae4c4e7243a318729 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 16:40:57 -0600 Subject: [PATCH 07/13] added || true to compound statment for shellnorms in getting STMP update in CI bash utils --- ci/scripts/utils/ci_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 4b85eb99b7..ae2f02c2b8 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -123,7 +123,7 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry - STMP=$(grep STMP "workflow/hosts/${MACHIND_ID}.yaml" | cut -d: -f2 | sed 's/[ "'\''"]*//g') + STMP=$(grep STMP "workflow/hosts/${MACHIND_ID}.yaml" | cut -d: -f2 | sed 's/[ "'\''"]*//g') || true rm -Rf "${STMP}/RUNDIRS/${pslot}" "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" From 45b565a67212d781ffdcf2e3936d57344a38ce8b Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 20:23:53 -0600 Subject: [PATCH 08/13] needed pslot when removing individual experments and STMP was needed in previous fix (months ago) for deleting RUNDIRS before creeating experment --- ci/Jenkinsfile | 4 ---- ci/scripts/utils/ci_utils.sh | 11 ++++++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index fc7c847322..b3bd6a917a 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -221,10 +221,6 @@ pipeline { def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" def build_system = yaml_case.experiment.system try { - // Remove any data in that might still be in the DATAROOT directory from previous runs - STMP=sh(script: "${HOMEgfs}/ci/scripts/utils/get_config_var.py STMP ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}", returnStdout: true).trim() - echo "Cleaning RUNDIRS if previously ran in: ${STMP}/RUNDIRS/${pslot}" - sh(script: "rm -Rf ${STMP}/RUNDIRS/${pslot}") sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") } catch (Exception error_experment) { diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index ae2f02c2b8..2a4598a6be 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -122,8 +122,9 @@ function create_experiment () { source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" source "${HOMEgfs}/workflow/gw_setup.sh" - # Remove RUNDIRS dir incase this is a retry - STMP=$(grep STMP "workflow/hosts/${MACHIND_ID}.yaml" | cut -d: -f2 | sed 's/[ "'\''"]*//g') || true + # Remove RUNDIRS dir incase this is a retry (STMP now in host file) + STMP=$(grep "${HOMEgfs}/ci/scripts/parse_yaml.py -y ${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml -k STMP -s") + echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" @@ -170,9 +171,9 @@ function cleanup_experiment() { # Use the Python utility to get the required variables read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true - #rm -Rf "${STMP}/RUNDIRS/${pslot}" rm -Rf "${ARCDIR:?}" rm -Rf "${ATARDIR:?}" - rm -Rf "${COMROOT:?}" - rm -Rf "${EXPDIR}" + rm -Rf "${COMROOT}/${pslot}" + rm -Rf "${EXPDIR}/${pslot}" + rm -Rf "${STMP}/RUNDIRS/${pslot}" } From 40bb91d25a1c00f5eff5b0e3632b0cb40095c8ee Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 20:29:14 -0600 Subject: [PATCH 09/13] fixed cut and paste error by removing extranous grep sring in front of parse_yaml line when getting STMP from host file --- ci/scripts/utils/ci_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 2a4598a6be..07b54f23db 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -123,7 +123,7 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry (STMP now in host file) - STMP=$(grep "${HOMEgfs}/ci/scripts/parse_yaml.py -y ${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml -k STMP -s") + STMP=$("${HOMEgfs}/ci/scripts/parse_yaml.py -y ${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml -k STMP -s") echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" From db78344ed19cb5ae3c0cff4b40a7781ccf97d011 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 20:32:51 -0600 Subject: [PATCH 10/13] adding :? at the end of rm paths for shellnorms --- ci/scripts/utils/ci_utils.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 07b54f23db..13f089370d 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -173,7 +173,7 @@ function cleanup_experiment() { rm -Rf "${ARCDIR:?}" rm -Rf "${ATARDIR:?}" - rm -Rf "${COMROOT}/${pslot}" - rm -Rf "${EXPDIR}/${pslot}" - rm -Rf "${STMP}/RUNDIRS/${pslot}" + rm -Rf "${COMROOT}/${pslot:?}" + rm -Rf "${EXPDIR}/${pslot:?}" + rm -Rf "${STMP}/RUNDIRS/${pslot:?}" } From bc47272e73d1c3f2f011e2c9dc9015aa54fc1a31 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 20:46:29 -0600 Subject: [PATCH 11/13] left out utils in path to pars_yaml utility --- ci/scripts/utils/ci_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 13f089370d..04640391a6 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -123,7 +123,7 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry (STMP now in host file) - STMP=$("${HOMEgfs}/ci/scripts/parse_yaml.py -y ${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml -k STMP -s") + STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py -y ${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml -k STMP -s") echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" From 1e17407f19bead756489db1deede8bd46d58d932 Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 21:00:11 -0600 Subject: [PATCH 12/13] OMG still had srting wrong to getting STMP with unbalanced quotes --- ci/scripts/utils/ci_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index 04640391a6..56b0571adc 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -123,7 +123,7 @@ function create_experiment () { source "${HOMEgfs}/workflow/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry (STMP now in host file) - STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py -y ${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml -k STMP -s") + STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" From 97271494f78641ea2bd8f9dac0498e2f88501e4d Mon Sep 17 00:00:00 2001 From: Terry McGuinness Date: Fri, 8 Nov 2024 21:09:04 -0600 Subject: [PATCH 13/13] extra white spaces in get config python script --- ci/scripts/utils/get_config_var.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/get_config_var.py b/ci/scripts/utils/get_config_var.py index 3e63fcf376..84559b8c14 100755 --- a/ci/scripts/utils/get_config_var.py +++ b/ci/scripts/utils/get_config_var.py @@ -33,7 +33,7 @@ def get_config_vars(var_names, config_path): parser = argparse.ArgumentParser(description="Get configuration variables from a config file or directory.") parser.add_argument("var_names", nargs='+', help="The names of the configuration variables to retrieve.") parser.add_argument("config_path", help="The path to the configuration file or directory.") - + args = parser.parse_args() var_names = args.var_names