#DRAFT/WIP 20 allow custom submissions to slurm

scripts/submit_csv.sh

@@ -28,12 +28,32 @@ shift
 TARGET=${1:-2}
 shift
 
+# environment variable to indicate whether the job is running on condor or slurm
+SYSTEM=${SYSTEM:-condor}
+
 # process csv file into jobs
-CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET})
+if [ -n "${CSV_FILE:-}" ]; then
+  # allow to set custom csv file for job instead of fetching from web archive
+  CSV_FILE=$(realpath -e ${CSV_FILE})                     
+else
+  CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET})
+fi
 
 # create command line
-EXECUTABLE="./scripts/run.sh"
-ARGUMENTS="${TYPE} EVGEN/\$(file).\$(ext) \$(nevents) \$(ichunk)"
+EXECUTABLE="$PWD/scripts/run.sh"
+if [ ${SYSTEM} = "condor" ]; then
+  ARGUMENTS="${TYPE} EVGEN/\$(file).\$(ext) \$(nevents) \$(ichunk)"
+elif [ ${SYSTEM} = "slurm" ]; then
+  ARGUMENTS="${TYPE}"
+  # FIXME: This is not ideal. It prevents from submitting multiple jobs with different JUG_XL_TAG simultaneously.
+  cd scripts 
+  wget --output-document install.sh https://get.epic-eic.org
+  sed -i 's/nightly/${JUG_XL_TAG}/g' install.sh
+  bash install.sh
+  cd ..
+else
+  echo "Enter a valid SYSTEM value (condor or slurm)"
+fi
 
 # construct environment file
 ENVIRONMENT=environment-$(date --iso-8601=minutes).sh
@@ -54,8 +74,12 @@ REQUIREMENTS=""
 # construct input files
 INPUT_FILES=${ENVIRONMENT}
 
+# calculate number of jobs being submitted
+NJOBS=$( wc -l ${CSV_FILE} | awk '{print $1}' )
+
 # construct submission file
 SUBMIT_FILE=$(basename ${CSV_FILE} .csv).submit
+
 sed "
   s|%EXECUTABLE%|${EXECUTABLE}|g;
   s|%ARGUMENTS%|${ARGUMENTS}|g;
@@ -65,14 +89,34 @@ sed "
   s|%INPUT_FILES%|${INPUT_FILES}|g;
   s|%REQUIREMENTS%|${REQUIREMENTS}|g;
   s|%CSV_FILE%|${CSV_FILE}|g;
+  s|%ACCOUNT%|${ACCOUNT:?Define ACCOUNT with the slurm account}|g;
+  s|%CAMPAIGN_LOG%|${CAMPAIGN_LOG:-$PWD}|g;
+  s|%TARGET%|$TARGET|g;
+  s|%NJOBS%|${NJOBS}|g;
 " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE}
 
 # submit job
-condor_submit -verbose -file ${SUBMIT_FILE}
 
-# create log dir
-if [ $? -eq 0 ] ; then
-  for i in `condor_q | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do
-    mkdir -p LOG/CONDOR/osg_$i/
-  done
+echo "Submitting ${NJOBS} to a ${SYSTEM} system"
+
+if [ ${SYSTEM} = "condor" ]; then
+  condor_submit -verbose -file ${SUBMIT_FILE}
+
+  # create log dir
+  if [ $? -eq 0 ] ; then
+    for i in `condor_q | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do
+      mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/CONDOR/osg_$i/
+    done
+  fi
+elif [ ${SYSTEM} = "slurm" ]; then
+  sbatch ${SUBMIT_FILE}
+
+  # create log dir
+  if [ $? -eq 0 ] ; then
+    for i in `squeue -u ${USER} | tail -n1 |  awk '{$1=$1; print}' | awk -F" |_" '{print($1)}' | cut -d. -f1` ; do
+      mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/SLURM/slurm_$i/
+    done
+  fi
+else
+  echo "Enter a valid SYSTEM value (condor or slurm)"
 fi

templates/narval_csv.sh.in

@@ -0,0 +1,8 @@
+export S3_ACCESS_KEY=%S3_ACCESS_KEY%
+export S3_SECRET_KEY=%S3_SECRET_KEY%
+export S3RW_ACCESS_KEY=%S3RW_ACCESS_KEY%
+export S3RW_SECRET_KEY=%S3RW_SECRET_KEY%
+export DETECTOR_VERSION=%DETECTOR_VERSION%
+export DETECTOR_CONFIG=%DETECTOR_CONFIG%
+export EBEAM=%EBEAM%
+export PBEAM=%PBEAM%

templates/narval_csv.submit.in

@@ -0,0 +1,22 @@
+#!/bin/bash
+#SBATCH --account=%ACCOUNT%
+#SBATCH --array=1-%NJOBS%
+#SBATCH --ntasks=1                   # number of MPI processes
+#SBATCH --mem-per-cpu=3G             # memory; default unit is megabytes
+#SBATCH --time=0-%TARGET%:00         # time (DD-HH:MM)
+#SBATCH --output=%CAMPAIGN_LOG%/LOG/SLURM/slurm_%A/slurm_%A_%a.out # standard output log
+#SBATCH --error=%CAMPAIGN_LOG%/LOG/SLURM/slurm_%A/slurm_%A_%a.err  # standard error log
+
+input_string=$(sed "${SLURM_ARRAY_TASK_ID}q;d" %CSV_FILE%)
+read -r file ext nevents ichunk <<< $(echo "$input_string" | awk -F',' '{print $1, $2, $3, $4}')
+
+echo $file
+echo $ext
+echo $nevents
+echo $ichunk
+
+cat << EOF | $(dirname %EXECUTABLE%)/eic-shell
+
+%EXECUTABLE% %ARGUMENTS% EVGEN/${file}.${ext} ${nevents} ${ichunk}
+
+EOF