Some preliminary work on emcee batch

batch/hpc_init.sh

@@ -0,0 +1,134 @@
+# Generic setup
+set -e
+
+# Cluster specific setup
+if [[ $1 == "longleaf" ]]; then
+    # Setup general purpose user variables needed for Longleaf
+    USERO=$( echo $USER | awk '{ print substr($0, 1, 1) }' )
+    USERN=$( echo $USER | awk '{ print substr($0, 2, 1) }' )
+    WORKDIR=$( realpath "/work/users/$USERO/$USERN/$USER/" )
+    USERDIR=$WORKDIR
+
+    # Load required modules
+    module purge
+    module load gcc/9.1.0
+    module load anaconda/2023.03
+    module load git
+elif [[ $1 == "rockfish" ]]; then
+    # Setup general purspose user variables needed for RockFish
+    WORKDIR=$( realpath "/scratch4/struelo1/flepimop-code/$USER/"  )
+    USERDIR=$WORKDIR
+    mkdir -vp $WORKDIR
+
+    # Load required modules
+    module purge
+    module load slurm
+    module load gcc/9.3.0
+    module load anaconda/2020.07
+    module load git/2.42.0
+else
+    echo "The cluster name '$1' is not recognized, must be one of: 'longleaf', 'rockfish'."
+    set +e
+    exit 1
+fi
+
+# Ensure we have a $FLEPI_PATH
+if [ -z "${FLEPI_PATH}" ]; then
+    echo -n "An explicit \$FLEPI_PATH was not provided, please set one (or press enter to use '$USERDIR/flepiMoP'): "
+    read FLEPI_PATH
+    if [ -z "${FLEPI_PATH}" ]; then
+        export FLEPI_PATH="$USERDIR/flepiMoP"
+    fi
+    export FLEPI_PATH=$( realpath "$FLEPI_PATH" )
+    echo "Using '$FLEPI_PATH' for \$FLEPI_PATH."
+fi
+
+# Conda init
+if [ -z "${FLEPI_CONDA}" ]; then
+    echo -n "An explicit \$FLEPI_CONDA was not provided, please set one (or press enter to use 'flepimop-env'): "
+    read FLEPI_CONDA
+    if [ -z "${FLEPI_CONDA}" ]; then
+        export FLEPI_CONDA="flepimop-env"
+    fi
+    echo "Using '$FLEPI_CONDA' for \$FLEPI_CONDA."
+fi
+conda activate $FLEPI_CONDA
+
+# Check the conda environment is valid
+WHICH_PYTHON=$( which python )
+WHICH_R=$( which R )
+PYTHON_ARROW_VERSION=$( python -c "import pyarrow; print(pyarrow.__version__)" )
+R_ARROW_VERSION=$( Rscript -e "cat(as.character(packageVersion('arrow')))" )
+COMPATIBLE_ARROW_VERSION=$( echo "$R_ARROW_VERSION" | grep "$PYTHON_ARROW_VERSION" | wc -l )
+if [[ "$COMPATIBLE_ARROW_VERSION" -ne 1 ]]; then
+    echo "The R version of arrow is '$R_ARROW_VERSION' and the python version is '$PYTHON_ARROW_VERSION'. These may not be compatible versions."
+fi
+
+# Make sure the credentials is is where we expect and have the right perms
+if [ ! -f "$USERDIR/slack_credentials.sh" ]; then
+    echo "You should place sensitive credentials in '$USERDIR/slack_credentials.sh'."
+else
+    chmod 600 $USERDIR/slack_credentials.sh
+    source $USERDIR/slack_credentials.sh
+fi
+
+# Set correct env vars
+export FLEPI_STOCHASTIC_RUN=false
+export FLEPI_RESET_CHIMERICS=TRUE
+export TODAY=`date --rfc-3339='date'`
+
+echo -n "Please set a project path (relative to '$WORKDIR'): "
+read PROJECT_PATH
+export PROJECT_PATH="$WORKDIR/$PROJECT_PATH"
+if [ ! -d $PROJECT_PATH ]; then
+    echo "> The project path provided, $PROJECT_PATH, is not a directory. Please ensure this is correct."
+fi
+
+echo -n "Please set a config path (relative to '$PROJECT_PATH'): "
+read CONFIG_PATH
+export CONFIG_PATH="$PROJECT_PATH/$CONFIG_PATH"
+if [ ! -f $CONFIG_PATH ]; then
+    echo "> The config path provided, $CONFIG_PATH, is not a file. Please ensure this is correct."
+fi
+
+echo -n "Please set a validation date (today is $TODAY): "
+read VALIDATION_DATE
+
+echo -n "Please set a resume location: "
+read RESUME_LOCATION
+
+echo -n "Please set a flepi run index: "
+read FLEPI_RUN_INDEX
+
+# Done
+cat << EOM
+> The HPC init script has successfully finished.
+
+If you are testing if this worked, say installing for the first time, you can use the inference example from the \`flepimop_sample\` repository:
+\`\`\`bash
+cd \$PROJECT_PATH
+flepimop-inference-main -c \$CONFIG_PATH -j 1 -n 1 -k 1
+\`\`\`
+Just make sure to \`rm -r model_output\` after running.
+
+Otherwise make sure this diagnostic info looks correct before continuing:
+* Cluster:         $1
+* User directory:  $USERDIR
+* Work directory:  $WORKDIR
+* Flepi conda:     $FLEPI_CONDA
+* Flepi path:      $FLEPI_PATH
+* Project path:    $PROJECT_PATH
+* Python:          $WHICH_PYTHON
+* R:               $WHICH_R
+* Python arrow:    $PYTHON_ARROW_VERSION
+* R arrow:         $R_ARROW_VERSION
+* Stochastic run:  $FLEPI_STOCHASTIC_RUN
+* Reset chimerics: $FLEPI_RESET_CHIMERICS
+* Today:           $TODAY
+* Config path:     $CONFIG_PATH
+* Validation date: $VALIDATION_DATE
+* Resume location: $RESUME_LOCATION
+* Flepi run index: $FLEPI_RUN_INDEX
+EOM
+
+set +e

build/hpc_install_or_update.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# Generic setup
+set -e
+
+# Cluster specific setup
+if [[ $1 == "longleaf" ]]; then
+    # Setup general purpose user variables needed for Longleaf
+    USERO=$( echo $USER | awk '{ print substr($0, 1, 1) }' )
+    USERN=$( echo $USER | awk '{ print substr($0, 2, 1) }' )
+    WORKDIR=$( realpath "/work/users/$USERO/$USERN/$USER/" )
+    USERDIR=$WORKDIR
+
+    # Load required modules
+    module purge
+    module load gcc/9.1.0
+    module load anaconda/2023.03
+    module load git
+elif [[ $1 == "rockfish" ]]; then
+    # Setup general purspose user variables needed for RockFish
+    WORKDIR=$( realpath "/scratch4/struelo1/flepimop-code/$USER/"  )
+    USERDIR=$WORKDIR
+    mkdir -vp $WORKDIR
+
+    # Load required modules
+    module purge
+    module load gcc/9.3.0
+    module load anaconda/2020.07
+    module load git/2.42.0
+else
+    echo "The cluster name '$1' is not recognized, must be one of: 'longleaf', 'rockfish'."
+    set +e
+    exit 1
+fi
+
+# Ensure we have a $FLEPI_PATH
+if [ -z "${FLEPI_PATH}" ]; then
+    echo -n "An explicit \$FLEPI_PATH was not provided, please set one (or press enter to use '$USERDIR/flepiMoP'): "
+    read FLEPI_PATH
+    if [ -z "${FLEPI_PATH}" ]; then
+        export FLEPI_PATH="$USERDIR/flepiMoP"
+    fi
+    export FLEPI_PATH=$( realpath "$FLEPI_PATH" )
+    echo "Using '$FLEPI_PATH' for \$FLEPI_PATH."
+fi
+
+# Test that flepiMoP is located there
+if [ ! -d "$FLEPI_PATH" ]; then
+    while true; do
+        read -p "Did not find flepiMoP at $FLEPI_PATH, do you want to clone the repo? (y/n) " resp
+        case "$resp" in
+            [yY])
+                echo "Cloning on your behalf."
+                git clone [email protected]:HopkinsIDD/flepiMoP.git $FLEPI_PATH
+                break
+                ;;
+            [nN])
+                echo "Then you need to set a \$FLEPI_PATH before running, cannot proceed with install."
+                set +e
+                exit 1
+                ;;
+            *)
+                echo "Invalid input. Please enter 'y' or 'n'. "
+                ;;
+        esac
+    done
+fi
+
+# Setup the conda environment
+if [ -z "${FLEPI_CONDA}" ]; then
+    echo -n "An explicit \$FLEPI_CONDA was not provided, please set one (or press enter to use 'flepimop-env'): "
+    read FLEPI_CONDA
+    if [ -z "${FLEPI_CONDA}" ]; then
+        export FLEPI_CONDA="flepimop-env"
+    fi
+    echo "Using '$FLEPI_CONDA' for \$FLEPI_CONDA."
+fi
+FLEPI_CONDA_ENV_MATCHES=$( conda info --envs | awk '{print $1}' | grep -x "$FLEPI_CONDA" | wc -l )
+if [ "$FLEPI_CONDA_ENV_MATCHES" -eq 0 ]; then
+conda env create --name $FLEPI_CONDA --file $FLEPI_PATH/environment.yml
+fi
+
+# Load the conda environment
+conda activate $FLEPI_CONDA
+[ -e "$CONDA_PREFIX/conda-meta/pinned" ] && rm $CONDA_PREFIX/conda-meta/pinned
+cat << EOF > $CONDA_PREFIX/conda-meta/pinned
+r-arrow==17.0.0
+arrow==17.0.0
+EOF
+
+# Install the gempyor package from local
+pip install --editable $FLEPI_PATH/flepimop/gempyor_pkg
+
+# Install the local R packages
+R -e "install.packages('covidcast', repos='https://cloud.r-project.org')"
+RETURNTO=$( pwd )
+cd $FLEPI_PATH/flepimop/R_packages/
+for d in $( ls ); do
+    R CMD INSTALL $d
+done
+cd $RETURNTO
+R -e "library(inference); inference::install_cli()"
+
+# Done
+echo "> Done installing/updating flepiMoP."
+set +e

environment.yml

@@ -0,0 +1,39 @@
+# Fri Oct 18 18:29:25 2024 UTC
+channels:
+- conda-forge
+- defaults
+- r
+- dnachun
+dependencies:
+- python=3.11
+- pip
+- r-base>=4.3
+- pyarrow=17.0.0
+- r-arrow=17.0.0
+- r-sf
+- r-data.table
+- r-doParallel
+- r-dplyr
+- r-foreach
+- r-ggplot2
+- r-ggraph
+- r-httr
+- r-jsonlite
+- r-lubridate
+- r-magrittr
+- r-MMWRweek
+- r-optparse
+- r-purrr
+- r-readr
+- r-reticulate
+- r-rlang
+- r-stringr
+- r-tibble
+- r-tidygraph
+- r-tidyr
+- r-tidyselect
+- r-tidyverse
+- r-truncnorm
+- r-vroom
+- r-xts
+- r-yaml