From 5b5303c13898be55a58273e304b2801d6eff2302 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 26 Sep 2024 10:19:51 -0400 Subject: [PATCH 1/7] initial tweaks to make flepimop-inference-* runnable --- batch/AWS_inference_runner.sh | 8 +++---- batch/SLURM_inference_job.run | 10 ++++----- .../quick-start-guide-conda.md | 10 ++++----- .../advanced-run-guides/running-on-aws.md | 2 +- .../running-with-docker-locally.md | 8 +++---- .../slurm-submission-on-marcc.md | 2 +- .../gitbook/how-to-run/quick-start-guide.md | 10 ++++----- .../running-with-docker-locally.md | 2 +- .../slurm-submission-on-marcc.md | 2 +- flepimop/R_packages/inference/R/install_cli.R | 22 +++++++++++++++++++ .../inst/scripts/flepimop-inference-main.R} | 8 ++++--- .../inst/scripts/flepimop-inference-slot.R} | 12 +++++----- flepimop/gempyor_pkg/docs/Rinterface.Rmd | 4 ++-- flepimop/gempyor_pkg/docs/Rinterface.html | 4 ++-- .../docs/integration_benchmark.ipynb | 2 +- 15 files changed, 66 insertions(+), 40 deletions(-) create mode 100644 flepimop/R_packages/inference/R/install_cli.R rename flepimop/{main_scripts/inference_main.R => R_packages/inference/inst/scripts/flepimop-inference-main.R} (97%) mode change 100644 => 100755 rename flepimop/{main_scripts/inference_slot.R => R_packages/inference/inst/scripts/flepimop-inference-slot.R} (99%) mode change 100644 => 100755 diff --git a/batch/AWS_inference_runner.sh b/batch/AWS_inference_runner.sh index c337cdb65..79fb0574c 100755 --- a/batch/AWS_inference_runner.sh +++ b/batch/AWS_inference_runner.sh @@ -145,14 +145,14 @@ echo "---" find data echo "===" -echo "***************** RUNNING inference_slot.R *****************" +echo "***************** RUNNING flepimop-inference-slot *****************" -Rscript flepiMoP/flepimop/main_scripts/inference_slot.R -p flepiMoP +flepimop-inference-slot -p flepiMoP dvc_ret=$? if [ $dvc_ret -ne 0 ]; then - error_handler "Error code returned from inference_main.R: $dvc_ret" + error_handler "Error code returned from flepimop-inference-slot: $dvc_ret" fi -echo "***************** DONE RUNNING inference_slot.R *****************" +echo "***************** DONE RUNNING flepimop-inference-slot *****************" echo "***************** UPLOADING RESULT TO S3 *****************" for type in "seir" "hosp" "llik" "spar" "snpi" "hnpi" "hpar" diff --git a/batch/SLURM_inference_job.run b/batch/SLURM_inference_job.run index 6cf44870e..6832b1667 100644 --- a/batch/SLURM_inference_job.run +++ b/batch/SLURM_inference_job.run @@ -136,9 +136,9 @@ fi ls -ltr model_output echo "***************** DONE FETCHING RESUME FILES *****************" -echo "***************** RUNNING INFERENCE_MAIN.R *****************" +echo "***************** RUNNING flepimop-inference-slot *****************" export LOG_FILE="$FS_RESULTS_PATH/log_${FLEPI_RUN_INDEX}_${FLEPI_SLOT_INDEX}.txt" -echo "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R --config $CONFIG_PATH # path to the config file +echo "flepimop-inference-slot --config $CONFIG_PATH # path to the config file --run_id $FLEPI_RUN_INDEX # Unique identifier for this run --seir_modifiers_scenarios $FLEPI_SEIR_SCENARIOS # name of the intervention to run, or 'all' --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS # name of the outcome scenarios to run, or 'all' @@ -155,12 +155,12 @@ echo "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R --config $CONFI --is-resume $RESUME_RUN # Is this run a resume --is-interactive FALSE # Is this run an interactive run" #> $LOG_FILE 2>&1 & -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_slot.R -p $FLEPI_PATH --config $CONFIG_PATH --run_id $FLEPI_RUN_INDEX --seir_modifiers_scenarios $FLEPI_SEIR_SCENARIOS --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS --jobs 1 --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT --this_slot $FLEPI_SLOT_INDEX --this_block 1 --stoch_traj_flag $FLEPI_STOCHASTIC_RUN --is-resume $RESUME_RUN --is-interactive FALSE #> $LOG_FILE 2>&1 +flepimop-inference-slot -p $FLEPI_PATH --config $CONFIG_PATH --run_id $FLEPI_RUN_INDEX --seir_modifiers_scenarios $FLEPI_SEIR_SCENARIOS --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS --jobs 1 --iterations_per_slot $FLEPI_ITERATIONS_PER_SLOT --this_slot $FLEPI_SLOT_INDEX --this_block 1 --stoch_traj_flag $FLEPI_STOCHASTIC_RUN --is-resume $RESUME_RUN --is-interactive FALSE #> $LOG_FILE 2>&1 dvc_ret=$? if [[ $dvc_ret -ne 0 ]]; then - echo "Error code returned from inference_slot.R: $dvc_ret" + echo "Error code returned from flepimop-inference-slot: $dvc_ret" fi -echo "***************** DONE RUNNING INFERENCE_SLOT.R *****************" +echo "***************** DONE flepimop-inference-slot *****************" echo "***************** UPLOADING RESULT TO S3 (OR NOT) *****************" diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md b/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md index 7bb822683..0d838727f 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md @@ -162,7 +162,7 @@ rm -r model_output/ # delete the outputs of past run if there are An inference run requires a configuration file that has an `inference` section. Stay in the `$DATA_PATH` folder, and run the inference script, providing the name of the configuration file you want to run (ex. `config.yml`). In the example data folder (flepimop\_sample), try out the example config XXX. ```bash -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config.yml +$ flepimop-inference-main.R -c config.yml ``` This will run the model and create [a lot of output files](../../gempyor/output-files.md) in `$DATA_PATH/model_output/`. @@ -179,14 +179,14 @@ The last few lines visible on the command prompt should be: If you want to quickly do runs with options different from those encoded in the configuration file, you can do that from the command line, for example -``` -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -j 1 -n 1 -k 1 -c config.yml +```bash +$ flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml ``` where: * `n` is the number of parallel inference slots, -* `j` is the number of CPU cores to use on your machine (if `j` > `n`, only `n` cores will actually be used. If `j` <`n`, some cores will run multiple slots in sequence) +* `j` is the number of CPU cores to use on your machine (if `j` > `n`, only `n` cores will actually be used. If `j` < `n`, some cores will run multiple slots in sequence) * `k` is the number of iterations per slots. #### Non-inference run @@ -220,6 +220,6 @@ cd $DATA_PATH rm -rf model_output export CONFIG_PATH=config.yml # set your configuration file path -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R.R -j 1 -n 1 -k 1 +flepimop-inference-main -j 1 -n 1 -k 1 ``` {% endcode %} diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md b/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md index 3f5bd5576..357df5cd7 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md @@ -199,7 +199,7 @@ Rscript $FLEPI_PATH/datasetup/build_flu_data.R Now you may want to test that it works : ```bash -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c $CONFIG_PATH -j 1 -n 1 -k 1 +$ flepimop-inference-main -c $CONFIG_PATH -j 1 -n 1 -k 1 ``` If this fails, you may want to investigate this error. In case this succeeds, then you can proceed by first deleting the model\_output: diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md b/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md index fef0d3666..948c7dfdd 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md @@ -146,7 +146,7 @@ rm -r model_output/ # delete the outputs of past run if there are An inference run requires a configuration file that has the `inference` section. Stay in the `$DATA_PATH` folder, and run the inference script, providing the name of the configuration file you want to run (ex. `config.yml`) ```bash -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config.yml +$ flepimop-inference-main -c config.yml ``` This will run the model and create a lot of output files in `$DATA_PATH/model_output/`. @@ -163,8 +163,8 @@ The last few lines visible on the command prompt should be: If you want to quickly do runs with options different from those encoded in the configuration file, you can do that from the command line, for example -``` -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -j 1 -n 1 -k 1 -c config.yml +```bash +flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml ``` where: @@ -187,7 +187,7 @@ Rscript build/local_install.R pip install --no-deps -e flepimop/gempyor_pkg/ cd $DATA_PATH rm -rf model_output -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -j 1 -n 1 -k 1 -c config.yml +flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml ### Non-inference run diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/slurm-submission-on-marcc.md b/documentation/gitbook/how-to-run/advanced-run-guides/slurm-submission-on-marcc.md index a862f666b..30e99d0ff 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/slurm-submission-on-marcc.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/slurm-submission-on-marcc.md @@ -184,7 +184,7 @@ export CONFIG_PATH=config_example.yml # TO DO: ADD AN EXAMPLE You may want to test that it works before launching a full batch: ```bash -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c $CONFIG_PATH -j 1 -n 1 -k 1 +flepimop-inference-main -c $CONFIG_PATH -j 1 -n 1 -k 1 ``` If this fails, you may want to investigate this error. In case this succeeds, then you can proceed (but remember to delete the existing model output). diff --git a/documentation/gitbook/how-to-run/quick-start-guide.md b/documentation/gitbook/how-to-run/quick-start-guide.md index 869e519b3..0772820c2 100644 --- a/documentation/gitbook/how-to-run/quick-start-guide.md +++ b/documentation/gitbook/how-to-run/quick-start-guide.md @@ -172,7 +172,7 @@ An inference run requires a configuration file that has the `inference` section. {% code overflow="wrap" %} ```bash -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config_sample_2pop_inference.yml +flepimop-inference-main.R -c config_sample_2pop_inference.yml ``` {% endcode %} @@ -190,8 +190,8 @@ The last few lines visible on the command prompt should be: If you want to quickly do runs with options different from those encoded in the configuration file, you can do that from the command line, for example -``` -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -j 1 -n 1 -k 1 -c config_inference.yml +```bash +flepimop-inference-main.R -j 1 -n 1 -k 1 -c config_inference.yml ``` where: @@ -204,7 +204,7 @@ Again, it is helpful to run the model output notebook (`model_output_notebook.Rm The first time you run all this, it's , it's better to run each command individually as described above to be sure each exits successfully. However, eventually you can **put all these steps together in a script**, like below -``` +```bash export FLEPI_PATH=/Users/YourName/Github/flepiMoP export PROJECT_PATH=/Users/YourName/Github/flepiMoP_sample cd $FLEPI_PATH @@ -212,7 +212,7 @@ pip install --no-deps -e flepimop/gempyor_pkg/ Rscript build/local_install.R cd $PROJECT_PATH rm -rf model_output -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config_inference.yml +flepimop-inference-main -c config_inference.yml ``` Note that you only have to re-run the installation steps once each time you update any of the files in the flepimop repository (either by pulling changes made by the developers and stored on Github, or by changing them yourself). If you're just running the same or different configuration file, just repeat the final steps diff --git a/documentation/gitbook/jhu-internal/us-specific-how-to-run/running-with-docker-locally.md b/documentation/gitbook/jhu-internal/us-specific-how-to-run/running-with-docker-locally.md index 18c0c8bed..5edab8276 100644 --- a/documentation/gitbook/jhu-internal/us-specific-how-to-run/running-with-docker-locally.md +++ b/documentation/gitbook/jhu-internal/us-specific-how-to-run/running-with-docker-locally.md @@ -99,7 +99,7 @@ Stay in `$DATA_PATH`, select a config, and build the setup. The setup creates th export CONFIG_PATH=config_SMH_R1_lowVac_optImm_2022.yml Rscript $FLEPI_PATH/datasetup/build_US_setup.R Rscript $FLEPI_PATH/datasetup/build_flu_data.R -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R.R -j 1 -n 1 -k 1 +flepimop-inference-main -j 1 -n 1 -k 1 ``` where: diff --git a/documentation/gitbook/jhu-internal/us-specific-how-to-run/slurm-submission-on-marcc.md b/documentation/gitbook/jhu-internal/us-specific-how-to-run/slurm-submission-on-marcc.md index ade236593..5ff12f760 100644 --- a/documentation/gitbook/jhu-internal/us-specific-how-to-run/slurm-submission-on-marcc.md +++ b/documentation/gitbook/jhu-internal/us-specific-how-to-run/slurm-submission-on-marcc.md @@ -248,7 +248,7 @@ export FLEPI_MEM_PROF_ITERS=50 Now you may want to test that it works : ```bash -Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c $CONFIG_PATH -j 1 -n 1 -k 1 +flepimop-inference-main -c $CONFIG_PATH -j 1 -n 1 -k 1 ``` If this fails, you may want to investigate this error. In case this succeeds, then you can proceed by first deleting the model\_output: diff --git a/flepimop/R_packages/inference/R/install_cli.R b/flepimop/R_packages/inference/R/install_cli.R new file mode 100644 index 000000000..b4fbe2da0 --- /dev/null +++ b/flepimop/R_packages/inference/R/install_cli.R @@ -0,0 +1,22 @@ + +#' @title Install Inference Scripts +#' +#' @description +#' Installs the scripts for R-based FlepiMoP inference. +#' +#' @param path The path to install the scripts to. Default is `usr/local/bin` (unix-like). +#' +#' @param overwrite Whether to overwrite existing scripts. Default is `TRUE`. see [base::file.copy()]. +#' +#' @export +install_cli <- function( + path = if (.Platform$OS.type == "unix") file.path("usr", "local", "bin") else stop("Unsupported OS"), + overwrite = TRUE +) { + scriptfiles <- list.files( + system.file("scripts", package = utils::packageName()), pattern = "flepimop-.*", full.names = TRUE + ) + from <- scriptfiles + to <- file.path(path, gsub("\.R$", "", basename(scriptfiles))) + file.copy(from, to, overwrite = overwrite) +} \ No newline at end of file diff --git a/flepimop/main_scripts/inference_main.R b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R old mode 100644 new mode 100755 similarity index 97% rename from flepimop/main_scripts/inference_main.R rename to flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R index 894ea3719..8f9dd5132 --- a/flepimop/main_scripts/inference_main.R +++ b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R @@ -1,3 +1,5 @@ +#! /usr/bin/env Rscript + # About ------------------------------------------------------------------------ ## This script processes the options for an inference run and then creates a separate parallel processing job for each combination of SEIR parameter modification scenario, outcome parameter modification scenario, and independent MCMC chain ("slot") @@ -11,8 +13,8 @@ suppressMessages(library(parallel)) suppressMessages(library(doParallel)) options(readr.num_columns = 0) -# There are multiple ways to specify options when inference_main.R is run, which take the following precedence: -# 1) (optional) options called along with the script at the command line (ie > Rscript inference_main.R -c my_config.yml) +# There are multiple ways to specify options when flepimop-inference-main is run, which take the following precedence: +# 1) (optional) options called along with the script at the command line (ie > flepimop-inference-main -c my_config.yml) # 2) (optional) environmental variables set by the user (ie user could set > export CONFIG_PATH = ~/flepimop_sample/my_config.yml to not have t specify it each time the script is run) # If neither are specified, then a default value is used, given by the second argument of Sys.getenv() commands below. # *3) For some options, a default doesn't exist, and the value specified in the config will be used if the option is not specified at the command line or by an environmental variable (iterations_per_slot, slots) @@ -126,7 +128,7 @@ foreach(seir_modifiers_scenario = seir_modifiers_scenarios) %:% err <- system( paste( opt$rpath, - file.path(opt$flepi_path, "flepimop", "main_scripts","inference_slot.R"), + "flepimop-inference-slot", "-c", opt$config, "-u", opt$run_id, "-s", opt$seir_modifiers_scenarios, diff --git a/flepimop/main_scripts/inference_slot.R b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R old mode 100644 new mode 100755 similarity index 99% rename from flepimop/main_scripts/inference_slot.R rename to flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R index 04ce4316e..cb8d40d64 --- a/flepimop/main_scripts/inference_slot.R +++ b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R @@ -1,6 +1,8 @@ +#! /usr/bin/env Rscript + # About -## This script runs a single slot (MCMC chain) of an inference run. It can be called directly, but is often called from inference_main.R if multiple slots are run. +## This script runs a single slot (MCMC chain) of an inference run. It can be called directly, but is often called from flepimop-inference-main if multiple slots are run. # Run Options --------------------------------------------------------------------- @@ -24,8 +26,8 @@ required_packages <- c("dplyr", "magrittr", "xts", "zoo", "stringr") #set.seed(1) # set within R #reticulate::py_run_string(paste0("rng_seed = ", 1)) #set within Python -# There are multiple ways to specify options when inference_slot.R is run, which take the following precedence: -# 1) (optional) options called along with the script at the command line (ie > Rscript inference_main.R -c my_config.yml) +# There are multiple ways to specify options when flepimop-inference-slot is run, which take the following precedence: +# 1) (optional) options called along with the script at the command line (ie > flepimop-inference-slot -c my_config.yml) # 2) (optional) environmental variables set by the user (ie user could set > export CONFIG_PATH = ~/flepimop_sample/my_config.yml to not have t specify it each time the script is run) # If neither are specified, then a default value is used, given by the second argument of Sys.getenv() commands below. # *3) For some options, a default doesn't exist, and the value specified in the config will be used if the option is not specified at the command line or by an environmental variable (iterations_per_slot, slots) @@ -451,7 +453,7 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { autowrite_seir = TRUE ) }, error = function(e) { - print("GempyorInference failed to run (call on l. 443 of inference_slot.R).") + print("GempyorInference failed to run (call on l. 443 of flepimop-inference-slot).") print("Here is all the debug information I could find:") for(m in reticulate::py_last_error()) print(m) stop("GempyorInference failed to run... stopping") @@ -622,7 +624,7 @@ for(seir_modifiers_scenario in seir_modifiers_scenarios) { load_ID=TRUE, sim_id2load=this_index) }, error = function(e) { - print("GempyorInference failed to run (call on l. 575 of inference_slot.R).") + print("GempyorInference failed to run (call on l. 575 of flepimop-inference-sl).") print("Here is all the debug information I could find:") for(m in reticulate::py_last_error()) print(m) stop("GempyorInference failed to run... stopping") diff --git a/flepimop/gempyor_pkg/docs/Rinterface.Rmd b/flepimop/gempyor_pkg/docs/Rinterface.Rmd index 0dffe6097..4d6b13ba0 100644 --- a/flepimop/gempyor_pkg/docs/Rinterface.Rmd +++ b/flepimop/gempyor_pkg/docs/Rinterface.Rmd @@ -175,7 +175,7 @@ here if source_filters is [["age0to17"], ["OMICRON", "WILD"]], it means filter ( ![while readable_graph look like this](./readable_graph.pdf){width=100% height=200} ## Simulate -The code in this section is very much like what is present in inference_slot.R for the inference runs: it runs the full gempyor (parameters, npi, simulation, outcomes). +The code in this section is very much like what is present in flepimop-inference-slot for the inference runs: it runs the full gempyor (parameters, npi, simulation, outcomes). These functions take as input a `seed` file with sim_id `sim_id2write`, and outputs files hpar, spar, snpi, hnpi, seir, hosp with sim_id `sim_id2write`. In case the flag load_ID is set as true, the function will also load from file the seeding, parameters and the NPIs (seed, hnpi, snpi, hpar, spar) from the simulation with sim_id `sim_id2load`. Here we simulate once from config, then a second time using the same values as generated by the first run: @@ -197,7 +197,7 @@ npi_outcome = gempyor_inference$get_outcome_npi(bypass_FN = 'model_output/hnpi/t ## Also... -Additional methods are provided, one of which updates the prefix, which is done by e.g inference_slot.R to indicate the slot +Additional methods are provided, one of which updates the prefix, which is done by e.g flepimop-inference-slot to indicate the slot ```{r} gempyor_inference$update_prefix(new_prefix="my new prefix") ``` diff --git a/flepimop/gempyor_pkg/docs/Rinterface.html b/flepimop/gempyor_pkg/docs/Rinterface.html index 784ffb634..6aae264d9 100644 --- a/flepimop/gempyor_pkg/docs/Rinterface.html +++ b/flepimop/gempyor_pkg/docs/Rinterface.html @@ -376,7 +376,7 @@

Get compartment graph image

Simulate

-

The code in this section is very much like what is present in inference_slot.R for the inference runs: it runs the full gempyor (parameters, npi, simulation, outcomes). These functions take as input a seed file with sim_id sim_id2write, and outputs files hpar, spar, snpi, hnpi, seir, hosp with sim_id sim_id2write. In case the flag load_ID is set as true, the function will also load from file the seeding, parameters and the NPIs (seed, hnpi, snpi, hpar, spar) from the simulation with sim_id sim_id2load.

+

The code in this section is very much like what is present in flepimop-inference-slot for the inference runs: it runs the full gempyor (parameters, npi, simulation, outcomes). These functions take as input a seed file with sim_id sim_id2write, and outputs files hpar, spar, snpi, hnpi, seir, hosp with sim_id sim_id2write. In case the flag load_ID is set as true, the function will also load from file the seeding, parameters and the NPIs (seed, hnpi, snpi, hpar, spar) from the simulation with sim_id sim_id2load.

Here we simulate once from config, then a second time using the same values as generated by the first run:

gempyor_inference$one_simulation(sim_id2write=0)                                  # simulate from config
## [1] 0
@@ -391,7 +391,7 @@

Little redudant example:

Also…

-

Additional methods are provided, one of which updates the prefix, which is done by e.g inference_slot.R to indicate the slot

+

Additional methods are provided, one of which updates the prefix, which is done by e.g flepimop-inference-slot to indicate the slot

gempyor_inference$update_prefix(new_prefix="my new prefix")
diff --git a/flepimop/gempyor_pkg/docs/integration_benchmark.ipynb b/flepimop/gempyor_pkg/docs/integration_benchmark.ipynb index 0a011dbd9..ce2880787 100644 --- a/flepimop/gempyor_pkg/docs/integration_benchmark.ipynb +++ b/flepimop/gempyor_pkg/docs/integration_benchmark.ipynb @@ -43,7 +43,7 @@ "export CONFIG_PATH=config_smh_r11_optsev_highie_base_deathscases_blk1.yml\n", "Rscript $FLEPI_PATH/datasetup/build_US_setup.R\n", "Rscript $FLEPI_PATH/flepimop/main_scripts/create_seeding.R\n", - "Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -j 1 -n 1 -k 1\n", + "flepimop-inference-main.R -j 1 -n 1 -k 1\n", "```\n", "to get the `run_id`, `prefix` and a seed file. Then copy model_output in the folder of this notebook to get a real seed file " ] From 5fbb8fbce07864bb78ba6b059450259c22bc218d Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 26 Sep 2024 11:33:07 -0400 Subject: [PATCH 2/7] further install scripts fixes --- .../advanced-run-guides/quick-start-guide-conda.md | 4 ++-- .../how-to-run/advanced-run-guides/running-on-aws.md | 2 +- .../advanced-run-guides/running-with-docker-locally.md | 2 +- flepimop/R_packages/inference/DESCRIPTION | 2 +- flepimop/R_packages/inference/NAMESPACE | 2 +- flepimop/R_packages/inference/R/install_cli.R | 7 +++---- .../inference/inst/scripts/flepimop-inference-main.R | 2 +- .../inference/inst/scripts/flepimop-inference-slot.R | 2 +- 8 files changed, 11 insertions(+), 12 deletions(-) diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md b/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md index 0d838727f..8609858a9 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md @@ -162,7 +162,7 @@ rm -r model_output/ # delete the outputs of past run if there are An inference run requires a configuration file that has an `inference` section. Stay in the `$DATA_PATH` folder, and run the inference script, providing the name of the configuration file you want to run (ex. `config.yml`). In the example data folder (flepimop\_sample), try out the example config XXX. ```bash -$ flepimop-inference-main.R -c config.yml +flepimop-inference-main.R -c config.yml ``` This will run the model and create [a lot of output files](../../gempyor/output-files.md) in `$DATA_PATH/model_output/`. @@ -180,7 +180,7 @@ The last few lines visible on the command prompt should be: If you want to quickly do runs with options different from those encoded in the configuration file, you can do that from the command line, for example ```bash -$ flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml +flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml ``` where: diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md b/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md index 357df5cd7..cd3feb15b 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/running-on-aws.md @@ -199,7 +199,7 @@ Rscript $FLEPI_PATH/datasetup/build_flu_data.R Now you may want to test that it works : ```bash -$ flepimop-inference-main -c $CONFIG_PATH -j 1 -n 1 -k 1 +flepimop-inference-main -c $CONFIG_PATH -j 1 -n 1 -k 1 ``` If this fails, you may want to investigate this error. In case this succeeds, then you can proceed by first deleting the model\_output: diff --git a/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md b/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md index 948c7dfdd..e78d90775 100644 --- a/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md +++ b/documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md @@ -146,7 +146,7 @@ rm -r model_output/ # delete the outputs of past run if there are An inference run requires a configuration file that has the `inference` section. Stay in the `$DATA_PATH` folder, and run the inference script, providing the name of the configuration file you want to run (ex. `config.yml`) ```bash -$ flepimop-inference-main -c config.yml +flepimop-inference-main -c config.yml ``` This will run the model and create a lot of output files in `$DATA_PATH/model_output/`. diff --git a/flepimop/R_packages/inference/DESCRIPTION b/flepimop/R_packages/inference/DESCRIPTION index 33363b166..cecd1ee72 100644 --- a/flepimop/R_packages/inference/DESCRIPTION +++ b/flepimop/R_packages/inference/DESCRIPTION @@ -21,6 +21,6 @@ Imports: reticulate, truncnorm, arrow -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.2 Suggests: testthat diff --git a/flepimop/R_packages/inference/NAMESPACE b/flepimop/R_packages/inference/NAMESPACE index 2e54f6c45..2a112e464 100644 --- a/flepimop/R_packages/inference/NAMESPACE +++ b/flepimop/R_packages/inference/NAMESPACE @@ -15,6 +15,7 @@ export(getStats) export(get_ground_truth) export(get_ground_truth_file) export(initialize_mcmc_first_block) +export(install_cli) export(iterateAccept) export(logLikStat) export(perform_MCMC_step_copies_chimeric) @@ -25,6 +26,5 @@ export(perturb_hnpi_from_file) export(perturb_hpar) export(perturb_seeding) export(perturb_snpi) -export(perturb_init) export(perturb_snpi_from_file) importFrom(magrittr,"%>%") diff --git a/flepimop/R_packages/inference/R/install_cli.R b/flepimop/R_packages/inference/R/install_cli.R index b4fbe2da0..28193e1db 100644 --- a/flepimop/R_packages/inference/R/install_cli.R +++ b/flepimop/R_packages/inference/R/install_cli.R @@ -10,13 +10,12 @@ #' #' @export install_cli <- function( - path = if (.Platform$OS.type == "unix") file.path("usr", "local", "bin") else stop("Unsupported OS"), - overwrite = TRUE + path = if (.Platform$OS.type == "unix") normalizePath(file.path("/usr", "local", "bin")) else stop("Unsupported OS") ) { scriptfiles <- list.files( system.file("scripts", package = utils::packageName()), pattern = "flepimop-.*", full.names = TRUE ) from <- scriptfiles - to <- file.path(path, gsub("\.R$", "", basename(scriptfiles))) - file.copy(from, to, overwrite = overwrite) + to <- file.path(path, gsub("\\.R$", "", basename(scriptfiles))) + file.symlink(from, to) } \ No newline at end of file diff --git a/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R index 8f9dd5132..7b22b6ce8 100755 --- a/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R +++ b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R @@ -1,4 +1,4 @@ -#! /usr/bin/env Rscript +#!/usr/bin/env Rscript # About ------------------------------------------------------------------------ diff --git a/flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R index cb8d40d64..c1ef1ae6d 100755 --- a/flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R +++ b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-slot.R @@ -1,4 +1,4 @@ -#! /usr/bin/env Rscript +#!/usr/bin/env Rscript # About From 04c55edb97225454ceeac5a659312a882c009eca Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 26 Sep 2024 11:48:59 -0400 Subject: [PATCH 3/7] fix reinvocation of inference-slot --- .../R_packages/inference/inst/scripts/flepimop-inference-main.R | 1 - 1 file changed, 1 deletion(-) diff --git a/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R index 7b22b6ce8..a847d13a9 100755 --- a/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R +++ b/flepimop/R_packages/inference/inst/scripts/flepimop-inference-main.R @@ -127,7 +127,6 @@ foreach(seir_modifiers_scenario = seir_modifiers_scenarios) %:% err <- system( paste( - opt$rpath, "flepimop-inference-slot", "-c", opt$config, "-u", opt$run_id, From e6c711bfab365e201502867d476e73b231ce5117 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Wed, 25 Sep 2024 16:32:33 -0400 Subject: [PATCH 4/7] initial installation for ubuntu re-org --- README.md | 16 ++++++++++++++++ build/setup.R | 22 ++++++++++++++++++++++ install_ubuntu.sh | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100755 build/setup.R create mode 100755 install_ubuntu.sh diff --git a/README.md b/README.md index 86a4ce4b6..53c103207 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ + +# FlepiMoP + +The *Fle*xible *Epi*demic *Mo*deling *P*ipeline, `FlepiMoP`, makes it easy to build an infectious disease model, infer that model's parameters, and project scenario outcomes. + +# Quickstart + +```bash +$ mkdir myflepimopworkspace && cd $_ +$ git clone git@github.com:HopkinsIDD/flepiMoP.git --depth 1 +$ ./flepiMop/install_ubuntu.sh +$ cp -r ./flepiMoP/examples/tutorial_two_subpops test_model && cd $_ +$ gempyor-simulate -c config_sample_2pop.yml +$ $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config_sample_2pop_inference.yml +``` + # flepiMoP Welcome to the Johns Hopkins University Infectious Disease Dynamics COVID-19 Working Group's `Flexible Epidemic Modeling Pipeline`(“FlepiMoP”, formerly the COVID Scenario Pipeline, “CSP”), a flexible modeling framework that projects epidemic trajectories and healthcare impacts under different suites of interventions in order to aid in scenario planning. The model is generic enough to be applied to different spatial scales given shapefiles, population data, and COVID-19 confirmed case data. There are multiple components to the pipeline, which may be characterized as follows: 1) epidemic seeding; 2) disease transmission and non-pharmaceutical intervention scenarios; 3) calculation of health outcomes (hospital and ICU admissions and bed use, ventilator use, and deaths); and 4) summarization of model outputs. diff --git a/build/setup.R b/build/setup.R new file mode 100755 index 000000000..c8879a156 --- /dev/null +++ b/build/setup.R @@ -0,0 +1,22 @@ +#!/usr/bin/env Rscript + +# TODO sniff for upgrade mode + +if (!require(remotes)) { + install.packages("remotes", repos = c(getOption("repos"), "http://cran.r-project.org")); + stopifnot("Could not load `remotes` package." = require(remotes)) +} + +for (pkg in c("flepicommon", "flepiconfig", "inference")) { + if (!requireNamespace(pkg, quietly = TRUE)) remotes::install_github("HopkinsIDD/flepiMoP", subdir = sprintf("flepimop/R_packages/%s", pkg)) +} + +# other dependencies for analysis scripts +for (pkg in c("ggfortify", "flextable", "optparse", "cowplot")) { + if (!requireNamespace(pkg, quietly = TRUE)) { + install.packages(pkg, repos = c(getOption("repos"), "http://cran.r-project.org")); + if (!requireNamespace(pkg)) { + stop(sprintf("Could not install and/or load `%s` package.", pkg)) + } + } +} diff --git a/install_ubuntu.sh b/install_ubuntu.sh new file mode 100755 index 000000000..ac6a57d76 --- /dev/null +++ b/install_ubuntu.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +script_path=$(dirname $(realpath $0)) + +# ensure FLEPI_PATH available as environment variable + +# if FLEPI_PATH is not equal to script path, set or update it +if [ "$FLEPI_PATH" != $script_path ]; then + if [ -z "$FLEPI_PATH" ]; then + echo "FLEPI_PATH not set; updating ~/.bashrc ..." + export FLEPI_PATH=$script_path + sudo echo "export FLEPI_PATH=$FLEPI_PATH" >> ~/.bashrc + sudo echo "export ''" >> ~/.bashrc + echo "appended to ./bashrc to set FLEPI_PATH=$FLEPI_PATH" + else + echo "current FLEPI_PATH=$FLEPI_PATH - updating ..." + export FLEPI_PATH=$script_path + sudo sed 's,FLEPI_PATH=[^;]*,"FLEPI_PATH=$script_path",' -i ~/.bashrc + echo "updated ./bashrc to set FLEPI_PATH=$FLEPI_PATH" + fi +else + echo "FLEPI_PATH=$FLEPI_PATH environment variable already set." + # TODO: ensure that FLEPI_PATH is set in bashrc? bash_profile? +fi + +# install necessary system dependencies +sudo apt install libudunits2-dev libssl-dev libfontconfig1-dev libxml2-dev libcurl4-openssl-dev libharfbuzz-dev libfribidi-dev libgdal-dev libcairo2-dev + +# install the python package +pip install -e $FLEPI_PATH/flepimop/gempyor_pkg/ + +# install the R packages +$FLEPI_PATH/build/setup.R \ No newline at end of file From 0ea541a8772a42f31d960ba9e08f22a7e3b14380 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 26 Sep 2024 11:29:09 -0400 Subject: [PATCH 5/7] updates addressing use of installed r scripts --- build/setup.R | 15 +++++++++++++-- install_ubuntu.sh | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/build/setup.R b/build/setup.R index c8879a156..a3ba72e05 100755 --- a/build/setup.R +++ b/build/setup.R @@ -1,5 +1,11 @@ #!/usr/bin/env Rscript +.args <- commandArgs(trailingOnly = TRUE) + +if (length(.args) != 1) { + stop("Usage: setup.R ") +} + # TODO sniff for upgrade mode if (!require(remotes)) { @@ -7,8 +13,10 @@ if (!require(remotes)) { stopifnot("Could not load `remotes` package." = require(remotes)) } -for (pkg in c("flepicommon", "flepiconfig", "inference")) { - if (!requireNamespace(pkg, quietly = TRUE)) remotes::install_github("HopkinsIDD/flepiMoP", subdir = sprintf("flepimop/R_packages/%s", pkg)) +rpkgs <- list.files(file.path(.args[1], "flepimop", "R_packages"), full.names = TRUE) + +for (pkg in rpkgs) { + install.packages(pkg, repos = NULL, type = "source") } # other dependencies for analysis scripts @@ -20,3 +28,6 @@ for (pkg in c("ggfortify", "flextable", "optparse", "cowplot")) { } } } + +# install the R scripts as executables +inference::install_cli() \ No newline at end of file diff --git a/install_ubuntu.sh b/install_ubuntu.sh index ac6a57d76..84c909eeb 100755 --- a/install_ubuntu.sh +++ b/install_ubuntu.sh @@ -30,4 +30,4 @@ sudo apt install libudunits2-dev libssl-dev libfontconfig1-dev libxml2-dev libcu pip install -e $FLEPI_PATH/flepimop/gempyor_pkg/ # install the R packages -$FLEPI_PATH/build/setup.R \ No newline at end of file +sudo $FLEPI_PATH/build/setup.R $FLEPI_PATH \ No newline at end of file From f2b93044736d150fa678abd5504c212eda7252e8 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 26 Sep 2024 11:48:34 -0400 Subject: [PATCH 6/7] README revs --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 53c103207..ecb831113 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ The *Fle*xible *Epi*demic *Mo*deling *P*ipeline, `FlepiMoP`, makes it easy to bu # Quickstart ```bash -$ mkdir myflepimopworkspace && cd $_ -$ git clone git@github.com:HopkinsIDD/flepiMoP.git --depth 1 -$ ./flepiMop/install_ubuntu.sh -$ cp -r ./flepiMoP/examples/tutorial_two_subpops test_model && cd $_ -$ gempyor-simulate -c config_sample_2pop.yml -$ $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config_sample_2pop_inference.yml +mkdir myflepimopworkspace && cd $_ +git clone git@github.com:HopkinsIDD/flepiMoP.git --depth 1 +./flepiMop/install_ubuntu.sh +cp -r ./flepiMoP/examples/tutorial_two_subpops test_model && cd $_ +gempyor-simulate -c config_sample_2pop.yml +flepimop-inference-main -c config_sample_2pop_inference.yml ``` # flepiMoP From 814e6078e65a58ac67a6d4d007936d4fc08f4b29 Mon Sep 17 00:00:00 2001 From: "Carl A. B. Pearson" Date: Thu, 26 Sep 2024 12:30:40 -0400 Subject: [PATCH 7/7] add arrow installation --- install_ubuntu.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/install_ubuntu.sh b/install_ubuntu.sh index 84c909eeb..98b86653d 100755 --- a/install_ubuntu.sh +++ b/install_ubuntu.sh @@ -23,8 +23,18 @@ else # TODO: ensure that FLEPI_PATH is set in bashrc? bash_profile? fi -# install necessary system dependencies -sudo apt install libudunits2-dev libssl-dev libfontconfig1-dev libxml2-dev libcurl4-openssl-dev libharfbuzz-dev libfribidi-dev libgdal-dev libcairo2-dev +# per arrow instructions: https://arrow.apache.org/install, with some modifications +sudo apt update +sudo apt install -y -V ca-certificates lsb-release wget +wget -O arrow_latest.deb https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +sudo apt install -y -V arrow_latest.deb +rm arrow_latest.deb +sudo apt update +sudo apt install -y -V libparquet-dev # For Apache Parquet C++ +sudo apt install -y -V libparquet-glib-dev # For Apache Parquet GLib (C) + +# install other necessary system dependencies +sudo apt install -y -V libudunits2-dev libssl-dev libfontconfig1-dev libxml2-dev libcurl4-openssl-dev libharfbuzz-dev libfribidi-dev libgdal-dev libcairo2-dev # install the python package pip install -e $FLEPI_PATH/flepimop/gempyor_pkg/