Release cycle, December 2024

.github/workflows/conda-env.yml

@@ -7,12 +7,13 @@ on:
       - build/create_environment_yml.R
       - flepimop/R_packages/*/DESCRIPTION
     branches:
-      - main
+      - dev
   pull_request:
     paths:
       - build/create_environment_yml.R
       - flepimop/R_packages/*/DESCRIPTION
     branches:
+      - dev
       - main
 
 jobs:

.github/workflows/flepicommon-ci.yml

@@ -1,16 +1,17 @@
-name: flepicommon-ci
+name: flepicommon CI
 
 on:
   workflow_dispatch:
   push:
     paths:
       - flepimop/R_packages/flepicommon/**/*
     branches:
-      - main
+      - dev
   pull_request:
     paths:
       - flepimop/R_packages/flepicommon/**/*
     branches:
+      - dev
       - main
 
 jobs:

.github/workflows/gempyor-ci.yml

@@ -1,4 +1,4 @@
-name: gempyor-ci
+name: gempyor CI
 
 on:
   workflow_dispatch:
@@ -7,12 +7,13 @@ on:
       - examples/**/*
       - flepimop/gempyor_pkg/**/*
     branches:
-      - main
+      - dev
   pull_request:
     paths:
       - examples/**/*
       - flepimop/gempyor_pkg/**/*
     branches:
+      - dev
       - main
 
 jobs:

.github/workflows/inference-ci.yml

@@ -1,4 +1,4 @@
-name: inference-ci
+name: inference CI
 
 on:
   workflow_dispatch:
@@ -10,11 +10,12 @@ on:
     paths:
       - flepimop/R_packages/inference/**/*
     branches:
-      - main
+      - dev
   pull_request:
     paths:
       - flepimop/R_packages/inference/**/*
     branches:
+      - dev
       - main
 
 jobs:

.gitignore

@@ -64,7 +64,7 @@ packrat/lib*/
 dist/
 SEIR.egg-info/
 Outcomes.egg-info/
-venv/
+*venv*/
 .venv/
 
 # R package manuals

batch/hpc_init.sh

@@ -52,6 +52,14 @@ if [ -z "${FLEPI_CONDA}" ]; then
     fi
     echo "Using '$FLEPI_CONDA' for \$FLEPI_CONDA."
 fi
+CURRENT_CONDA_ENV=$( conda info | grep "active environment" | awk -F ':' '{print $2}' | xargs )
+if [ "$CURRENT_CONDA_ENV" = "$FLEPI_CONDA" ]; then
+    echo "Detected the activate conda environment is '$FLEPI_CONDA' already, but will refresh."
+    conda deactivate
+elif [ "$CURRENT_CONDA_ENV" != "None" ]; then
+    echo "Detected an active conda environment '$CURRENT_CONDA_ENV'. This will be deactivated and the '$FLEPI_CONDA' environment wil be activated."
+    conda deactivate
+fi
 conda activate $FLEPI_CONDA
 
 # Check the conda environment is valid

documentation/gitbook/SUMMARY.md

@@ -48,6 +48,7 @@
 
 * [Before any run](how-to-run/before-any-run.md)
 * [Quick Start Guide](how-to-run/quick-start-guide.md)
+* [Multiple Configuration Files](multi-configs.md)
 * [Advanced run guides](how-to-run/advanced-run-guides/README.md)
   * [Running with Docker locally 🛳](how-to-run/advanced-run-guides/running-with-docker-locally.md)
   * [Running locally in a conda environment 🐍](how-to-run/advanced-run-guides/quick-start-guide-conda.md)

documentation/gitbook/how-to-run/advanced-run-guides/quick-start-guide-conda.md

@@ -191,10 +191,10 @@ where:
 
 #### Non-inference run
 
-Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor`. To do this, call `gempyor-simulate` providing the name of the configuration file you want to run (ex. `config.yml`). An example config is provided in `flepimop_sample/config_sample_2pop_interventions.yml.`
+Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor`. To do this, call `flepimop simulate` providing the name of the configuration file you want to run (ex. `config.yml`). An example config is provided in `flepimop_sample/config_sample_2pop_interventions.yml.`
 
 ```
-gempyor-simulate -c config.yml
+flepimop simulate config.yml
 ```
 
 {% hint style="warning" %}

documentation/gitbook/how-to-run/advanced-run-guides/running-with-docker-locally.md

@@ -192,10 +192,10 @@ flepimop-inference-main -j 1 -n 1 -k 1 -c config.yml
 
 ### Non-inference run
 
-Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor,`call `gempyor-simulate` providing the name of the configuration file you want to run (ex. `config.yml` ;
+Stay in the `$DATA_PATH` folder, and run a simulation directly from forward-simulation Python package `gempyor,`call `flepimop simulate` providing the name of the configuration file you want to run (ex. `config.yml`):
 
 ```
-gempyor-simulate -c config.yml
+flepimop simulate config.yml
 ```
 
 {% hint style="warning" %}
@@ -216,7 +216,7 @@ Rscript build/local_install.R
 pip install --no-deps -e flepimop/gempyor_pkg/
 cd $DATA_PATH
 rm -rf model_output
-gempyor-simulate -c config.yml
+flepimop simulate config.yml
 </code></pre>
 
 ## Finishing up

documentation/gitbook/how-to-run/multi-configs.md

@@ -0,0 +1,77 @@
+---
+description: >-
+  Patching together multiple configuration files.
+---
+
+# Using Multiple Configuration Files
+
+## 🧱 Set up
+
+Create a sample project by copying from the examples folder:
+
+```bash
+mkdir myflepimopexample # or wherever
+cd myflepimopexample
+cp -r $FLEPI_PATH/examples/tutorials/* .
+ls
+```
+
+You should see an assortment of yml files as a result of that `ls` command.
+
+## Usage
+
+If you run
+
+```bash
+flepimop simulate config_sample_2pop.yml
+```
+
+you'll get a basic foward simulation of this example model. However, you might also note there are several `*_part.yml` files, corresponding to partial configs. You can `simulate` using the combination of multiple configs with, for example:
+
+```bash
+flepimop simulate config_sample_2pop.yml config_sample_2pop_outcomes_part.yml
+```
+
+if want to see what the combined configuration is, you can use the `patch` command:
+
+```bash
+flepimop patch config_sample_2pop.yml config_sample_2pop_outcomes_part.yml
+```
+
+You may provide an arbitrary number of separate configuration files to combine to create a complete configuration.
+
+## Caveats
+
+At this time, only `simulate` supports multiple configuration files. Also, the patching operation is fairly crude: configuration options override previous ones completely, though with a warning. The files provided from left to right are from lowest priority (i.e. for the first file, only options specified in no other files are used) to highest priority (i.e. for the last file, its options override any other specification).
+
+We are expanding coverage of this capability to other flepimop actions, e.g. inference, and are exploring options for smarter patching.
+
+However, currently there are pitfalls like
+
+```yaml
+# config1
+seir_modifiers:
+  scenarios: ["one", "two"]
+  one:
+    # ...
+  two:
+    # ...
+```
+
+```yaml
+# config2
+seir_modifiers:
+  scenarios: ["one", "three"]
+  one:
+    # ...
+  three:
+    # ...
+```
+
+Then you might expect
+
+```bash
+flepimop simulate config1.yml config2.yml
+```
+
+...to override seir scenario one and add scenario three, but what actually happens is that the entire seir_modifiers from config1 is overriden by config2. Specifying the configuration files in the reverse order would lead to a different outcome (the config1 seir_modifiers overrides config2 settings). If you're doing complex combinations of configuration files, you should use `flepimop patch ...` to ensure you're getting what you expect.

documentation/gitbook/how-to-run/quick-start-guide.md

@@ -170,10 +170,10 @@ To get started, let's start with just running a forward simulation (non-inferenc
 
 ### Non-inference run
 
-Stay in the `PROJECT_PATH` folder, and run a simulation directly from forward-simulation Python package gempyor. Call `gempyor-simulate` providing the name of the configuration file you want to run. For example here, we use `config_sample_2pop.yml` from _flepimop\_sample_.
+Stay in the `PROJECT_PATH` folder, and run a simulation directly from forward-simulation Python package gempyor. Call `flepimop simulate` providing the name of the configuration file you want to run. For example here, we use `config_sample_2pop.yml` from _flepimop\_sample_.
 
 ```
-gempyor-simulate -c config_sample_2pop.yml
+flepimop simulate config_sample_2pop.yml
 ```
 
 This will produce a `model_output` folder, which you can look using provided post-processing functions and scripts.
@@ -189,14 +189,14 @@ cd $FLEPI_PATH
 pip install --no-deps -e flepimop/gempyor_pkg/
 cd $PROJECT_PATH
 rm -rf model_output
-gempyor-simulate -c config.yml
+flepimop simulate config.yml
 ```
 
 Note that you only have to re-run the installation steps once each time you update any of the files in the flepimop repository (either by pulling changes made by the developers and stored on Github, or by changing them yourself). If you're just running the same or different configuration file, just repeat the final steps
 
 ```
 rm -rf model_output
-gempyor-simulate -c new_config.yml
+flepimop simulate new_config.yml
 ```
 
 ### Inference run
@@ -257,7 +257,7 @@ Rscript $FLEPI_PATH/flepimop/main_scripts/inference_main.R -c config_inference_n
 
 ## 📈 Examining model output
 
-If your run is successful, you should see your output files in the model\_output folder. The structure of the files in this folder is described in the [Model Output](../gempyor/output-files.md) section. By default, all the output files are .parquet format (a compressed format which can be imported as dataframes using R's arrow package `arrow::read_parquet` or using the free desktop application [Tad ](https://www.tadviewer.com/)for quick viewing). However, you can add the option `--write-csv` to the end of the commands to run the code (e.g., `> gempyor-simulate -c config.yml --write-csv)` to have everything saved as .csv files instead ;
+If your run is successful, you should see your output files in the model\_output folder. The structure of the files in this folder is described in the [Model Output](../gempyor/output-files.md) section. By default, all the output files are .parquet format (a compressed format which can be imported as dataframes using R's arrow package `arrow::read_parquet` or using the free desktop application [Tad ](https://www.tadviewer.com/) for quick viewing). However, you can add the option `--write-csv` to the end of the commands to run the code (e.g.,  `flepimop simulate --write-csv config.yml`) to have everything saved as .csv files instead ;
 
 ## 🪜 Next steps
 

examples/test_cli.py

@@ -1,6 +1,9 @@
-from click.testing import CliRunner
-from gempyor.simulate import simulate
 import os
+import subprocess
+
+from click.testing import CliRunner
+
+from gempyor.simulate import _click_simulate
 
 # See here to test click application https://click.palletsprojects.com/en/8.1.x/testing/
 # would be useful to also call the command directly
@@ -9,7 +12,18 @@
 def test_config_sample_2pop():
     os.chdir(os.path.dirname(__file__) + "/tutorials")
     runner = CliRunner()
-    result = runner.invoke(simulate, ["-c", "config_sample_2pop.yml"])
+    result = runner.invoke(_click_simulate, ["config_sample_2pop.yml"])
+    print(result.output)  # useful for debug
+    print(result.exit_code)  # useful for debug
+    print(result.exception)  # useful for debug
+    assert result.exit_code == 0
+    assert "completed in" in result.output
+
+
+def test_config_sample_2pop_deprecated():
+    os.chdir(os.path.dirname(__file__) + "/tutorials")
+    runner = CliRunner()
+    result = runner.invoke(_click_simulate, ["-c", "config_sample_2pop.yml"])
     print(result.output)  # useful for debug
     print(result.exit_code)  # useful for debug
     print(result.exception)  # useful for debug
@@ -20,20 +34,62 @@ def test_config_sample_2pop():
 def test_sample_2pop_modifiers():
     os.chdir(os.path.dirname(__file__) + "/tutorials")
     runner = CliRunner()
-    result = runner.invoke(simulate, ["-c", "config_sample_2pop_modifiers.yml"])
+    result = runner.invoke(
+        _click_simulate,
+        [
+            "config_sample_2pop.yml",
+            "config_sample_2pop_outcomes_part.yml",
+            "config_sample_2pop_modifiers_part.yml",
+        ],
+    )
+    print(result.output)  # useful for debug
+    print(result.exit_code)  # useful for debug
+    print(result.exception)  # useful for debug
+    assert result.exit_code == 0
+    assert "completed in" in result.output
+
+
+def test_sample_2pop_modifiers_combined():
+    os.chdir(os.path.dirname(__file__) + "/tutorials")
+    runner = CliRunner()
+    result = runner.invoke(_click_simulate, ["config_sample_2pop_modifiers.yml"])
     print(result.output)  # useful for debug
     print(result.exit_code)  # useful for debug
     print(result.exception)  # useful for debug
     assert result.exit_code == 0
     assert "completed in" in result.output
 
 
-def test_simple_usa_statelevel():
+def test_sample_2pop_modifiers_combined_deprecated():
+    os.chdir(os.path.dirname(__file__) + "/tutorials")
+    runner = CliRunner()
+    result = runner.invoke(_click_simulate, ["-c", "config_sample_2pop_modifiers.yml"])
+    print(result.output)  # useful for debug
+    print(result.exit_code)  # useful for debug
+    print(result.exception)  # useful for debug
+    assert result.exit_code == 0
+    assert "completed in" in result.output
+
+
+def test_simple_usa_statelevel_deprecated():
     os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel")
     runner = CliRunner()
-    result = runner.invoke(simulate, ["-c", "simple_usa_statelevel.yml", "-n", "1"])
+    result = runner.invoke(_click_simulate, ["-n", "1", "-c", "simple_usa_statelevel.yml"])
     print(result.output)  # useful for debug
     print(result.exit_code)  # useful for debug
     print(result.exception)  # useful for debug
     assert result.exit_code == 0
     assert "completed in" in result.output
+
+
+def test_simple_usa_statelevel_more_deprecated():
+    os.chdir(os.path.dirname(__file__) + "/simple_usa_statelevel")
+    result = subprocess.run(
+        ["gempyor-simulate", "-n", "1", "-c", "simple_usa_statelevel.yml"],
+        capture_output=True,
+        text=True,
+    )
+    print(result.stdout)  # useful for debug
+    print(result.stderr)  # useful for debug
+    print(result.returncode)  # useful for debug
+    assert result.returncode == 0

examples/tutorials/config_sample_2pop.yml

@@ -1,7 +1,7 @@
 name: sample_2pop
 setup_name: minimal
 start_date: 2020-02-01
-end_date: 2020-05-31
+end_date: 2020-08-31
 nslots: 1
 
 subpop_setup:
@@ -11,6 +11,7 @@ subpop_setup:
 initial_conditions:
   method: SetInitialConditions
   initial_conditions_file: model_input/ic_2pop.csv
+  allow_missing_subpops: TRUE
   allow_missing_compartments: TRUE
 
 compartments:

examples/tutorials/config_sample_2pop_interventions_test_part.yml

@@ -0,0 +1,18 @@
+
+seeding:
+  method: FromFile
+  seeding_file: model_input/seeding_2pop.csv
+
+modifiers:
+  scenarios:
+    - None
+  settings:
+    None:
+      template: Reduce
+      parameter: r0
+      period_start_date: 2020-04-01
+      period_end_date: 2020-05-15
+      value:
+        distribution: fixed
+        value: 0
+