Functional tests for MNIST (#87)

* ADD: use case tests

* FIX: move use case models out of itwinai

* FIX: rearrange modules

* ADD: ConsoleLogger and LoggersCollection

* FIX: loggers filter

* FIX: add TF env creation

* UPDATE: test flag

* ADD: early pytest on slurm

* FIX: duplicated code in TF Trainer

.github/linters/.jscpd.json

@@ -1,6 +1,6 @@
 {
     "threshold": 2.0,
     "ignore": [
-        "**/itwinai/backend/loggers.py"
+        "**/itwinai/loggers.py"
     ]
 }

.github/workflows/workflows-dt.yml

@@ -21,11 +21,15 @@ jobs:
         shell: bash -l {0}
         run: make torch-cpu
 
+      - name: Make tensorflow env
+        shell: bash -l {0}
+        run: make tf-2.13
+
       - name: Install dev version
         shell: bash -l {0}
         run: micromamba run -p ./.venv-pytorch pip install .[dev]
 
       - name: Run pytest for workflows
         shell: bash -l {0}
-        run: micromamba run -p ./.venv-pytorch pytest -v ./tests/ -m "not distributed"
+        run: micromamba run -p ./.venv-pytorch pytest -v ./tests/ -m "not slurm"
 

.gitignore

@@ -1,5 +1,5 @@
 TODO
-data
+/data
 nohup*
 lightning_logs
 mlruns

README.md

@@ -96,11 +96,25 @@ adding the `dev` extra:
 pip install -e .[dev]
 ```
 
-To **run tests** on itwinai package:
+#### Test with `pytest`
+
+To run tests on itwinai package:
 
 ```bash
 # Activate env
 micromamba activate ./.venv-pytorch # or ./.venv-tf
 
-pytest -v tests/
+pytest -v -m "not slurm" tests/ 
+```
+
+However, some tests are intended to be executed only on an HPC system,
+where SLURM is available. They are marked with "slurm" tag. To run also
+those tests, use the dedicated job script:
+
+```bash
+sbatch tests/slurm_tests_startscript
+
+# Upon completion, check the output:
+cat job.err
+cat job.out
 ```

pyproject.toml

@@ -70,5 +70,6 @@ itwinai = "itwinai.cli:app"
 [tool.pytest.ini_options]
 markers = [
     "integration: integration tests (deselect with '-m \"not integration\"')",
-    "distributed: test distributed ML on multiple GPUs/nodes (deselect with '-m \"not distributed\"')",
+    "slurm: needs SLURM and HPC resources (multiple GPUs/nodes). (deselect with '-m \"not slurm\"')",
+    "functional: functional tests. (deselect with '-m \"not functional\"')",
 ]