ADD: use case tests

.github/workflows/workflows-dt.yml

@@ -27,5 +27,5 @@ jobs:
 
       - name: Run pytest for workflows
         shell: bash -l {0}
-        run: micromamba run -p ./.venv-pytorch pytest -v ./tests/ -m "not distributed"
+        run: micromamba run -p ./.venv-pytorch pytest -v ./tests/ -m "not hpc"
 

pyproject.toml

@@ -70,5 +70,6 @@ itwinai = "itwinai.cli:app"
 [tool.pytest.ini_options]
 markers = [
     "integration: integration tests (deselect with '-m \"not integration\"')",
-    "distributed: test distributed ML on multiple GPUs/nodes (deselect with '-m \"not distributed\"')",
+    "hpc: require HPC resources (multiple GPUs/nodes). (deselect with '-m \"not hpc\"')",
+    "functional: functional tests. (deselect with '-m \"not functional\"')",
 ]

tests/backend/torch/test_distribtued_training.py

@@ -4,7 +4,7 @@
 import pytest
 
 
-@pytest.mark.distributed
+@pytest.mark.hpc
 def test_distributed_decorator():
     """Test function decorator. Needs torchrun cmd."""
     cmd = ("micromamba run -p ./ai/.venv-pytorch "
@@ -14,7 +14,7 @@ def test_distributed_decorator():
     subprocess.run(cmd.split(), check=True)
 
 
-@pytest.mark.distributed
+@pytest.mark.hpc
 def test_distributed_trainer():
     """Test vanilla torch distributed trainer. Needs torchrun cmd."""
     cmd = ("micromamba run -p ./ai/.venv-pytorch "

tests/use-cases/conftest.py

@@ -0,0 +1,37 @@
+import os
+import pytest
+import subprocess
+
+pytest.TORCH_PREFIX = './.venv-pytorch'
+pytest.TF_PREFIX = './.venv-tf'
+
+FNAMES = [
+    'pipeline.yaml',
+    'startscript',
+    'train.py',
+]
+
+
+@pytest.fixture
+def check_folder_structure():
+    """
+    Verify that the use case folder complies with some predefined
+    structure.
+    """
+    def _check_structure(root: str):
+        for fname in FNAMES:
+            fpath = os.path.join(root, fname)
+            assert os.path.isfile(fpath), f"'{fname}' is missing in '{fpath}'"
+    return _check_structure
+
+
+@pytest.fixture
+def install_requirements():
+    """Install requirements.txt, if present in root folder."""
+    def _install_reqs(root: str, env_prefix: str):
+        req_path = os.path.join(root, 'requirements.txt')
+        if os.path.isfile(req_path):
+            cmd = (f"micromamba run -p {env_prefix} "
+                   f"pip install -r {req_path}")
+            subprocess.run(cmd.split(), check=True)
+    return _install_reqs

tests/use-cases/test_cyclones.py

@@ -8,10 +8,23 @@
 import pytest
 import subprocess
 
-# TODO: add tests for use case folder format:
-#   - structure
-#   - naming convention
-#   - file exist
+CYCLONES_PATH = "use-cases/cyclones"
+
+
+def test_structure_cyclones(check_folder_structure):
+    """Test cyclones folder structure."""
+    check_folder_structure(CYCLONES_PATH)
+
+
+@pytest.mark.functional
+def test_mnist_train_torch(install_requirements):
+    """
+    Test MNIST torch native trainer by running it end-to-end.
+    """
+    install_requirements(CYCLONES_PATH, pytest.TF_PREFIX)
+    cmd = (f"micromamba run -p {pytest.TF_PREFIX} python "
+           f"{CYCLONES_PATH}/train.py -p {CYCLONES_PATH}/pipeline.yaml")
+    subprocess.run(cmd.split(), check=True)
 
 
 @pytest.mark.skip(reason="workflow changed")

tests/use-cases/test_mnist.py

@@ -8,15 +8,62 @@
 import pytest
 import subprocess
 
-# TODO: add tests for use case folder format:
-#   - structure
-#   - naming convention
-#   - file exist
+TORCH_PATH = "use-cases/mnist/torch"
+LIGHTNING_PATH = "use-cases/mnist/torch-lightning"
+TF_PATH = "use-cases/mnist/tensorflow"
 
 
-@pytest.mark.skip(reason="workflow changed")
+def test_structure_mnist_torch(check_folder_structure):
+    """Test MNIST folder structure for torch native trainer."""
+    check_folder_structure(TORCH_PATH)
+
+
+def test_structure_mnist_lightning(check_folder_structure):
+    """Test MNIST folder structure for torch lightning trainer."""
+    check_folder_structure(LIGHTNING_PATH)
+
+
+def test_structure_mnist_tf(check_folder_structure):
+    """Test MNIST folder structure for tensorflow trainer."""
+    check_folder_structure(TF_PATH)
+
+
+@pytest.mark.functional
+def test_mnist_train_torch(install_requirements):
+    """
+    Test MNIST torch native trainer by running it end-to-end.
+    """
+    install_requirements(TORCH_PATH, pytest.TORCH_PREFIX)
+    cmd = (f"micromamba run -p {pytest.TORCH_PREFIX} python "
+           f"{TORCH_PATH}/train.py -p {TORCH_PATH}/pipeline.yaml")
+    subprocess.run(cmd.split(), check=True)
+
+
+@pytest.mark.functional
+def test_mnist_train_lightning(install_requirements):
+    """
+    Test MNIST torch lightning trainer by running it end-to-end.
+    """
+    install_requirements(TORCH_PATH, pytest.TORCH_PREFIX)
+    cmd = (f"micromamba run -p {pytest.TORCH_PREFIX} python "
+           f"{LIGHTNING_PATH}/train.py -p {LIGHTNING_PATH}/pipeline.yaml")
+    subprocess.run(cmd.split(), check=True)
+
+
+@pytest.mark.functional
+def test_mnist_train_tf(install_requirements):
+    """
+    Test MNIST tensorflow trainer by running it end-to-end.
+    """
+    install_requirements(TF_PATH, pytest.TF_PREFIX)
+    cmd = (f"micromamba run -p {pytest.TF_PREFIX} python "
+           f"{TF_PATH}/train.py -p {TF_PATH}/pipeline.yaml")
+    subprocess.run(cmd.split(), check=True)
+
+
+@pytest.mark.skip(reason="workflow changed. Left as example")
 @pytest.mark.integration
-def test_mnist_train():
+def test_mnist_train_legacy():
     """
     Test MNIST training workflow(s) by running it end-to-end.
     """

use-cases/cyclones/lib/utils.py

@@ -4,7 +4,7 @@
 
 from .macros import Network
 from itwinai.models.tensorflow.cyclones_vgg import (
-    custom_VGG_V1, custom_VGG_V2, custom_VGG_V3, ModelV5
+    custom_VGG_V1, custom_VGG_V2, custom_VGG_V3  # , ModelV5
 )
 
 
@@ -61,12 +61,12 @@ def get_network_config(network, **kwargs):
             patch_size=kwargs['patch_size'], channels=kwargs['channels'],
             activation=kwargs['activation'], regularizer=kwargs['regularizer'])
 
-    elif network == Network.MODEL_V5.value:
-        print('Using Model V5')
-        model = ModelV5(
-            patch_size=kwargs['patch_size'], channels=kwargs['channels'],
-            last_activation=kwargs['activation'],
-            kernel_size=kwargs['kernel_size'])
+    # elif network == Network.MODEL_V5.value:
+    #     print('Using Model V5')
+    #     model = ModelV5(
+    #         patch_size=kwargs['patch_size'], channels=kwargs['channels'],
+    #         last_activation=kwargs['activation'],
+    #         kernel_size=kwargs['kernel_size'])
 
     return model