Update dev

.dockerignore

@@ -0,0 +1,135 @@
+**/TODO
+**/mamba*
+pl-training.yml
+.vscode
+
+# Project folders/files
+# use-cases
+workflows
+tests
+CHANGELOG
+
+# Docs
+docs
+
+# interLink pods
+**/interLink
+**/interlink
+
+# Data
+**/MNIST
+**/*-predictions/
+**/*-data/
+**/*.tar.gz
+**/exp_data
+
+# Logs
+**/logs
+**/lightning_logs
+**/mlruns
+**/.logs
+**/mllogs
+**/nohup*
+**/*.out
+**/*.err
+**/checkpoints/
+**/*_logs
+**/tmp*
+**/.tmp*
+
+# Markdown
+**/*.md
+
+# Custom envs
+**/.venv*
+
+# Git
+.git
+.gitignore
+.github
+
+# CI
+.codeclimate.yml
+.travis.yml
+.taskcluster.yml
+
+# Docker
+docker-compose.yml
+.docker
+.dockerignore
+Dockerfile
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+**/*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+**/eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+**/*.egg-info/
+**/.installed.cfg
+**/*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Virtual environment
+.env/
+.venv/
+venv/
+
+# PyCharm
+.idea
+
+# Python mode for VIM
+.ropeproject
+*/.ropeproject
+*/*/.ropeproject
+*/*/*/.ropeproject
+
+# Vim swap files
+*.swp
+*/*.swp
+*/*/*.swp
+*/*/*/*.swp

.github/linters/.jscpd.json

@@ -0,0 +1,6 @@
+{
+    "threshold": 2.0,
+    "ignore": [
+        "**/itwinai/loggers.py"
+    ]
+}

.github/workflows/lint.yml

@@ -36,16 +36,22 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           MARKDOWN_CONFIG_FILE: .markdownlint.json
           PYTHON_FLAKE8_CONFIG_FILE: .flake8
+          JSCPD_CONFIG_FILE: .jscpd.json
           VALIDATE_PYTHON_BLACK: false
           VALIDATE_PYTHON_ISORT: false
           VALIDATE_PYTHON_MYPY: false
           VALIDATE_PYTHON_PYLINT: false
           VALIDATE_HTML: false
           VALIDATE_GITLEAKS: false
+          VALIDATE_BASH_EXEC: false
+          VALIDATE_CHECKOV: false # activate to lint k8s pods
+          VALIDATE_SHELL_SHFMT: false
+          VALIDATE_JSCPD: false
 
           # Only check new or edited files
           VALIDATE_ALL_CODEBASE: false
           # Fail on errors
           DISABLE_ERRORS: false
           # Skip linting of docs
-          FILTER_REGEX_EXCLUDE: .*docs/index.md|.*docs/docs/.*|.*ISSUE_TEMPLATE/.*
+          FILTER_REGEX_EXCLUDE: .*docs/index.md|.*docs/docs/.*|.*ISSUE_TEMPLATE/.*|use-cases/.*|experimental/.*
+          BASH_SEVERITY: error

.github/workflows/pypi.yml

@@ -0,0 +1,31 @@
+name: Upload Python Package to PyPI when a Release is Created
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/itwinai
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -q build
+        #   pip install setuptools wheel
+      - name: Build package
+        run:  python -m build
+        #   python setup.py sdist bdist_wheel  # Could also be python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/pytest.yml

@@ -0,0 +1,38 @@
+---
+name: Unit and integration tests
+
+on:
+  pull_request:
+    branches: [main, dev]
+
+jobs:
+  test-itwinai:
+    name: Test itwinai with pytest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install compilers for Horovod
+        run: |
+          sudo apt-get update &&
+          sudo apt-get install -y gcc-11 g++-11 &&
+          sudo apt-get install -y cmake &&
+          sudo apt-get install openmpi-bin  openmpi-common openmpi-doc libopenmpi-dev &&
+          gcc --version &&
+          cmake --version &&
+          mpirun --version
+
+      - name: Make PyTorch virtualenv
+        shell: bash -l {0}
+        run: make torch-env-cpu
+
+      - name: Make Tensorflow virtualenv
+        shell: bash -l {0}
+        run: make tensorflow-env-cpu
+
+      # NOTE, to change the name in which tests are run, set custom TORCH_ENV and TF_ENV env variables.
+      # Default environment names are ".venv-pytorch" and ".venv-tf"
+      - name: Run pytest for workflows
+        shell: bash -l {0}
+        run: .venv-pytorch/bin/pytest -v ./tests/ -m "not slurm"
+

.github/workflows/sqaaas.yml

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright contributors to the Software Quality Assurance as a Service (SQAaaS) project.
+#
+# SPDX-License-Identifier: GPL-3.0-only
+---
+name: SQAaaS
+
+on:
+  push:
+    branches: [main, dev]
+  # pull_request:
+  #   branches: [main, dev]
+
+jobs:
+  sqaaas_job:
+    runs-on: ubuntu-latest
+    name: Job that triggers SQAaaS platform
+    steps:
+      - name: Step definition for validating the workflow
+        uses: eosc-synergy/sqaaas-step-action@v1
+        with:
+          name: workflow_validation_step
+          tool: commands
+          commands: |
+            make torch-env-cpu
+            make tensorflow-env-cpu
+            .venv-pytorch/bin/pytest -v ./tests/ -m "not slurm and not memory_heavy"
+          container: eoscsynergy/sqaaas-micromamba:1.5.3-1
+      - name: Print out payload
+        run: cat workflow_validation_step.json
+      - name: SQAaaS assessment with unit testing (QC.Uni) step
+        uses: eosc-synergy/sqaaas-assessment-action@v2
+        with:
+          qc_uni_steps: workflow_validation_step