.gitlab-ci.yml

stages:
  - pretest
  - test
  - nightly
  - docs
  - deploy


# --------------------------  Templates ------------------------------------------------------------------------------------

# Base configuration for jobs meant to run at every commit
.every-commit:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"

# Configuration for jobs meant to run on each commit to pycodegen/pystencils/master
.every-commit-master:
  rules:
    - if: '$CI_PIPELINE_SOURCE != "schedule" && $CI_PROJECT_PATH == "pycodegen/pystencils" && $CI_COMMIT_BRANCH == "master"'

# Base configuration for jobs meant to run at a schedule
.scheduled:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"

# --------------------------  Tests ------------------------------------------------------------------------------------

# Normal test - runs on every commit all but "long run" tests
tests-and-coverage:
  stage: pretest
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  before_script:
    - pip install -e .
  script:
    - env
    - pip list
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
    - pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
    - python -m coverage xml
  tags:
    - docker
    - cuda11
    - AVX
  coverage: /Total coverage:\s\d+.\d+\%/
  artifacts:
    when: always
    paths:
      - coverage_report
      - test-report
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
      junit: report.xml

# Normal test with longruns
tests-and-coverage-with-longrun:
  stage: test
  when: manual
  allow_failure: true
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  before_script:
    - pip install sympy --upgrade
    - pip install -e .
  script:
    - env
    - pip list
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
    - py.test -v -n $NUM_CORES
  tags:
    - docker
    - cuda11
    - AVX

# pipeline with latest python version
latest-python:
  stage: test
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python
  before_script:
    - pip install -e .
  script:
    - env
    - pip list
    - pip install -e .
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
    - py.test -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
  tags:
    - docker
    - AVX
  artifacts:
    when: always
    reports:
      junit: report.xml


# Minimal tests in windows environment
#minimal-windows:
#  stage: test
#  tags:
#    - win
#  script:
#    - export NUM_CORES=$(nproc --all)
#    - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
#    - source activate pystencils
#    - pip install joblib
#    - pip list
#    - python -c "import numpy"
#    - py.test -v -m "not (notebook or longrun)"

ubuntu:
  stage: test
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
  before_script:
    - ln -s /usr/include/locale.h /usr/include/xlocale.h
    - pip3 install -e .
  script:
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - sed -i 's/--doctest-modules //g' pytest.ini
    - env
    - pip list
    - pytest -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
  tags:
    - docker
    - cuda11
    - AVX
  artifacts:
    when: always
    reports:
      junit: report.xml

.multiarch_template:
  stage: test
  extends: .every-commit
  before_script: &multiarch_before_script
    # - pip3 install -v .
    - export PYTHONPATH=src
    - python3 -c "import pystencils as ps; ps.cpu.cpujit.read_config()"
    - sed -i '/^fail_under.*/d' pytest.ini
  script:
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - sed -i 's/--doctest-modules //g' pytest.ini
    - env
    - pip3 list
    - python3 -m pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov=. --junitxml=report.xml tests/test_*vec*.py tests/test_random.py tests/test_half_precision.py
    - python3 -m coverage xml
  tags:
    - docker
    - AVX
  artifacts:
    when: always
    paths:
      - coverage_report
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
      junit: report.xml

arm64v8:
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64
  variables:
    QEMU_CPU: "cortex-a76"
  before_script:
    - *multiarch_before_script
    - sed -i s/march=native/march=armv8-a/g ~/.config/pystencils/config.json

ppc64le:
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ppc64le
  before_script:
    - *multiarch_before_script
    - sed -i s/mcpu=native/mcpu=power8/g ~/.config/pystencils/config.json

arm64v9:
  # SVE support is still unreliable in GCC 11 (incorrect code for fixed-width vectors, internal compiler errors).
  # For half precision Clang is necessary
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64
  before_script:
    - *multiarch_before_script
    - sed -i s/march=native/march=armv9-a+sve2+sme/g ~/.config/pystencils/config.json
    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json

riscv64:
  # RISC-V vector extension are currently not supported by GCC.
  extends: .multiarch_template
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/riscv64
  variables:
    # explicitly set SIMD as detection requires QEMU >= 8.1
    PYSTENCILS_SIMD: "rvv"
    QEMU_CPU: "rv64,v=true,zicboz=true"
  before_script:
    - *multiarch_before_script
    - sed -i 's/march=native/march=rv64imfdvzicboz/g' ~/.config/pystencils/config.json
    - sed -i s/g\+\+/clang++-15/g ~/.config/pystencils/config.json

minimal-conda:
  stage: pretest
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
  before_script:
    - pip install -e .
  script:
    - python quicktest.py
  tags:
    - docker
    - cuda


minimal-sympy-master:
  stage: test
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
  before_script:
    - pip install -e .
  script:
    - python -m pip install --upgrade git+https://github.com/sympy/sympy.git
    - python quicktest.py
  allow_failure: true
  tags:
    - docker
    - cuda


pycodegen-integration:
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  stage: test
  when: manual
  allow_failure: true
  script:
    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@i10git.cs.fau.de/pycodegen/pycodegen.git
    - cd pycodegen
    - git submodule sync --recursive
    - git submodule update --init --recursive
    - git submodule foreach git fetch origin   # compare the latest master version!
    - git submodule foreach git reset --hard origin/master
    - cd pystencils
    - git remote add test $CI_REPOSITORY_URL
    - git fetch test
    - git reset --hard $CI_COMMIT_SHA
    - cd ..
    - pip install -e pystencils/
    - pip install -e lbmpy/
    - cmake --version
    - ./install_walberla.sh
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - cd pystencils
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
    - cd ../lbmpy
    - py.test -v -n $NUM_CORES --junitxml=report.xml .
    - cd ../walberla/build/
    - make -j $NUM_CORES CodegenJacobiCPU CodegenJacobiGPU CodegenPoissonCPU CodegenPoissonGPU MicroBenchmarkGpuLbm LbCodeGenerationExample
    - make -j $NUM_CORES multiphaseCPU multiphaseGPU FluctuatingMRT FlowAroundSphereCodeGen FieldLayoutAndVectorizationTest GeneratedOutflowBC
    - cd apps/benchmarks/UniformGridGPU
    - make -j $NUM_CORES
    - cd ../UniformGridCPU
    - make -j $NUM_CORES
  tags:
    - docker
    - cuda11
    - AVX
  artifacts:
    when: always
    reports:
      junit: pycodegen/*/report.xml


# -------------------- Scheduled Tasks --------------------------------------------------------------------------


# Nightly test against the latest (pre-release) version of SymPy published on PyPI
nightly-sympy:
  stage: nightly
  needs: []
  extends: .scheduled
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python
  before_script:
    - pip install -e .
    - pip install --upgrade --pre sympy
  script:
    - env
    - pip list
    - export NUM_CORES=$(nproc --all)
    - mkdir -p ~/.config/matplotlib
    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
    - mkdir public
    - pytest -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
  tags:
    - docker
    - AVX
    - cuda
  artifacts:
    when: always
    reports:
      junit: report.xml

# -------------------- Linter & Documentation --------------------------------------------------------------------------


flake8-lint:
  stage: pretest
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  script:
    - flake8 src/pystencils
  tags:
    - docker


build-documentation:
  stage: docs
  extends: .every-commit
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation
  needs: []
  before_script:
    - pip install -e .
  script:
    - mkdir html_doc
    - sphinx-build -b html doc  html_doc
    - sphinx-build -W -b html doc  html_doc
  tags:
    - docker
  artifacts:
    paths:
      - html_doc


pages:
  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
  extends: .every-commit-master
  stage: deploy
  needs: ["tests-and-coverage", "build-documentation"]
  script:
    - ls -l
    - mv coverage_report html_doc
    - mv html_doc public  # folder has to be named "public" for gitlab to publish it
  artifacts:
    paths:
      - public
  tags:
    - docker