From 5e1f9f84743bf4cefbe552dfea22f16d5a58bf07 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 17 May 2024 08:32:50 +0200 Subject: [PATCH 1/3] add preference for default compiler --- docs/recipes.md | 29 +++++++++++++++---- stackinator/recipe.py | 14 +++++++++ stackinator/templates/environments.spack.yaml | 3 ++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/recipes.md b/docs/recipes.md index b6337edf..04fc2673 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -101,7 +101,7 @@ The `compiler` field describes a list compilers to use to build the software sta Each compiler toolchain is specified using toolchain and spec ```yaml title="compile all packages with gcc@11.3" - compiler + compiler: - toolchain: gcc spec: gcc@11.3 ``` @@ -110,20 +110,37 @@ Sometimes two compiler toolchains are required, for example when using the `nvhp The example below uses the `nvhpc` compilers with gcc@11.3. ```yaml title="compile all packages with gcc@11.3" - compiler - - toolchain: llvm - spec: nvhpc@22.7 + compiler: - toolchain: gcc spec: gcc@11.3 + - toolchain: llvm + spec: nvhpc@22.7 ``` !!! note If more than one version of gcc has been installed, use the same version that was used to install `nvhpc`. !!! warning - As a rule, use a single compiler wherever possible - keep it simple! + Stackinator does not test or support using two versions of gcc in the same toolchain. - We don't test or support using two versions of gcc in the same toolchain. +The order of the compilers is significant. The first compiler is the default, and the other compilers will only be used to build packages when explicitly added to a spec. +For example, in the recipe below, only `netcdf-fortran` will be built with the `nvhpc` toolchain, while the root specs `cmake` and `netcdf-c` and all dependencies will be built using the `gcc` toolchain. + + +```yaml title="compile all packages with gcc@11.3" + compiler: + - toolchain: gcc + spec: gcc + - toolchain: llvm + spec: nvhpc + specs + - cmake + - netcdf-c + - netcdf-fortran%nvhpc +``` + +!!! note + This approach is typically used to build Fortran applications and packages with one toolchain (e.g. `nvhpc`), and all of the C/C++ dependencies with a different toolchain (e.g. `gcc`). ### MPI diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4031e7a0..f8afcb5a 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -293,6 +293,20 @@ def generate_environment_specs(self, raw): # TODO: Create a custom exception type raise Exception(f"Unsupported mpi: {mpi_impl}") + # set constraints that ensure the the main compiler is always used to build packages + # that do not explicitly request a compiler. + for name, config in environments.items(): + compilers = config["compiler"] + if len(compilers)==1: + config["toolchain_constraints"] = [] + continue + requires = [f"%{compilers[0]['spec']}"] + for spec in config["specs"]: + if '%' not in spec: + requires.append(spec) + + config["toolchain_constraints"] = requires + # An awkward hack to work around spack not supporting creating activation # scripts for each file system view in an environment: it only generates them # for the "default" view. diff --git a/stackinator/templates/environments.spack.yaml b/stackinator/templates/environments.spack.yaml index 0f2a3252..55f92f55 100644 --- a/stackinator/templates/environments.spack.yaml +++ b/stackinator/templates/environments.spack.yaml @@ -23,6 +23,9 @@ spack: {% for variant in config.variants %} - {{ variant }} {% endfor %} + {% for constraint in config.toolchain_constraints %} + - {{ constraint }} + {% endfor %} {% endif %} {% if config.mpi.spec %} mpi: From 6de518d113951f49d60462087a41e0093e78a7ce Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Fri, 17 May 2024 16:29:06 +0200 Subject: [PATCH 2/3] fix yaml generation --- stackinator/recipe.py | 2 +- stackinator/templates/environments.spack.yaml | 20 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index f8afcb5a..af1eb561 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -302,7 +302,7 @@ def generate_environment_specs(self, raw): continue requires = [f"%{compilers[0]['spec']}"] for spec in config["specs"]: - if '%' not in spec: + if '%' in spec: requires.append(spec) config["toolchain_constraints"] = requires diff --git a/stackinator/templates/environments.spack.yaml b/stackinator/templates/environments.spack.yaml index 55f92f55..31220d7c 100644 --- a/stackinator/templates/environments.spack.yaml +++ b/stackinator/templates/environments.spack.yaml @@ -1,4 +1,3 @@ -{% set separator = joiner(', ') %} spack: include: {% if config.packages %} @@ -13,23 +12,22 @@ spack: reuse: false specs: {% for spec in config.specs %} - - {{ spec }} + - '{{ spec }}' {% endfor %} packages: all: - compiler: [{% for c in config.compiler %}{{ separator() }}{{ c.spec }}{% endfor %}] +{% set separator = joiner(', ') %} + compiler: [{% for c in config.compiler %}{{ separator() }}'{{ c.spec }}'{% endfor %}] + require: +{% set separator = joiner(', ') %} + - one_of: [{% for c in config.toolchain_constraints %}{{ separator() }}'{{ c }}'{% endfor %}] {% if config.variants %} - variants: - {% for variant in config.variants %} - - {{ variant }} - {% endfor %} - {% for constraint in config.toolchain_constraints %} - - {{ constraint }} - {% endfor %} +{% set separator = joiner(', ') %} + variants: [{% for v in config.variants %}{{ separator() }}'{{ v }}'{% endfor %}] {% endif %} {% if config.mpi.spec %} mpi: - require: {{ config.mpi.spec }} + require: '{{ config.mpi.spec }}' {% endif %} {% if config.view %} view: From fa4212fa3538a1a2567e035c487e958cc7880e17 Mon Sep 17 00:00:00 2001 From: bcumming Date: Wed, 29 May 2024 07:48:52 +0200 Subject: [PATCH 3/3] fix black formatting, remove isort --- .github/workflows/lint.yaml | 5 +---- stackinator/recipe.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 7b7975bf..d33b4dca 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -16,13 +16,10 @@ jobs: - name: Install Tools run: | python -m pip install --upgrade pip - python -m pip install black flake8 isort mypy + python -m pip install black flake8 mypy - name: Black run: | black --check --verbose stackinator unittests - - name: isort - run: | - isort --check --skip stackinator/repo --diff . - name: flake8 run: | flake8 --count --show-source --statistics . diff --git a/stackinator/recipe.py b/stackinator/recipe.py index af1eb561..31330cca 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -297,12 +297,12 @@ def generate_environment_specs(self, raw): # that do not explicitly request a compiler. for name, config in environments.items(): compilers = config["compiler"] - if len(compilers)==1: + if len(compilers) == 1: config["toolchain_constraints"] = [] continue requires = [f"%{compilers[0]['spec']}"] for spec in config["specs"]: - if '%' in spec: + if "%" in spec: requires.append(spec) config["toolchain_constraints"] = requires