Improve CI (#123)

CI seems to have stopped failing for a while (see output of https://github.com/eth-cscs/stackinator/actions/runs/5486778221/jobs/9997318839?pr=122, for example). This PR improves how linting tools are run.

The whole code base is automatically re-formatted to make `black` and `isort` happy again.

Additionally, the `docs` and `publish` actions are only allowed to run from the main repo.

.github/workflows/docs.yaml

@@ -9,6 +9,7 @@ permissions:
   contents: write
 jobs:
   deploy:
+    if: github.repository == 'eth-cscs/stackinator'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/lint.yaml

@@ -19,10 +19,10 @@ jobs:
         python -m pip install black flake8 isort mypy
     - name: Black
       run: |
-        black .
+        black --check --verbose .
     - name: isort
       run: |
-        isort .
+        isort --check --diff .
     - name: flake8
       run: |
-        flake8
+        flake8 --count --show-source --statistics .

.github/workflows/publish.yaml

@@ -6,6 +6,7 @@ on:
 
 jobs:
   publish:
+    if: github.repository == 'eth-cscs/stackinator'
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3

stackinator/builder.py

@@ -10,7 +10,8 @@
 import jinja2
 import yaml
 
-from . import VERSION, root_logger, cache
+from . import VERSION, cache, root_logger
+
 
 class Builder:
     def __init__(self, args):
@@ -72,7 +73,7 @@ def environment_meta(self):
 
     @environment_meta.setter
     def environment_meta(self, recipe):
-        '''
+        """
         The output that we want to generate looks like the following,
         Which should correspond directly to the environment_view_meta provided
         by the recipe.
@@ -96,7 +97,7 @@ def environment_meta(self, recipe):
             }
           }
         }
-        '''
+        """
         conf = recipe.config
         meta = {}
         meta["name"] = conf["name"]
@@ -204,9 +205,12 @@ def generate(self, recipe):
             # print warning if mirrors.yaml is found
             if f_config.name in ["mirrors.yaml"]:
                 self._logger.error(
-                        "mirrors.yaml have been removed from cluster configurations,"
-                        " use the --cache option on stack-config instead.")
-                raise RuntimeError("Unsupported mirrors.yaml file in cluster configuration.")
+                    "mirrors.yaml have been removed from cluster configurations,"
+                    " use the --cache option on stack-config instead."
+                )
+                raise RuntimeError(
+                    "Unsupported mirrors.yaml file in cluster configuration."
+                )
 
             # construct full file path
             src = system_config_path / f_config.name
@@ -219,7 +223,7 @@ def generate(self, recipe):
         if recipe.mirror:
             dst = config_path / "mirrors.yaml"
             self._logger.debug(f"generate the build cache mirror: {dst}")
-            with dst.open('w') as fid:
+            with dst.open("w") as fid:
                 fid.write(cache.generate_mirrors_yaml(recipe.mirror))
 
         # append recipe packages to packages.yaml
@@ -366,8 +370,9 @@ def generate(self, recipe):
         with debug_script_path.open("w") as f:
             f.write(
                 debug_script_template.render(
-                    mount_path=recipe.config["store"], build_path=str(self.path), verbose=False
+                    mount_path=recipe.config["store"],
+                    build_path=str(self.path),
+                    verbose=False,
                 )
             )
             f.write("\n")
-

stackinator/cache.py

@@ -1,10 +1,11 @@
+import os
 import pathlib
 
-import os
 import yaml
 
 from . import schema
 
+
 def configuration_from_file(file, mount):
     with file.open() as fid:
         # load the raw yaml input
@@ -16,13 +17,9 @@ def configuration_from_file(file, mount):
         # verify that the root path exists
         path = pathlib.Path(os.path.expandvars(raw["root"]))
         if not path.is_absolute():
-            raise FileNotFoundError(
-                    f"The build cache path '{path}' is not absolute"
-            )
+            raise FileNotFoundError(f"The build cache path '{path}' is not absolute")
         if not path.is_dir():
-            raise FileNotFoundError(
-                    f"The build cache path '{path}' does not exist"
-            )
+            raise FileNotFoundError(f"The build cache path '{path}' does not exist")
 
         raw["root"] = path
 
@@ -35,36 +32,33 @@ def configuration_from_file(file, mount):
         if key is not None:
             key = pathlib.Path(os.path.expandvars(key))
             if not key.is_absolute():
-                raise FileNotFoundError(
-                        f"The build cache key '{key}' is not absolute"
-                )
+                raise FileNotFoundError(f"The build cache key '{key}' is not absolute")
             if not key.is_file():
-                raise FileNotFoundError(
-                        f"The build cache key '{key}' does not exist"
-                )
+                raise FileNotFoundError(f"The build cache key '{key}' does not exist")
             raw["key"] = key
 
         return raw
 
+
 def generate_mirrors_yaml(config):
-    path = config['path'].as_posix()
+    path = config["path"].as_posix()
     mirrors = {
-        'mirrors': {
-            'alpscache': {
-                'fetch': {
-                    'url': f"file://{path}",
-                    'access_pair': [None, None],
-                    'access_token': None,
-                    'profile': None,
-                    'endpoint_url': None,
+        "mirrors": {
+            "alpscache": {
+                "fetch": {
+                    "url": f"file://{path}",
+                    "access_pair": [None, None],
+                    "access_token": None,
+                    "profile": None,
+                    "endpoint_url": None,
+                },
+                "push": {
+                    "url": f"file://{path}",
+                    "access_pair": [None, None],
+                    "access_token": None,
+                    "profile": None,
+                    "endpoint_url": None,
                 },
-                'push': {
-                    'url': f"file://{path}",
-                    'access_pair': [None, None],
-                    'access_token': None,
-                    'profile': None,
-                    'endpoint_url': None,
-                }
             }
         }
     }

stackinator/etc/add-compiler-links.py

@@ -4,39 +4,47 @@
 
 import argparse
 import os
+
 import yaml
 
+
 # parse compilers.yaml file.
 # return a list with the compiler descriptions from the yaml file.
 def load_compilers_yaml(path):
-    with open(path, 'r') as file:
+    with open(path, "r") as file:
         data = yaml.safe_load(file)
     compilers = [c["compiler"] for c in data["compilers"]]
     return compilers
 
+
 def parse_export(line):
-    s = line.replace('=', ' ').split()
+    s = line.replace("=", " ").split()
     var = s[1]
     paths = None
-    if len(s)>2:
-        paths = s[2].rstrip(';').split(':')
+    if len(s) > 2:
+        paths = s[2].rstrip(";").split(":")
     return {"variable": var, "paths": paths}
 
+
 def split_line(line):
-    return line.strip().rstrip(';').replace('=', ' ').split()
+    return line.strip().rstrip(";").replace("=", " ").split()
+
 
 def is_export(parts):
-    return len(parts)>1 and parts[0]=="export"
+    return len(parts) > 1 and parts[0] == "export"
+
 
 def is_alias(parts):
-    return len(parts)>0 and parts[0]=="alias"
+    return len(parts) > 0 and parts[0] == "alias"
+
 
 # Returns True if the given path is a descendant of prefix, False otherwise.
 def has_prefix(path, prefix):
     prefix = os.path.realpath(prefix)
     path = os.path.realpath(path)
     return os.path.commonprefix([path, prefix]) == prefix
 
+
 parser = argparse.ArgumentParser()
 parser.add_argument("compiler_path", help="Path to the compilers.yaml file")
 parser.add_argument("activate_path", help="Path to the activate script to configure")
@@ -59,12 +67,12 @@ def has_prefix(path, prefix):
 
 paths = []
 for c in compilers:
-    local_paths = set([os.path.dirname(v) for k,v in c["paths"].items()])
+    local_paths = set([os.path.dirname(v) for k, v in c["paths"].items()])
     paths += local_paths
     print(f'adding compiler {c["spec"]} -> {[p for p in local_paths]}')
 
 # find unique paths and concatenate them
-pathstring = ':'.join(set(paths))
+pathstring = ":".join(set(paths))
 
 # Parse the spack env activation script line by line.
 # Remove spack-specific environment variables and references the build path.
@@ -73,7 +81,7 @@ def has_prefix(path, prefix):
 # etc. This may or may not be surprising for users, and we may have to append
 # :$PATH, :$CPATH, etc.
 
-lines=[]
+lines = []
 with open(args.activate_path) as fid:
     for line in fid:
         parts = split_line(line)
@@ -86,22 +94,24 @@ def has_prefix(path, prefix):
 
             # parse PATH to remove references to the build directory
             if export["variable"] == "PATH":
-                paths=[p for p in export["paths"] if not has_prefix(p, args.build_path)]
+                paths = [
+                    p for p in export["paths"] if not has_prefix(p, args.build_path)
+                ]
                 lines.append(f"export PATH={':'.join(paths)};\n")
 
             # drop the SPACK_ENV variable
             elif export["variable"] == "SPACK_ENV":
                 pass
 
             else:
-                lines.append(line.strip()+"\n")
+                lines.append(line.strip() + "\n")
         else:
-            lines.append(line.strip()+"\n")
+            lines.append(line.strip() + "\n")
 
 # Prepend the compiler paths to PATH
 lines.append("# compiler paths added by stackinator\n")
 lines.append(f"export PATH={pathstring}:$PATH;\n")
 
 # Write a modified version of the activation script.
-with open(args.activate_path, 'w') as fid:
+with open(args.activate_path, "w") as fid:
     fid.writelines(lines)