wip

stackinator/builder.py

@@ -253,7 +253,7 @@ def generate(self, recipe):
             f.write("\n")
 
         etc_path = self.root / "etc"
-        for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "add-compiler-links.py"]:
+        for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py"]:
             shutil.copy2(etc_path / f_etc, self.path / f_etc)
 
         # used to configure both pre and post install hooks, if they are provided.
@@ -490,7 +490,7 @@ def generate(self, recipe):
             f.write("\n")
 
         # write a json file with the environment view meta data
-        with (meta_path / "env.json").open("w") as f:
+        with (meta_path / "env.json.in").open("w") as f:
             # default serialisation is str to serialise the pathlib.PosixPath
             f.write(json.dumps(self.environment_meta, sort_keys=True, indent=2, default=str))
             f.write("\n")

unittests/envvars.py → stackinator/etc/envvars.py

@@ -4,6 +4,7 @@
 from enum import Enum
 import json
 import os
+import yaml
 from typing import Optional, List
 
 class EnvVarOp (Enum):
@@ -86,6 +87,13 @@ def updates(self):
     def concat(self, other: 'ListEnvVar'):
         self._updates += other.updates
 
+    @property
+    def paths(self):
+        paths = []
+        for u in self._updates:
+            paths += u.value
+        return paths
+
     # Given the current value, return the value that should be set
     # current is None implies that the variable is not set
     #
@@ -365,58 +373,59 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet]=None) -> EnvV
 
     return env
 
-def impl1():
-    # parse CLI arguments
-    parser = argparse.ArgumentParser()
-    parser.add_argument("script", help="Path to the activate script to parse",
-                        type=str)
-    parser.add_argument("json", help="Path to where the json file should be ouput",
-                        type=str)
-    parser.add_argument("meta", help="path to the meta",
-                        type=str)
-    parser.add_argument("view", help="the name of the view",
-                        type=str)
-    args = parser.parse_args()
+def spack_impl(args):
+    print(f"parsing activate script {args.activate} with compilers {args.compilers} and LD_LIBRARY_PATH {args.ld_library_path}")
 
-    # verify that the paths exist
-    if not os.path.exists(args.script):
-        print(f"error - activation script '{args.script}' does not exist.")
-        exit(2)
-    if not os.path.exists(args.meta):
-        print(f"error - uenv meta data file '{args.meta}' does not exist.")
-        exit(2)
+    if not os.path.isfile(args.activate):
+        print(f"error - activation script {args.activate} does not exist")
 
-    # parse the activation script for its environment variable changes
-    envvars = read_activation_script(args.script)
+    envvars = read_activation_script(args.activate)
 
-    # write the environment variable update to a json file
-    print(f"writing environment variable information to json: {args.json}")
-    with open(args.json, "w") as fid:
-        fid.write(envvars.as_json())
-        fid.write("\n")
+    if args.compilers is not None:
+        if not os.path.isfile(args.compilers):
+            print(f"error - compiler yaml file {args.compilers} does not exist")
 
-    # parse the uenv meta data from file
-    print(f"loading meta data to update: {args.meta}")
-    with open(args.meta) as fid:
-        meta = json.load(fid)
+        with open(args.compilers, "r") as file:
+            data = yaml.safe_load(file)
+        compilers = [c["compiler"] for c in data["compilers"]]
 
-    # TODO: handle the case where there is no matching view description
-    meta["views"][args.view]["json"] = { "version": 1, "values": envvars.as_dict() }
-    meta["views"][args.view]["type"] = "spack-view"
+        compiler_paths = []
+        for c in compilers:
+            local_paths = set([os.path.dirname(v) for _, v in c["paths"].items() if v is not None])
+            compiler_paths += local_paths
+            print(f'adding compiler {c["spec"]} -> {[p for p in local_paths]}')
 
-    # update the uenv meta data file with the new env. variable description
-    with open(args.meta, "w") as fid:
-        # write updated meta data
-        json.dump(meta, fid)
+        envvars.set_list("PATH", compiler_paths, EnvVarOp.PREPEND)
+
+    if args.set_ld_library_path:
+        # get the root path of the env
+        root_path = os.path.dirname(args.activate)
+        print(f"LD_LIBRARY_PATH: root path {root_path}")
+
+        # search for root/lib, root/lib64
+        paths = []
+        for p in ["lib", "lib64"]:
+            test_path = f"{root_path}/{p}"
+            if os.path.isdir(test_path):
+                paths.append(test_path)
+
+        print(f"LD_LIBRARY_PATH: found {paths}")
+
+        # TODO: only update 
+        if "LD_LIBRARY_PATH" in envvars.lists:
+            ld_paths = envvars.lists["LD_LIBRARY_PATH"].paths
+            final_paths = [p for p in paths if p not in ld_paths]
+            envvars.set_list("LD_LIBRARY_PATH", final_paths, EnvVarOp.PREPEND)
+        else:
+            envvars.set_list("LD_LIBRARY_PATH", paths, EnvVarOp.PREPEND)
 
-def impl2():
-    # parse CLI arguments
-    parser = argparse.ArgumentParser()
-    parser.add_argument("mount",    help="mount point of the image",type=str)
-    parser.add_argument("--modules", help="configure a module view", action="store_true")
-    parser.add_argument("--spack",  help="configure a spack view. Format is \"spack_url,git_commit\"",  type=str, default=None)
-    args = parser.parse_args()
 
+    #args.activate
+    #args.set_ld_library_path
+    #args.compilers
+
+
+def meta_impl(args):
     # verify that the paths exist
     if not os.path.exists(args.mount):
         print(f"error - uenv mount '{args.mount}' does not exist.")
@@ -452,7 +461,7 @@ def impl2():
     if args.modules:
         module_path = f"{args.mount}/modules"
         meta["views"]["modules"] = {
-            "activate": None,
+            "activate": "/dev/null",
             "description": "activate modules",
             "root": module_path,
             "json": {
@@ -474,15 +483,14 @@ def impl2():
         url, version = args.spack.split(',')
         spack_path = f"{args.mount}/config".replace("//", "/")
         meta["views"]["spack"] = {
-            "activate": None,
+            "activate": "/dev/null",
             "description": "configure spack upstream",
             "root": spack_path,
             "json": {
                 "version": 1,
                 "values": {
                     "list": {},
                     "scalar": {
-                        "SPACK_SYSTEM_CONFIG_PATH": spack_path,
                         "UENV_SPACK_CONFIG_PATH": spack_path,
                         "UENV_SPACK_COMMIT": version,
                         "UENV_SPACK_URL": url
@@ -498,5 +506,29 @@ def impl2():
         fid.write("\n")
 
 if __name__ == "__main__":
-    #impl1()
-    impl2()
+    # parse CLI arguments
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="command")
+    spack_parser = subparsers.add_parser("spack-env",
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            help="generate environment configuration for a specific spack env")
+    spack_parser.add_argument("activate", help="path of the activation script",type=str)
+    spack_parser.add_argument("--set_ld_library_path", help="force setting of LD_LIBRARY_PATH", action="store_true")
+    # only add compilers if this argument is passed
+    spack_parser.add_argument("--compilers",  help="path of the compilers.yaml file",  type=str, default=None)
+
+    uenv_parser = subparsers.add_parser("uenv",
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            help="generate final meta data for a whole uenv.")
+    uenv_parser.add_argument("mount",    help="mount point of the image",type=str)
+    uenv_parser.add_argument("--modules", help="configure a module view", action="store_true")
+    uenv_parser.add_argument("--spack",  help="configure a spack view. Format is \"spack_url,git_commit\"",  type=str, default=None)
+
+    args = parser.parse_args()
+
+    if args.command == "uenv":
+        print("!!! running meta")
+        meta_impl(args)
+    elif args.command == "spack":
+        print("!!! running spack")
+        spack_impl(args)

stackinator/recipe.py

@@ -310,16 +310,22 @@ def generate_environment_specs(self, raw):
             for view, vc in config["views"].items():
                 if view in env_names:
                     raise Exception(f"An environment view with the name '{view}' already exists.")
+                # set some default values:
+                # vc["link"] = "roots"
+                # vc["uenv"]["add_compilers"] = True
+                # vc["uenv"]["set_ld_library_path"] = True
+                if vc is None: vc = {}
+                vc.setdefault("link", "roots")
+                vc.setdefault("uenv", {})
+                vc["uenv"].setdefault("add_compilers", True)
+                vc["uenv"].setdefault("set_ld_library_path", True)
                 # save a copy of the view configuration
                 env_name_map[name].append((view, vc))
 
         # Iterate over each environment:
         # - creating copies of the env so that there is one copy per view.
         # - configure each view
         for name, views in env_name_map.items():
-            print(f"{name}:")
-            for vvv in views:
-                print(f" {vvv}")
             numviews = len(env_name_map[name])
 
             # The configuration of the environment without views
@@ -340,7 +346,11 @@ def generate_environment_specs(self, raw):
                     view_config = {"root": str(self.mount / "env" / view_name)}
                 else:
                     view_config["root"] = str(self.mount / "env" / view_name)
-                print(f"  = {view_name} {view_config}")
+
+                # The "uenv" field is not spack configuration, it is additional information
+                # used by stackinator additionally set compiler paths and LD_LIBRARY_PATH
+                # Remove it from the view_config that will be passed directly to spack, and pass
+                # it separately for configuring the envvars.py helper during the uenv build.
                 extra = view_config.pop("uenv")
 
                 environments[cname]["view"] = {"name": view_name, "config": view_config, "extra": extra}

stackinator/schema/environments.json

@@ -63,8 +63,8 @@
                     "type": "object",
                     "patternProperties": {
                         "\\w+": {
-                            "type": "object",
                             "additionalProperties": false,
+                            "comment": "we can't set default values in such a construction, so they are set inside stackinator",
                             "properties": {
                                 "link": {
                                     "type": "string",
@@ -94,14 +94,7 @@
                                 }
                             }
                         }
-                    },
-                            "default": {
-                                "link": "roots",
-                                "uenv": {
-                                    "add_compilers": true,
-                                    "set_ld_library_path": false
-                                }
-                            }
+                    }
                  }
              }
          }

stackinator/templates/Make.user

@@ -3,7 +3,7 @@
 # Copy this file to Make.user and set some variables.
 
 # This is the root of the software stack directory.
-SOFTWARE_STACK_PROJECT := {{ build_path }}
+BUILD_ROOT := {{ build_path }}
 
 # What Spack should we use?
 SPACK := spack
@@ -25,10 +25,10 @@ STORE := {{ store }}
 {% if no_bwrap %}
 SANDBOX :=
 {% else %}
-SANDBOX := $(SOFTWARE_STACK_PROJECT)/bwrap-mutable-root.sh $\
+SANDBOX := $(BUILD_ROOT)/bwrap-mutable-root.sh $\
 	--tmpfs ~ $\
-	--bind $(SOFTWARE_STACK_PROJECT)/tmp /tmp $\
-	--bind $(SOFTWARE_STACK_PROJECT)/store $(STORE)
+	--bind $(BUILD_ROOT)/tmp /tmp $\
+	--bind $(BUILD_ROOT)/store $(STORE)
 {% endif %}
 # Makes sure that make -Orecurse continues to print in color.
 export SPACK_COLOR := always
@@ -41,10 +41,10 @@ export SPACK_USER_CONFIG_PATH := /dev/null
 
 # Set up the system config scope that has the system packages we don't want
 # build, for example slurm, pmix, etc. Also should have the system compiler.
-export SPACK_SYSTEM_CONFIG_PATH := $(SOFTWARE_STACK_PROJECT)/config
+export SPACK_SYSTEM_CONFIG_PATH := $(BUILD_ROOT)/config
 
 # Put clingo and friends here...
-export SPACK_USER_CACHE_PATH := $(SOFTWARE_STACK_PROJECT)/cache
+export SPACK_USER_CACHE_PATH := $(BUILD_ROOT)/cache
 
 # Output the full build log to stdout.
 {% if verbose %}

stackinator/templates/Makefile

@@ -55,15 +55,18 @@ environments: compilers
 	$(SANDBOX) $(MAKE) -C $@
 
 modules: environments generate-config
-	$(SANDBOX) $(SPACK) -C $(SOFTWARE_STACK_PROJECT)/modules module tcl refresh --upstream-modules --delete-tree --yes-to-all
+	$(SANDBOX) $(SPACK) -C $(BUILD_ROOT)/modules module tcl refresh --upstream-modules --delete-tree --yes-to-all
 
 post-install: generate-config environments{% if modules %} modules{% endif %}
 
 	$(SANDBOX) $(STORE)/post-install-hook
+	touch post-install
 
-# Create a squashfs file from the installed software.
-store.squashfs: environments generate-config{% if modules %} modules{% endif %}{% if post_install_hook %} post-install{% endif %}
+$(BUILD_ROOT)/store/meta/env.json: post-install
+	$(SANDBOX) envvars.py {% if modules %} --modules {% endif %} --spack $(STORE)/meta/env.json.in $(STORE)/meta/env.json
 
+# Create a squashfs file from the installed software.
+store.squashfs: $(BUILD_ROOT)/store/meta/env.json
 	# clean up the __pycache__ paths in the repo
 	$(SANDBOX) find $(STORE)/repo -type d -name __pycache__ -exec rm -r {} +
 	$(SANDBOX) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -e ./compilers/bootstrap find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -all-root -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3

stackinator/templates/Makefile.environments

@@ -37,8 +37,9 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %}
 {% else %}
 	$(SPACK) env activate --with-view --sh ./{{ env }} > $(STORE)/env/{{ config.view.name }}/activate.sh
 {% endif %}
-{% if config.view.extra.add_compilers or config.view.extra.add_ld_library_path %}
-	$(SOFTWARE_STACK_PROJECT)/add-compiler-links.py  ./{{ env }}/compilers.yaml $(STORE)/env/{{ config.view.name }}/activate.sh $(SOFTWARE_STACK_PROJECT)
+{% if config.view.extra.add_compilers or config.view.extra.set_ld_library_path %}
+	$(BUILD_ROOT)/env-vars.py spack {% if config.view.extra.add_compilers %}--compilers=./{{ env }}/compilers.yaml {% endif %}{% if config.view.extra.set_ld_library_path %}--set_ld_library_path {% endif %}\
+	$(STORE)/env/{{ config.view.name }}/activate.sh $(BUILD_ROOT)
 {% endif %}
 {% endif %}
 	touch $@

stackinator/templates/Makefile.generate-config

@@ -1,7 +1,7 @@
 include ../Make.user
 
 CONFIG_DIR = $(STORE)/config
-MODULE_DIR = $(SOFTWARE_STACK_PROJECT)/modules
+MODULE_DIR = $(BUILD_ROOT)/modules
 
 # These will be the prefixes of the GCCs, LLVMs and NVHPCs in the respective environments.
 ALL_COMPILER_PREFIXES ={% for compiler in all_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %}
@@ -22,10 +22,10 @@ $(CONFIG_DIR)/upstreams.yaml:
 # Copy the cluster-specific packages.yaml file to the configuration.
 # requires compilers.yaml to ensure that the path $(CONFIG_DIR) has been created.
 $(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/compilers.yaml
-	install -m 644 $(SOFTWARE_STACK_PROJECT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml
+	install -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml
 
 $(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/compilers.yaml
-	install -m 644 $(SOFTWARE_STACK_PROJECT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml
+	install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml
 
 # Generate a configuration used to generate the module files
 # The configuration in CONFIG_DIR can't be used for this purpose, because a compilers.yaml

stackinator/templates/stack-debug.sh

@@ -1 +1 @@
-env --ignore-environment PATH=/usr/bin:/bin:{{ build_path }}/spack/bin HOME=$HOME SOFTWARE_STACK_PROJECT={{ build_path }} STORE={{ mount_path }} SPACK_SYSTEM_CONFIG_PATH={{ build_path }}/config SPACK_USER_CACHE_PATH={{ build_path }}/cache SPACK=spack SPACK_COLOR=always SPACK_USER_CONFIG_PATH=/dev/null LC_ALL=en_US.UTF-8 TZ=UTC SOURCE_DATE_EPOCH=315576060 {% if use_bwrap %} {{ build_path }}/bwrap-mutable-root.sh --tmpfs ~ --bind {{ build_path }}/tmp /tmp --bind {{ build_path }}/store {{ mount_path }} {% endif %} bash -noprofile -l
+env --ignore-environment PATH=/usr/bin:/bin:{{ build_path }}/spack/bin HOME=$HOME BUILD_ROOT={{ build_path }} STORE={{ mount_path }} SPACK_SYSTEM_CONFIG_PATH={{ build_path }}/config SPACK_USER_CACHE_PATH={{ build_path }}/cache SPACK=spack SPACK_COLOR=always SPACK_USER_CONFIG_PATH=/dev/null LC_ALL=en_US.UTF-8 TZ=UTC SOURCE_DATE_EPOCH=315576060 {% if use_bwrap %} {{ build_path }}/bwrap-mutable-root.sh --tmpfs ~ --bind {{ build_path }}/tmp /tmp --bind {{ build_path }}/store {{ mount_path }} {% endif %} bash -noprofile -l

unittests/.gitignore

@@ -0,0 +1 @@
+scratch

unittests/data/arbor/env/arbor/env.json

@@ -0,0 +1 @@
+{"version": 1, "values": {"list": {"ACLOCAL_PATH": [{"op": "set", "value": ["/user-environment/env/arbor/share/aclocal", "/usr/share/aclocal"]}], "CMAKE_PREFIX_PATH": [{"op": "set", "value": ["/user-environment/env/arbor"]}], "MANPATH": [{"op": "set", "value": ["/user-environment/env/arbor/share/man", "/usr/share/man", "/user-environment/env/arbor/man"]}], "PATH": [{"op": "set", "value": ["/user-environment/env/arbor/bin", "/usr/bin", "/bin"]}, {"op": "prepend", "value": ["/user-environment/linux-archrolling-zen4/gcc-13.3.0/gcc-13.2.0-rmq2jx2h54owhxopaprg7yg4ocbdqv2j/bin"]}], "PKG_CONFIG_PATH": [{"op": "set", "value": ["/user-environment/env/arbor/share/pkgconfig", "/user-environment/env/arbor/lib/pkgconfig", "/usr/share/pkgconfig", "/usr/lib64/pkgconfig", "/usr/lib/pkgconfig", "/user-environment/env/arbor/lib64/pkgconfig"]}]}, "scalar": {"SPACK_ENV_VIEW": "default", "PYTHONPATH": "/user-environment/env/arbor/lib/python3.11/site-packages:/user-environment/env/arbor/misc"}}}

unittests/data/arbor/env/develop/env.json

@@ -0,0 +1 @@
+{"version": 1, "values": {"list": {"ACLOCAL_PATH": [{"op": "set", "value": ["/user-environment/env/develop/share/aclocal", "/usr/share/aclocal"]}], "CMAKE_PREFIX_PATH": [{"op": "set", "value": ["/user-environment/env/develop"]}], "MANPATH": [{"op": "set", "value": ["/user-environment/env/develop/share/man", "/usr/share/man", "/user-environment/env/develop/man"]}], "PATH": [{"op": "set", "value": ["/user-environment/env/develop/bin", "/usr/bin", "/bin"]}, {"op": "prepend", "value": ["/user-environment/linux-archrolling-zen4/gcc-13.3.0/gcc-13.2.0-rmq2jx2h54owhxopaprg7yg4ocbdqv2j/bin"]}], "PKG_CONFIG_PATH": [{"op": "set", "value": ["/user-environment/env/develop/share/pkgconfig", "/user-environment/env/develop/lib/pkgconfig", "/usr/share/pkgconfig", "/usr/lib64/pkgconfig", "/usr/lib/pkgconfig", "/user-environment/env/develop/lib64/pkgconfig"]}]}, "scalar": {"SPACK_ENV_VIEW": "default", "PYTHONPATH": "/user-environment/env/develop/lib/python3.11/site-packages:/user-environment/env/develop/misc"}}}