lib.packagesFromDirectoryRecursive: use explicit recursion, support nested scopes

lib/filesystem.nix

@@ -12,11 +12,6 @@ let
     toString
     ;
 
-  inherit (lib.attrsets)
-    mapAttrs'
-    filterAttrs
-    ;
-
   inherit (lib.filesystem)
     pathIsDirectory
     pathIsRegularFile
@@ -26,7 +21,6 @@ let
 
   inherit (lib.strings)
     hasSuffix
-    removeSuffix
     ;
 in
 
@@ -312,11 +306,13 @@ in
     # Type
 
     ```
-    packagesFromDirectoryRecursive :: {
+    packagesFromDirectoryRecursive :: (args :: {
       callPackage :: Path -> {} -> a,
+      newScope? :: AttrSet -> scope,
       directory :: Path,
-      ...
-    } -> AttrSet
+      recurseIntoDirectory? :: (args -> AttrSet) -> args -> AttrSet,
+      recurseArgs? :: Any
+    }) -> AttrSet
     ```
 
     # Inputs
@@ -325,9 +321,36 @@ in
     : The function used to convert a Nix file's path into a leaf of the attribute set.
       It is typically the `callPackage` function, taken from either `pkgs` or a new scope corresponding to the `directory`.
 
+    `newScope`
+    : If present, this function is used by the default `recurseIntoDirectory` to generate a new scope.
+      The arguments are updated with the scope's `callPackage` and `newScope` functions, so packages can require
+      anything in their scope, or in an ancestor of their scope.
+      This argument has no effect when `recurseIntoDirectory` is provided.
+
     `directory`
     : The directory to read package files from.
 
+    `recurseIntoDirectory`
+    : This argument is applied to the function which processes directories.
+    : Equivalently, this function takes `processDir` and `args`, and can modify arguments passed to `processDir`
+      (same as above) before calling it, as well as modify its output (which is then returned by `recurseIntoDirectory`).
+
+      :::{.note}
+      When `newScope` is set, the default `recurseIntoDirectory` is equivalent to:
+      ```nix
+      processDir: { newScope, ... }@args:
+        # create a new scope and mark it `recurseForDerivations`
+        lib.recurseIntoAttrs (lib.makeScope newScope (self:
+          # generate the attrset representing the directory, using the new scope's `callPackage` and `newScope`
+          processDir (args // {
+            inherit (self) callPackage newScope;
+          })
+        ))
+      ```
+      :::
+
+    `recurseArgs`
+    : Optional argument, which can be hold data used by `recurseIntoDirectory`
 
     # Examples
     :::{.example}
@@ -348,12 +371,10 @@ in
     ::::{.example}
     ## Create a scope for the nix files found in a directory
     ```nix
-    lib.makeScope pkgs.newScope (
-      self: packagesFromDirectoryRecursive {
-        inherit (self) callPackage;
-        directory = ./my-packages;
-      }
-    )
+    packagesFromDirectoryRecursive {
+      inherit (pkgs) callPackage newScope;
+      directory = ./my-packages;
+    }
     => { ... }
     ```
 
@@ -372,46 +393,100 @@ in
     :::{.note}
     `a.nix` cannot directly take as inputs packages defined in a child directory, such as `b1`.
     :::
+    ::::
 
-    :::{.warning}
-    As of now, `lib.packagesFromDirectoryRecursive` cannot create nested scopes for sub-directories.
+    :::{.example}
+    ## Mark with `recurseIntoAttrs` when recursing into a directory
+    ```nix
+    packagesFromDirectoryRecursive {
+      inherit (pkgs) callPackage;
+      directory = ./my-packages;
 
-    In particular, files under `b/` can only require (as inputs) other files under `my-packages`,
-    but not to those in the same directory, nor those in a parent directory; e.g, `b2.nix` cannot directly
-    require `b1`.
+      recurseIntoDirectory = processDir: args: lib.recurseIntoAttrs (processDir args);
+    }
+    ```
+    :::
+
+    :::{.example}
+    ## Express custom recursion behaviour with `recurseIntoDirectory`
+    For instance, only mark attrsets produced by `packagesFromDirectoryRecursive` with `recurseForDerivations`
+    if they (transitively) contain derivations.
+
+    ```nix
+    packagesFromDirectoryRecursive {
+      inherit (pkgs) callPackage;
+      directory = ./my-packages;
+
+      recurseIntoDirectory = processDir: args: let
+        result = processDir args;
+      in result // {
+        recurseForDerivations = with lib;
+          any (child: isDerivation child || child.recurseForDerivations or false) result;
+      };
+    }
+    ```
     :::
-    ::::
   */
   packagesFromDirectoryRecursive =
+    let
+      inherit (lib) concatMapAttrs id makeScope recurseIntoAttrs removeSuffix;
+      inherit (lib.path) append;
+
+      # Generate an attrset corresponding to a given directory.
+      # This function is outside `packagesFromDirectoryRecursive`'s lambda expression,
+      #  to prevent accidentally using its parameters.
+      processDir = { callPackage, directory, ... }@args:
+        concatMapAttrs (name: type:
+          # for each directory entry
+          let path = append directory name; in
+          if type == "directory" then {
+            # recurse into directories
+            "${name}" = packagesFromDirectoryRecursive (args // {
+              directory = path;
+            });
+          } else if type == "regular" && hasSuffix ".nix" name then {
+            # call .nix files
+            "${removeSuffix ".nix" name}" = callPackage path {};
+          } else if type == "regular" then {
+            # ignore non-nix files
+          } else throw ''
+            lib.filesystem.packagesFromDirectoryRecursive: Unsupported file type ${type} at path ${toString path}
+          ''
+        ) (builtins.readDir directory);
+    in
     {
       callPackage,
+      newScope ? throw "lib.packagesFromDirectoryRecursive: newScope wasn't passed in args",
       directory,
-      ...
-    }:
+      # recurseIntoDirectory can modify the function used when processing directory entries
+      #  and recurseArgs can (optionally) hold data for its use ; see nixdoc above
+      recurseArgs ? throw "lib.packagesFromDirectoryRecursive: recurseArgs wasn't passed in args",
+      recurseIntoDirectory ?
+        if args ? newScope then
+          # `processDir` is the same function as defined above
+          # `args` are the arguments passed to (this recursive call of) `packagesFromDirectoryRecursive`
+          processDir: { newScope, ... }@args:
+            # Create a new scope and mark it `recurseForDerivations`.
+            # This lets the packages refer to each other.
+            # See:
+            #  [lib.makeScope](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.customisation.makeScope) and
+            #  [lib.recurseIntoAttrs](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.customisation.makeScope)
+            recurseIntoAttrs (makeScope newScope (self:
+              # generate the attrset representing the directory, using the new scope's `callPackage` and `newScope`
+              processDir (args // {
+                inherit (self) callPackage newScope;
+              })
+            ))
+        else
+          # otherwise, no modification is necessary
+          id,
+    }@args:
     let
-      inherit (lib) concatMapAttrs removeSuffix;
-      inherit (lib.path) append;
       defaultPath = append directory "package.nix";
     in
     if pathExists defaultPath then
       # if `${directory}/package.nix` exists, call it directly
       callPackage defaultPath {}
-    else concatMapAttrs (name: type:
-      # otherwise, for each directory entry
-      let path = append directory name; in
-      if type == "directory" then {
-        # recurse into directories
-        "${name}" = packagesFromDirectoryRecursive {
-          inherit callPackage;
-          directory = path;
-        };
-      } else if type == "regular" && hasSuffix ".nix" name then {
-        # call .nix files
-        "${removeSuffix ".nix" name}" = callPackage path {};
-      } else if type == "regular" then {
-        # ignore non-nix files
-      } else throw ''
-        lib.filesystem.packagesFromDirectoryRecursive: Unsupported file type ${type} at path ${toString path}
-      ''
-    ) (builtins.readDir directory);
+    else
+      recurseIntoDirectory processDir args;
 }

lib/tests/misc.nix

@@ -2559,7 +2559,7 @@ runTests {
   testPackagesFromDirectoryRecursive = {
     expr = packagesFromDirectoryRecursive {
       callPackage = path: overrides: import path overrides;
-      directory = ./packages-from-directory;
+      directory = ./packages-from-directory/plain;
     };
     expected = {
       a = "a";
@@ -2584,8 +2584,36 @@ runTests {
   testPackagesFromDirectoryRecursiveTopLevelPackageNix = {
     expr = packagesFromDirectoryRecursive {
       callPackage = path: overrides: import path overrides;
-      directory = ./packages-from-directory/c;
+      directory = ./packages-from-directory/plain/c;
     };
     expected = "c";
   };
+
+  # Check that `packagesFromDirectoryRecursive` can be used to create scopes
+  # for sub-directories
+  testPackagesFromDirectoryNestedScopes = let
+    inherit (lib) makeScope recurseIntoAttrs;
+    emptyScope = makeScope lib.callPackageWith (_: {});
+  in {
+    expr = lib.filterAttrsRecursive (name: value: !lib.elem name [ "callPackage" "newScope" "overrideScope" "packages" ]) (packagesFromDirectoryRecursive {
+      inherit (emptyScope) callPackage newScope;
+      directory = ./packages-from-directory/scope;
+    });
+    expected = lib.recurseIntoAttrs {
+      a = "a";
+      b = "b";
+      # Note: Other files/directories in `./test-data/c/` are ignored and can be
+      # used by `package.nix`.
+      c = "c";
+      my-namespace = lib.recurseIntoAttrs {
+        d = "d";
+        e = "e";
+        f = "f";
+        my-sub-namespace = lib.recurseIntoAttrs {
+          g = "g";
+          h = "h";
+        };
+      };
+    };
+  };
 }

lib/tests/packages-from-directory/scope/a.nix

@@ -0,0 +1 @@
+{ }: "a"

lib/tests/packages-from-directory/scope/b.nix

@@ -0,0 +1,3 @@
+{ a }:
+assert a == "a";
+"b"

lib/tests/packages-from-directory/scope/c/not-a-namespace/not-a-package.nix

@@ -0,0 +1 @@
+{ }

lib/tests/packages-from-directory/scope/c/package.nix

@@ -0,0 +1 @@
+{ }: "c"

lib/tests/packages-from-directory/scope/c/support-definitions.nix

@@ -0,0 +1 @@
+{ }

lib/tests/packages-from-directory/scope/my-namespace/d.nix

@@ -0,0 +1,5 @@
+{ a, e }:
+# Check we can get parameter from the parent scope(s) as well as the current one
+assert a == "a";
+assert e == "e";
+"d"

lib/tests/packages-from-directory/scope/my-namespace/e.nix

@@ -0,0 +1,3 @@
+{ d }:
+# Check that mutual recursion is possible
+"e"

lib/tests/packages-from-directory/scope/my-namespace/f/package.nix

@@ -0,0 +1 @@
+{ }: "f"

lib/tests/packages-from-directory/scope/my-namespace/my-sub-namespace/g.nix

@@ -0,0 +1,7 @@
+{
+  a,
+  d,
+  h,
+}:
+# Check we can get parameters from ancestral scopes (e.g. the scope's grandparent)
+"g"

lib/tests/packages-from-directory/scope/my-namespace/my-sub-namespace/h.nix

@@ -0,0 +1 @@
+{ }: "h"

nixos/doc/manual/redirects.json

@@ -149,6 +149,9 @@
   "sec-changing-config": [
     "index.html#sec-changing-config"
   ],
+  "sec-release-25.05-lib-notable-changes": [
+    "release-notes.html#sec-release-25.05-lib-notable-changes"
+  ],
   "sec-upgrading": [
     "index.html#sec-upgrading"
   ],

nixos/doc/manual/release-notes/rl-2505.section.md

@@ -310,3 +310,11 @@
 - Structure of the `functor` of some types has changed. `functor` is an implementation detail and should not be relied upon. If you did rely on it let us know in this [PR](https://github.com/NixOS/nixpkgs/pull/363565).
   - [`lib.types.enum`](https://nixos.org/manual/nixos/unstable/#sec-option-types-basic): Previously the `functor.payload` was the list of enum values directly. Now it is an attribute set containing the values in the `values` attribute.
   - [`lib.types.separatedString`](https://nixos.org/manual/nixos/unstable/#sec-option-types-string): Previously the `functor.payload` was the seperator directly. Now it is an attribute set containing the seperator in the `sep` attribute.
+
+- [`lib.packagesFromDirectoryRecursive`] now rejects unknown arguments, and applies [`lib.recurseIntoAttrs`] when recursing into a directory.
+  [`lib.packagesFromDirectoryRecursive`]: https://nixos.org/manual/nixpkgs/stable/#function-library-lib.filesystem.packagesFromDirectoryRecursive
+  [`lib.recurseIntoAttrs`]: https://nixos.org/manual/nixpkgs/stable/#function-library-lib.attrsets.recurseIntoAttrs
+
+### Other notable changes {#sec-release-25.05-lib-notable-changes}
+
+- [`lib.packagesFromDirectoryRecursive`] can now construct nested scopes matching the directory tree passed as input.