Add an expected failure test to ensure things fail if they need to

.github/workflows/_internal_lint_failure.yml

@@ -0,0 +1,40 @@
+# This workflow calls the reusable test workflow and ensures that the
+# tests do not pass.
+#
+# Please don't use this as a reusable workflow, because it is very
+# much not.
+
+name: Internal job invocation that expects a lint failure result
+on:
+  workflow_call:
+    inputs:
+      root:
+        description: "Directory containing the go.mod of the codebase under test"
+        type: string
+        default: "."
+      failing_job:
+        description: "Job that is intended to fail"
+        type: "string"
+
+jobs:
+  lints:
+    uses: "./.github/workflows/lints.yml"
+    with:
+      root: ${{inputs.root}}
+      _internal_continue_on_error: ${{inputs.failing_job}}
+
+  expect_lint_failure:
+    runs-on: ubuntu-latest
+    needs: lints
+    steps:
+      - name: transform expected failure
+        id: expected_failure
+        env:
+          NEEDS_JSON: ${{toJSON(needs)}}
+          JQ_FAIL_PROGRAM: >
+            .lints.outputs["_internal_build_result"] == "failure"
+        run: >
+          echo "did_fail=$(echo "$NEEDS_JSON" | jq -r "$JQ_FAIL_PROGRAM")" | tee -a $GITHUB_OUTPUT
+      - name: expect failure
+        run: exit 1
+        if: steps.expected_failure.outputs.did_fail != 'true'

.github/workflows/e2e.yml

@@ -18,3 +18,9 @@ jobs:
     uses: "./.github/workflows/tests.yml"
     with:
       root: "./tests/success"
+
+  fail_on_path_inputs:
+    uses: "./.github/workflows/_internal_lint_failure.yml"
+    with:
+      root: "./tests/fail-safety-check"
+      failing_job: "flake_safety"

.github/workflows/lints.yml

@@ -7,9 +7,18 @@ on:
         description: "Directory that the nix flake resides in."
         default: "."
 
+      # Used for testing this repo:
+      _internal_continue_on_error:
+        description: "Name of the job to set continue-on-error on; pass this only in the tests _inside this repo_. Otherwise your workflow run will pass when it shouldn't."
+        type: string
+        default: ""
+
 jobs:
   fmt:
     name: "nix fmt ${{inputs.root}}"
+    continue-on-error: ${{inputs._internal_continue_on_error == 'fmt'}}
+    outputs:
+      result: ${{steps.result.outcome}}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -18,9 +27,13 @@ jobs:
       - run: cd ${{ inputs.root }} && nix fmt
       - name: Show unformatted files, if any
         run: git diff --exit-code
+        id: result
 
   flake_safety:
     name: "nix flake safety"
+    continue-on-error: ${{inputs._internal_continue_on_error == 'flake_safety'}}
+    outputs:
+      result: ${{steps.result.outcome}}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -36,6 +49,7 @@ jobs:
         id: path_nodes
       - name: fail if bad lock entries are present
         if: ${{ steps.path_nodes.outputs.entries != '' }}
+        id: result
         run: |
           echo "The following lock entries are locked as 'path' types, causing the flake to probably be un-usable on machines that don't have that path present:"
           echo "${{steps.path_nodes.outputs.entries}}"

tests/fail-safety-check/flake.nix

@@ -0,0 +1,32 @@
+{
+  description = "a flake that should successfully pass the baseline-nix tests";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    some_input = {
+      url = "path:./some_input";
+      flake = false;
+    };
+  };
+
+  outputs = {nixpkgs, ...}: let
+    systems = [
+      "aarch64-darwin"
+      "aarch64-linux"
+      "riscv64-linux"
+      "x86_64-darwin"
+      "x86_64-linux"
+    ];
+    eachSystem = f:
+      nixpkgs.lib.genAttrs systems (
+        system:
+          f rec {
+            inherit system;
+            pkgs = nixpkgs.legacyPackages.${system};
+          }
+      );
+  in {
+    formatter = eachSystem ({pkgs, ...}: pkgs.alejandra);
+    packages = eachSystem ({pkgs, ...}: {default = pkgs.hello;});
+  };
+}