Add builtin testing mechanism (#365)

.run/spice.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="spice" type="CMakeRunConfiguration" factoryName="Application" PROGRAM_PARAMS="run -O0 -d -ir ../../media/test-project/test.spice" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="Spice" TARGET_NAME="spice" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="Spice" RUN_TARGET_NAME="spice">
+  <configuration default="false" name="spice" type="CMakeRunConfiguration" factoryName="Application" PROGRAM_PARAMS="test -O0 ../../media/test-project/test.spice" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="Spice" TARGET_NAME="spice" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="Spice" RUN_TARGET_NAME="spice">
     <envs>
       <env name="SPICE_STD_DIR" value="$PROJECT_DIR$/std" />
     </envs>

.run/spicetest.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="spicetest" type="CMakeGoogleTestRunConfigurationType" factoryName="Google Test" PROGRAM_PARAMS="--update-refs=false" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="Spice" TARGET_NAME="spicetest" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="Spice" RUN_TARGET_NAME="spicetest" TEST_MODE="SUITE_TEST">
+  <configuration default="false" name="spicetest" type="CMakeGoogleTestRunConfigurationType" factoryName="Google Test" PROGRAM_PARAMS="--update-refs=true" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="Spice" TARGET_NAME="spicetest" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="Spice" RUN_TARGET_NAME="spicetest" TEST_CLASS="IRGeneratorTests" TEST_MODE="SUITE_TEST">
     <envs>
       <env name="RUN_TESTS" value="ON" />
       <env name="SPICE_STD_DIR" value="$PROJECT_DIR$/std" />

Dockerfile

@@ -5,7 +5,7 @@ ENV TERM="xterm-256color"
 ENV SPICE_DOCKERIZED=1
 
 RUN apk update && apk add --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main --no-cache \
-    build-base ncurses-dev libc6-compat gcompat zstd-dev gcc clang lld && rm -rf /var/cache/apk/*
+    build-base ncurses-dev libc6-compat gcompat zstd-dev gcc libgcc clang lld && rm -rf /var/cache/apk/*
 RUN ln -sf /usr/lib/libncursesw.so.6 /usr/lib/libtinfo.so.6
 
 COPY std/ /usr/lib/spice/std/

docs/docs/cli/build.md

@@ -1,23 +1,23 @@
 ---
-title: Build command
+title: Build subcommand
 ---
 
-The `build` command can be used to compile your Spice project to an executable file and save it to the current directory.
+The `build` subcommand can be used to compile your Spice project to an executable file and save it to the current directory.
 
 ## Usage
 === "Long form"
-    Use the `build` command by executing:
+    Use the `build` subcommand by executing:
     ```sh
-    $ spice build [main-source-file]
+    $ spice build [options] <main-source-file>
     ```
 === "Short form"
     Use the `build` command by executing:
     ```sh
-    $ spice b [mail-source-file]
+    $ spice b [options] <mail-source-file>
     ```
 
 ## Options
-You can apply following options to the `build` command:
+You can apply following options to the `build` subcommand:
 
 | Option       | Long                 | Description                                                                                                     |
 |--------------|----------------------|-----------------------------------------------------------------------------------------------------------------|
@@ -33,11 +33,11 @@ You can apply following options to the `build` command:
 | `-t`         | `--target`           | Target triple for the emitted executable (for cross-compiling). <br> Format: `<arch><sub>-<vendor>-<sys>-<abi>` |
 | `-o`         | `--output`           | Set path for executable output.                                                                                 |
 | `-O<n>`      | -                    | Set optimization level. <br> Valid options: `-O0`, `-O1`, `-O2`, `-O3`, `-Os`, `-Oz`                            |
-| `-m`         | `--build-mode`       | Controls the build mode. Valid values are `debug` and `release`                                                 |
+| `-m`         | `--build-mode`       | Controls the build mode. Valid values are `debug`, `release` and `test`.                                        |
 | `-lto`       | -                    | Enable link-time-optimization                                                                                   |
 | `-g`         | `--debug-info`       | Generate debug info to debug the executable in GDB, etc.                                                        |
 | -            | `--static`           | Produce stand-alone executable by linking statically                                                            |
 | -            | `--no-entry`         | Do not require or generate main function (useful for web assembly target)                                       |
 | -            | `--disable-verifier` | Disable LLVM module and function verification (only recommended for debugging the compiler)                     |
 | -            | `--ignore-cache`     | Compile always and ignore the compile cache                                                                     |
-| -            | `--enable-ast-opt`   | Enable AST optimization (recommended for debugging the compiler)                                                |
+| -            | `--enable-ast-opt`   | Enable AST optimization                                                                                         |

docs/docs/cli/install.md

@@ -1,23 +1,23 @@
 ---
-title: Install command
+title: Install subcommand
 ---
 
-The `install` command can be used to compile your Spice project to an executable file and copy it to a binary directory in the PATH variable.
+The `install` subcommand can be used to compile your Spice project to an executable file and copy it to a binary directory in the PATH variable.
 
 ## Usage
 === "Long form"
-    Use the `install` command by executing:
+    Use the `install` subcommand by executing:
     ```sh
-    $ spice install [main-source-file]
+    $ spice install [options] <main-source-file>
     ```
 === "Short form"
-    Use the `install` command by executing:
+    Use the `install` subcommand by executing:
     ```sh
-    $ spice i [mail-source-file]
+    $ spice i [options] <mail-source-file>
     ```
 
 ## Options
-You can apply following options to the `install` command:
+You can apply following options to the `install` subcommand:
 
 | Option       | Long                 | Description                                                                          |
 |--------------|----------------------|--------------------------------------------------------------------------------------|
@@ -35,4 +35,4 @@ You can apply following options to the `install` command:
 | `-O<n>`      | -                    | Set optimization level. <br> Valid options: `-O0`, `-O1`, `-O2`, `-O3`, `-Os`, `-Oz` |
 | `-m`         | `--build-mode`       | Controls the build mode. Valid values are `debug` and `release`                      |
 | -            | `--ignore-cache`     | Compile always and ignore the compile cache                                          |
-| -            | `--enable-ast-opt`   | Enable AST optimization (recommended for debugging the compiler)                     |
+| -            | `--enable-ast-opt`   | Enable AST optimization                                                              |

docs/docs/cli/run.md

@@ -1,23 +1,23 @@
 ---
-title: Run command
+title: Run subcommand
 ---
 
-The `run` command can be used to compile your Spice project to an executable file and run it immediately.
+The `run` subcommand can be used to compile your Spice project to an executable file and run it immediately.
 
 ## Usage
 === "Long form"
-    Use the `run` command by executing:
+    Use the `run` subcommand by executing:
     ```sh
-    $ spice run [main-source-file]
+    $ spice run [options] <main-source-file>
     ```
 === "Short form"
-    Use the `run` command by executing:
+    Use the `run` subcommand by executing:
     ```sh
-    $ spice r [mail-source-file]
+    $ spice r [options] <main-source-file>
     ```
 
 ## Options
-You can apply following options to the `run` command:
+You can apply following options to the `run` subcommand:
 
 | Option       | Long                 | Description                                                                                 |
 |--------------|----------------------|---------------------------------------------------------------------------------------------|
@@ -36,4 +36,4 @@ You can apply following options to the `run` command:
 | `-g`         | `--debug-info`       | Generate debug info to debug the executable in GDB, etc.                                    |
 | -            | `--disable-verifier` | Disable LLVM module and function verification (only recommended for debugging the compiler) |
 | -            | `--ignore-cache`     | Compile always and ignore the compile cache                                                 |
-| -            | `--enable-ast-opt`   | Enable AST optimization (recommended for debugging the compiler)                            |
+| -            | `--enable-ast-opt`   | Enable AST optimization                                                                     |

docs/docs/cli/test.md

@@ -0,0 +1,37 @@
+---
+title: Test subcommand
+---
+
+The `test` subcommand can be used to run individual tests or the whole test suite.
+
+## Usage
+=== "Long form"
+    Use the `test` subcommand by executing:
+    ```sh
+    $ spice test [options] <test-source-file>
+    ```
+=== "Short form"
+    Use the `test` subcommand by executing:
+    ```sh
+    $ spice t [options] <test-source-file>
+    ```
+
+## Options
+You can apply following options to the `test` subcommand:
+
+| Option       | Long                 | Description                                                                                 |
+|--------------|----------------------|---------------------------------------------------------------------------------------------|
+| `-a`         | `--run-all`          | Print compiler output for debugging.                                                        |
+| `-t`         | `--test`             | Runs a single test case by its name                                                         |
+| `-d`         | `--debug-output`     | Print compiler output for debugging.                                                        |
+| `-cst`       | `--dump-cst`         | Dump CST as serialized string and SVG image                                                 |
+| `-ast`       | `--dump-ast`         | Dump AST as serialized string and SVG image                                                 |
+| `-symtab`    | `--dump-symtab`      | Dump serialized symbol tables                                                               |
+| `-ir`        | `--dump-ir`          | Dump LLVM-IR                                                                                |
+| `-s`, `-asm` | `--dump-assembly`    | Dump Assembly code                                                                          |
+| `-j <n>`     | `--jobs <n>`         | Set number of jobs to parallelize compilation (default is auto)                             |
+| `-O<x>`      | -                    | Set optimization level. <br> Valid options: `-O0`, `-O1`, `-O2`, `-O3`, `-Os`, `-Oz`        |
+| `-g`         | `--debug-info`       | Generate debug info to debug the executable in GDB, etc.                                    |
+| -            | `--disable-verifier` | Disable LLVM module and function verification (only recommended for debugging the compiler) |
+| -            | `--ignore-cache`     | Compile always and ignore the compile cache                                                 |
+| -            | `--enable-ast-opt`   | Enable AST optimization                                                                     |

docs/docs/cli/uninstall.md

@@ -1,17 +1,17 @@
 ---
-title: Uninstall command
+title: Uninstall subcommand
 ---
 
-The `uninstall` command can be used to delete an installed executable again.
+The `uninstall` subcommand can be used to delete an installed executable again.
 
 ## Usage
 === "Long form"
-    Use the `uninstall` command by executing:
+    Use the `uninstall` subcommand by executing:
     ```sh
-    $ spice uninstall [main-source-file]
+    $ spice uninstall [options] <main-source-file>
     ```
 === "Short form"
-    Use the `uninstall` command by executing:
+    Use the `uninstall` subcommand by executing:
     ```sh
-    $ spice u [mail-source-file]
+    $ spice u [options] <mail-source-file>
     ```

docs/docs/language/attributes.md

@@ -47,6 +47,9 @@ ext f<heap byte*> malloc(unsigned long);
 - `core.compiler.mangle: bool (default: true)`: Enable/disable name mangling for the annotated function
 - `core.compiler.mangledName: string`: Set the mangled name for the annotated function
 - `core.linker.dll: bool`: Enable linkage as dll (only relevant for Windows)
+- `test: bool`: Mark the annotated function as test
+- `test.name: string`: Set the name of the test (only procedures with the `test` attribute are considered as tests)
+- `test.skip: bool`: Skip the annotated function (only procedures with the `test` attribute are considered as tests)
 
 
 ## Struct attributes

docs/mkdocs.yml

@@ -76,6 +76,7 @@ nav:
   - CLI Usage:
     - cli/build.md
     - cli/run.md
+    - cli/test.md
     - cli/install.md
     - cli/uninstall.md
   - Tutorials:

media/test-project/test.spice

@@ -1,4 +1,31 @@
-type T dyn;
-type T int|double;
+f<int> add(int a, int b) {
+    return a + b;
+}
 
-f<int> main() {}
+#[test]
+f<bool> testAdd() {
+    assert add(1, 2) == 3;
+    assert add(2, 2) == 4;
+    assert add(3, 2) == 5;
+    return true;
+}
+
+#[test]
+f<bool> testAdd1() {
+    assert add(1, 2) == 3;
+    assert add(2, 2) == 4;
+    assert add(3, 2) == 5;
+    return false;
+}
+
+#[test]
+f<bool> testAdd2() {
+    assert add(1, 2) == 3;
+    assert add(2, 2) == 4;
+    assert add(3, 2) == 5;
+    return false;
+}
+
+f<int> main() {
+    printf("%d\n", add(1, 2));
+}

media/test-project/test2.spice

@@ -1,4 +1,9 @@
 public type Driveable interface {
     public p drive(int);
     public f<bool> isDriving();
+}
+
+#[test]
+f<bool> testDriveable() {
+    return true;
 }

src/SourceFile.h

@@ -167,6 +167,7 @@ class SourceFile {
   std::unordered_map<std::string, std::pair<SourceFile *, const ASTNode *>> dependencies;
   std::vector<const SourceFile *> dependants;
   std::unordered_map<std::string, NameRegistryEntry> exportedNameRegistry;
+  std::vector<const Function *> testFunctions;
 
 private:
   // Private fields

src/ast/Attributes.h

@@ -17,6 +17,9 @@ static constexpr const char *const ATTR_CORE_COMPILER_MANGLED_NAME = "core.compi
 static constexpr const char *const ATTR_CORE_COMPILER_KEEP_ON_NAME_COLLISION = "core.compiler.alwaysKeepOnNameCollision";
 static constexpr const char *const ATTR_CORE_COMPILER_EMIT_VTABLE = "core.compiler.alwaysEmitVTable";
 static constexpr const char *const ATTR_CORE_COMPILER_PACKED = "core.compiler.packed";
+static constexpr const char *const ATTR_TEST = "test";
+static constexpr const char *const ATTR_TEST_NAME = "test.name";
+static constexpr const char *const ATTR_TEST_SKIP = "test.skip";
 
 static const CompileTimeValue DEFAULT_BOOL_COMPILE_VALUE{.boolValue = true};
 
@@ -84,6 +87,27 @@ static const std::unordered_map<std::string, AttrConfigValue> ATTR_CONFIGS = {
             .type = AttrNode::TYPE_BOOL,
         },
     },
+    {
+        ATTR_TEST,
+        {
+            .target = AttrNode::TARGET_FCT_PROC,
+            .type = AttrNode::TYPE_BOOL,
+        },
+    },
+    {
+        ATTR_TEST_NAME,
+        {
+            .target = AttrNode::TARGET_FCT_PROC,
+            .type = AttrNode::TYPE_STRING,
+        },
+    },
+    {
+        ATTR_TEST_SKIP,
+        {
+            .target = AttrNode::TARGET_FCT_PROC,
+            .type = AttrNode::TYPE_BOOL,
+        },
+    },
 };
 
 } // namespace spice::compiler

src/driver/Driver.cpp

@@ -29,6 +29,7 @@ void Driver::init() {
   // Create sub-commands
   addBuildSubcommand();
   addRunSubcommand();
+  addTestSubcommand();
   addInstallSubcommand();
   addUninstallSubcommand();
 
@@ -136,6 +137,14 @@ void Driver::enrich() {
   // Always preserve IR value names when dumping IR
   if (cliOptions.dumpSettings.dumpIR)
     cliOptions.namesForIRValues = true;
+
+  // Enable test mode when test mode was selected
+  if (cliOptions.buildMode == BuildMode::TEST) {
+    cliOptions.testMode = true;
+    cliOptions.noEntryFct = true;
+    cliOptions.generateTestMain = true;
+    cliOptions.buildMode = BuildMode::DEBUG;
+  }
 }
 
 /**
@@ -213,6 +222,29 @@ void Driver::addRunSubcommand() {
   subCmd->add_flag<bool>("--disable-verifier", cliOptions.disableVerifier, "Disable LLVM module and function verification");
 }
 
+/**
+ * Add test subcommand to cli interface
+ */
+void Driver::addTestSubcommand() {
+  // Create sub-command itself
+  CLI::App *subCmd = app.add_subcommand("test", "Builds your Spice program and runs all enclosed tests");
+  subCmd->alias("t");
+  subCmd->ignore_case();
+  subCmd->callback([&]() {
+    shouldCompile = shouldExecute = true; // Requires the source file to be compiled
+    cliOptions.testMode = true;           // Always enable assertions for tests, also in higher opt levels
+    cliOptions.generateTestMain = true;   // An alternative entry function is generated
+    cliOptions.noEntryFct = true;         // To not have two main functions, disable normal main
+  });
+
+  addCompileSubcommandOptions(subCmd);
+
+  // --debug-info
+  subCmd->add_flag<bool>("--debug-info,-g", cliOptions.generateDebugInfo, "Generate debug info");
+  // --disable-verifier
+  subCmd->add_flag<bool>("--disable-verifier", cliOptions.disableVerifier, "Disable LLVM module and function verification");
+}
+
 /**
  * Add install subcommand to cli interface
  */
@@ -259,14 +291,16 @@ void Driver::addCompileSubcommandOptions(CLI::App *subCmd) {
       cliOptions.buildMode = BuildMode::DEBUG;
     else if (inputString == BUILD_MODE_RELEASE)
       cliOptions.buildMode = BuildMode::RELEASE;
+    else if (inputString == BUILD_MODE_TEST)
+      cliOptions.buildMode = BuildMode::TEST;
     else
       throw CliError(INVALID_BUILD_MODE, "Invalid build mode: " + inputString);
 
     return true;
   };
 
   // --build-mode
-  subCmd->add_option("--build-mode,-m", buildModeCallback, "Build mode (debug, release)");
+  subCmd->add_option("--build-mode,-m", buildModeCallback, "Build mode (debug, release, test)");
   // --llvm-args
   subCmd->add_option<std::string>("--llvm-args,-llvm", cliOptions.llvmArgs, "Additional arguments for LLVM")->join(' ');
   // --jobs