Adds CI testing (#4)

* Moves things around for better packaging and testing

* Adds mypy linting and prepares for automated testing

* Minor grammar fix

* Re-ups the lockfile

* Starts adding unit tests

* Starts working on git tests

* Tweaks

* Installs pytest order and mock

* Migrates fixtures into conftest

* Adds tests for the compile route

* Adds test for building docker images

This is a preliminary test, the docker_actions will need some refactoring to make this a bit less cumbersome

* Some extra cleanup

* Finishes out docker tests

* Adds the github action

* oops

* Sorts the results for test_build.py

* Better naming

* Fixes trailing whitespace in readme

* Adds linter checks

* Corrects the linter checks job

* Caching for pre-commit

* Bumps the lockfile

* Adds the CI badge to the readme

* Adds a unit test for more complex compiles

* Docker test ordering

* Corrects flake8 line-length

* Version bump to 0.1.3

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 120
+per-file-ignores =
+	tests/*:F401,F811

.github/workflows/ci.yml

@@ -0,0 +1,62 @@
+name: CI
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - master
+  push:
+
+jobs:
+  run_pytest:
+    name: Run Checks
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Python setup
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install poetry
+        run: |
+          python -m pip install -U poetry
+      - id: cache-poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+      - name: Install dependencies
+        if: steps.cache-poetry.outputs.cache-hit != 'true'
+        run: |
+          poetry install
+      - name: Run tests
+        run: |
+          poetry run test -v
+
+  run_lints:
+    name: Run linters
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Python setup
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install poetry
+        run: |
+          python -m pip install -U poetry
+      - id: cache-poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+      - name: Install dependencies
+        if: steps.cache-poetry.outputs.cache-hit != 'true'
+        run: |
+          poetry install
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-pre_commit-${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Pre-commit linters
+        run: |
+          poetry run pre-commit run --show-diff-on-failure --color=always --all-files

.gitignore

@@ -162,3 +162,6 @@ runs/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# VSCode
+.vscode

.pre-commit-config.yaml

@@ -18,8 +18,6 @@ repos:
     rev: b9a7794
     hooks:
       - id: flake8
-        args:
-          - "--ignore=E501"
         language_version: python3
 
   - repo: https://github.com/asottile/reorder-python-imports
@@ -33,3 +31,8 @@ repos:
     - id: codespell
       additional_dependencies:
       - tomli
+
+  -   repo: https://github.com/pre-commit/mirrors-mypy
+      rev: bd424e4
+      hooks:
+      -   id: mypy

README.md

@@ -1,3 +1,5 @@
+[![CI](https://github.com/OpenDreamProject/od-compiler-bot/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenDreamProject/od-compiler-bot/actions/workflows/ci.yml)
+
 ## Requirements:
 
 1. Python 3.11
@@ -9,7 +11,7 @@
 
 A simple bot that takes arbitrary code and compiles/executes it within a Docker sandbox environment. The Docker containers lack network, compile/execute code as an unprivileged user, only have access to a read-only volume, and self destruct after 30-seconds. It is recommended that you pair this with a frontend, such as the Discord RedBot cog found here: https://github.com/OpenDreamProject/od-cogs
 
-Whenever the OpenDream repository is updated, the server will build an updated Docker image on the next request. This garuntees that the code is always running on the latest version, but, it can also take a few minutes depending on your network speed and hardware. 
+Whenever the OpenDream repository is updated, the server will build an updated Docker image on the next request. This garuntees that the code is always running on the latest version, but, it can also take a few minutes depending on your network speed and hardware.
 
 ## Install:
 

od_compiler/__init__.py

@@ -0,0 +1,38 @@
+from flask import abort
+from flask import Blueprint
+from flask import Flask
+from flask import jsonify
+from flask import request
+from flask import Response
+
+from od_compiler.util.compiler_logger import compile_logger
+from od_compiler.util.docker_actions import compileOD
+
+compile = Blueprint("compile", __name__, url_prefix="/")
+
+
+def create_app(logger_override=None) -> Flask:
+    app = Flask(__name__)
+
+    if logger_override:
+        compile_logger.setLevel(logger_override)
+
+    app.register_blueprint(compile)
+
+    return app
+
+
+@compile.route("/compile", methods=["POST"])
+def startCompile() -> Response:
+    """
+    Takes in arbitrary OD/DM code and returns a JSON response containing compiler and server logs
+    """
+    posted_data = request.get_json()
+    compile_logger.debug(f"Request incoming containing: {posted_data}")
+    if "code_to_compile" in posted_data:
+        compile_logger.info("Request received. Attempting to compile...")
+        args = posted_data["extra_arguments"] if "extra_arguments" in posted_data else None
+        return jsonify(compileOD(posted_data["code_to_compile"], compile_args=args))
+    else:
+        compile_logger.warning(f"Bad request received:\n{request.get_json()}")
+        return abort(400)

app/util/docker_actions.py → od_compiler/util/docker_actions.py

@@ -4,26 +4,28 @@
 
 from gitdb.exc import BadName
 
-import docker
-from app.util.compiler_logger import compile_logger
-from app.util.git_actions import updateOD
-from app.util.utilities import cleanOldRuns
-from app.util.utilities import splitLogs
-from app.util.utilities import stageBuild
-from app.util.utilities import writeOutput
+from docker.client import from_env as docker_from_env
+from docker.errors import BuildError
+from od_compiler.util.compiler_logger import compile_logger
+from od_compiler.util.git_actions import updateOD
+from od_compiler.util.utilities import cleanOldRuns
+from od_compiler.util.utilities import splitLogs
+from od_compiler.util.utilities import stageBuild
+from od_compiler.util.utilities import writeOutput
 
-client = docker.from_env()
+client = docker_from_env()
 
 
 def updateBuildImage() -> None:
     """
     Update OpenDream and then use Docker's build context to see if we need to build a new image.
     """
+    od_path = Path.cwd().joinpath("OpenDream")
     try:
-        updateOD()
+        updateOD(od_repo_path=od_path)
     except BadName:
         compile_logger.warning("There was an error updating the repo. Cleaning up and trying again.")
-        updateOD(clean=True)
+        updateOD(od_repo_path=od_path, clean=True)
 
     compile_logger.info("Building the docker image...")
     client.images.build(
@@ -37,7 +39,7 @@ def updateBuildImage() -> None:
     client.images.prune(filters={"dangling": True})
 
 
-def compileOD(codeText: str, compile_args: list, timeout: int = 30) -> dict:
+def compileOD(codeText: str, compile_args: list[str], timeout: int = 30) -> dict[str, object]:
     """
     Create an OpenDream docker container to compile and run arbitrary code.
     Returns A dictionary containing the compiler and server logs.
@@ -49,13 +51,13 @@ def compileOD(codeText: str, compile_args: list, timeout: int = 30) -> dict:
     """
     try:
         updateBuildImage()
-    except docker.errors.BuildError as e:
+    except BuildError as e:
         results = {"build_error": True, "exception": str(e)}
         return results
 
-    timestamp = datetime.now()
-    timestamp = timestamp.strftime("%Y%m%d-%H.%M.%S.%f")
+    timestamp = datetime.now().strftime("%Y%m%d-%H.%M.%S.%f")
     randomDir = Path.cwd().joinpath(f"runs/{timestamp}")
+    randomDir.mkdir(parents=True)
 
     stageBuild(codeText=codeText, dir=randomDir)
 
@@ -89,7 +91,7 @@ def compileOD(codeText: str, compile_args: list, timeout: int = 30) -> dict:
     parsed_logs = splitLogs(logs=logs, killed=test_killed)
     container.remove(v=True, force=True)
     writeOutput(logs=logs, dir=randomDir)
-    cleanOldRuns()
+    cleanOldRuns(run_dir=Path.cwd().joinpath("runs"))
     compile_logger.info("Run complete!")
 
     if "error" in parsed_logs.keys():

app/util/git_actions.py → od_compiler/util/git_actions.py

@@ -1,31 +1,30 @@
 from pathlib import Path
 
-from git import Repo
+from git.repo import Repo
 
-from app.util.compiler_logger import compile_logger
+from od_compiler.util.compiler_logger import compile_logger
 
 
-OD_REPO_PATH = Path.cwd().joinpath("OpenDream")
-
-
-def updateOD(clean: int = False) -> None:
+def updateOD(od_repo_path: Path, clean: int = False) -> None:
     """
     Update the OpenDream repository if it exists. If it doesn't, clone a fresh copy.
     """
     if clean:
         from shutil import rmtree
 
-        rmtree(OD_REPO_PATH)
+        rmtree(od_repo_path)
 
-    if Path.exists(OD_REPO_PATH):
-        od = Repo(OD_REPO_PATH)
+    if Path.exists(od_repo_path):
+        od = Repo(od_repo_path)
         od.remote().fetch()
         # We reset HEAD to the upstream commit as a faster and more reliable way to stay up to date
         od.head.reset(commit="origin/master", working_tree=True)
     else:
         compile_logger.info("Repo not found. Cloning from GitHub.")
         od = Repo.clone_from(
-            url="https://github.com/OpenDreamProject/OpenDream.git", to_path=OD_REPO_PATH, multi_options=["--depth 1"]
+            url="https://github.com/OpenDreamProject/OpenDream.git",
+            to_path=od_repo_path,
+            multi_options=["--depth 1", "--recurse-submodules", "--shallow-submodules"],
         )
 
     compile_logger.info(f"The OpenDream repo is at: {od.head.commit.hexsha}")
@@ -40,4 +39,4 @@ def updateSubmodules(od_repo: Repo) -> None:
     """
     for submodule in od_repo.submodules:
         submodule.update(init=True, recursive=True)
-        compile_logger.info(f"{submodule.name} is at {submodule.hexsha}")
+        compile_logger.info(f"{submodule.name} is at: {submodule.hexsha}")

app/util/utilities.py → od_compiler/util/utilities.py

@@ -4,22 +4,21 @@
 from os.path import getctime
 from pathlib import Path
 
-from app.util.compiler_logger import compile_logger
+from od_compiler.util.compiler_logger import compile_logger
 
-MAIN_PROC = "proc/main()"
+MAIN_PROC = "/proc/main()"
 CODE_FILE = Path.cwd().joinpath("templates/code.dm")
 TEST_DME = Path.cwd().joinpath("templates/test.dme")
 MAP_FILE = Path.cwd().joinpath("templates/map.dmm")
 OD_CONF = Path.cwd().joinpath("templates/server_config.toml")
 
 
-def cleanOldRuns(num_to_keep: int = 5) -> None:
+def cleanOldRuns(run_dir: Path, num_to_keep: int = 5) -> None:
     """
     Remove the oldest runs, keeping the n most recent runs.
 
     num_to_keep: Number of historic runs that should be maintained
     """
-    run_dir = Path.cwd().joinpath("runs")
     runs = [x for x in run_dir.iterdir() if x.is_dir()]
     runs.sort(key=getctime)
 
@@ -28,12 +27,13 @@ def cleanOldRuns(num_to_keep: int = 5) -> None:
         shutil.rmtree(runs.pop(0))
 
 
-def splitLogs(logs: str, killed: bool = False) -> dict:
+def splitLogs(logs: str, killed: bool = False) -> dict[str, str]:
     """
     Split the container logs into compiler and server logs.
     Returns a dictionary containing 'compiler' and 'server' logs.
 
     logs: Docker container log output to be parsed
+    killed: Boolean indicating if the run was killed early or not
     """
     if killed:
         logs_regex = re.compile(
@@ -66,7 +66,7 @@ def splitLogs(logs: str, killed: bool = False) -> dict:
     return parsed
 
 
-def loadTemplate(line: str, includeProc=True) -> string:
+def loadTemplate(line: str, includeProc=True) -> str:
     """
     Replaces the placeholder lines within the template file with the provided run-code.
     Returns a template string which can be written to a run file.
@@ -79,7 +79,7 @@ def loadTemplate(line: str, includeProc=True) -> string:
 
     if includeProc:
         line = "\n\t".join(line.splitlines())
-        d = {"proc": MAIN_PROC, "code": f"{line}\n"}
+        d = {"proc": MAIN_PROC, "code": f"\t{line}\n"}
     else:
         d = {"proc": line, "code": ""}
 
@@ -88,12 +88,12 @@ def loadTemplate(line: str, includeProc=True) -> string:
 
 def stageBuild(codeText: str, dir: Path) -> None:
     """
-    Create a directory for the current run and copy over the needed files. Creates the run file containing provided arbitrary code.
+    Create a directory for the current run and copy over the needed files.
+    Creates the run file containing provided arbitrary code.
 
     codeText: Arbitrary code to be loaded into a template file
     dir: Run directory that'll house all of the needed files
     """
-    dir.mkdir(parents=True)
     shutil.copyfile(TEST_DME, dir.joinpath("test.dme"))
     shutil.copyfile(MAP_FILE, dir.joinpath("map.dmm"))
     shutil.copyfile(OD_CONF, dir.joinpath("server_config.toml"))