Phoenix-CI v1.0.5 Public Release

.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

.gitlab-ci.yml

@@ -0,0 +1,24 @@
+---
+test:
+  only:
+    variables:
+      - $CI_RUN != "1"
+  script:
+    - echo "Here we can run some actual CI tests..."
+
+scale:
+  only:
+    variables:
+      - $CI_RUN == "1"
+  before_script:
+    - echo "Preparing the cloud-init config files for runner scaling"
+    - sed -i "s,CI_MASTER_SSHKEY,${CI_MASTER_SSHKEY},g" cloud-config/*.yml
+    - sed -i "s,CI_REGISTRATION_URL,${CI_REGISTRATION_URL},g" cloud-config/*.yml
+    - sed -i "s,CI_REGISTRATION_TOKEN,${CI_REGISTRATION_TOKEN},g" cloud-config/*.yml
+    - echo "Initialize Python virtual environment"
+    - virtualenv -q -p python3 .venv
+    - source .venv/bin/activate
+    - echo "Installing Python modules"
+    - pip -q install -r requirements.txt
+  script:
+    - python phoenix_ci.py -t ${HCLOUD_TOKEN} -d ${CI_DOCKER_RUNNER} -s ${CI_SHELL_RUNNER} --servertype=${CI_SERVER_TYPE} --docker-userdata=cloud-config/docker_runner.yml --shell-userdata=cloud-config/shell_runner.yml

CHANGELOG.md

@@ -0,0 +1,45 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [1.0.5] - 2021-03-14
+- first public release of Phoenix-CI
+- renamed all occurences of "worker" to "runner" to be more consistent
+- fixed using default servertype if none is given in a CI run
+
+## [1.0.4] - 2020-01-24 (Internal release)
+- added documentation and code comments for better understanding
+
+## [1.0.3] - 2019-09-12 (Internal release)
+- added a shell-runner fix for [debian buster issue](https://gitlab.com/gitlab-org/gitlab-runner/issues/4449)
+
+## [1.0.2] - 2019-09-11 (Internal release)
+- Changed default OS to Debian 10 Buster
+
+## [1.0.1] - 2019-07-23 (Internal release)
+- Reduced verbose output when running scale-up/down for pip/virtualenv
+- Added feature to check for a running docker-daemon before registering runner
+- Fixed issue with newest docker:dind image and automatic TLS generation
+- Fixed smaller bugs
+
+## [1.0.0] - 2019-06-05 (Internal release)
+- Rewrite of worker creation/deletion without static numbering
+  Now UUIDs are being used, so it's possible to delete any worker
+  without having Phoenix-CI to fail deleting the others afterwards
+
+## [1.0.0-beta2] - 2019-06-01 (Internal release)
+- Renamed Gitlab-HCloud-CI to Phoenix-CI
+- Removed unused hcloud imports
+- Removed support for Python 2.x due to upcoming EOL
+- Fixed reading cloud-config data without using CLI tools
+- Merged both worker scaling methods to a single generic one
+- Refactoring of the print methods
+- Renamed cloud-config yaml to yml
+- Added packer 1.4.1 installation for shell workers
+- Added docker & docker-compose installation for shell workers
+- Added shellcheck & bashate installation for shell workers
+- Added ansible installation for shell workers
+- Made shell workers only run jobs with "shell" tag
+
+## [1.0.0-beta1] - 2019-05-12 (Internal release)
+- Initial release / MVP

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Barzahlen / viacash
+Copyright (C) 2019-2021 viafintech GmbH
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,2 +1,59 @@
-# phoenix-ci
-Phoenix-CI automates the management of Gitlab-CI nodes on the Hetzner Cloud.
+![phoenix_icon](phoenix_icon.png)
+# Phoenix-CI
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/viafintech/phoenix-ci) ![GitHub repo size](https://img.shields.io/github/repo-size/viafintech/phoenix-ci)
+
+Phoenix-CI is a small python-based tool to automate the creation and removal of Gitlab CI runners on the Hetzner Cloud.
+Phoenix-CI was originally developed in May 2019 and has been used since then in production by viafintech GmbH for almost all of their CI jobs - except some that require explicit virtualization for Qemu/VirtualBox.
+
+Phoenix-CI helped us to reduce our monthly CI costs by about 45%* while increasing ability to run parallel jobs and thus also increasing speed for each job. By default one CI job will run on one runner at a time to also reduce interference and fight for resources.
+
+\*We compared costs from a single Hetzner EX41S-SSD (4C/8T Core i7-6700, 64GB, 500GB SSD) to 9 CX21 Cloud servers
+
+Read more about Phoenix-CI in our [dedicated blogpost here](https://www.sysorchestra.com/introducing-phoenix-ci-for-gitlab-for/).
+
+# How does it work?
+Phoenix-CI is a simple python script that utilizes the Hetzner Cloud API to dynamically spawn dedicated gitlab-runner instances for your Gitlab CI without the need and operational costs of Kubernetes.
+For example, it runs in the morning on business days to spawn up fresh runners for the day by a simple Gitlab CI Schedule. At the end of the day another Schedule runs to remove the runners again and deleting the cloud servers to save money. While removing runners in the evening you can also define how many runners you expect to be there, so you can have some spare servers available during the night in case you need to run emergency CI jobs quickly.
+
+Once a week you can also run a full cleanup jobs where you can tell Phoenix-CI to reduce the number of runners to zero. So the next schedule on a monday will spawn fresh runners in the morning.
+
+# What does it support?
+Phoenix-CI currently supports spawning docker (DinD) runners and shell runners but it can easily be extended using own cloud-init configs.
+It also supports defining which types of cloud servers you want to spawn (default is CX21), in which Hetzner Cloud location (default is Falkenstein) and with which operating system (default is Debian 10).
+
+# Requirements
+Phoenix-CI is built to run from two or more Gitlab Schedule Pipelines. One project-specific gitlab-runner is needed to run Phoenix-CI. As a best practice you could run that single gitlab-runner also directly on the Gitlab instance itself and limit it to only run the Phoenix-CI repository.
+This runner must be able to run python3 and needs python-virtualenv and pip to install it's required modules on each run.
+
+# Steps to configure Phoenix-CI
+Follow the steps below to install Phoenix-CI to your on-premise Gitlab. There is currently no official support for hosted Gitlab.
+For a more in-depth explanation of the steps including pictures, please read [this Blogpost for a setup on Debian 10](https://sysorchestra.com/).
+
+- Make sure that you have at least one gitlab-runner "shell" instance running for this project, for example on the Gitlab main instance with the following requirements
+  - Install `python3`, `python3-virtualenv` and `virtualenv` package (tested on Debian)
+  - Become `gitlab-runner` user and generate an ed25519 keypair using `ssh-keygen -t ed25519` and just press enter when asked for location and password
+- Clone this repository into your Gitlab instance and configure the gitlab-runner to only run this project for scaling new gitlab-runner instances on the Hetzner Cloud as well as disable "Shared Runners" in the project's CI configuration.
+- Edit the cloud-config files depending on your needs or create a new one. The 2 examples will work just fine though.
+- Go to project settings -> CI/CD -> Variables and create the following variables needed to properly run Phoenix-CI
+  - CI_MASTER_SSHKEY: The master ssh key used by Phoenix-CI to login to a machine and unregister it from Gitlab
+    - Insert the `id_ed25519.pub` contents here that you generated in step 1
+  - CI_REGISTRATION_TOKEN: Gitlabs CIs runner registration token which can be found in the Admin area under Runners
+  - CI_REGISTRATION_URL: Gitlab CIs runner registration URL which also can be found in the Admin area under Runners
+  - HCLOUD_TOKEN: The Hetzner Cloud token to be used to create the Hetzner Cloud handle/session
+    - Create a separate Hetzner Cloud project for these runners, e.g. "Phoenix-CI"
+    - Then see [here](https://docs.hetzner.cloud/#overview-getting-started) for details how to create an API ke for this project
+- Go to CI/CD -> Schedules and create a schedule to scale up runners (Docker in this example)
+  - Define a time when they should be spawned/deleted as a cron-time
+  - Create at least 2 variables named `CI_DOCKER_RUNNER` and `CI_SHELL_RUNNER` with the amount of desired runners (can be zero though!) and CI_RUN with the value 1
+    - The CI_RUN variable is checked if a scheduled run is intentionally initiated or not
+- Create another Schedule to downscale runners with the same variables but a lower desired amount of runners or zero to completely remove all runners. If you want to hold back runners for emergency runs just use values of 1 or 2 to scale the amount down.
+
+You can also define more schedules depending on your actual needs. You can also define individual schedules for work days and weekends or for times during the day when there should be no Cloud VMs running at all to save money.
+
+# .gitlab-ci.yml
+
+The default .gitlab-ci.yml is automatically used to run a scheduled job. It will check if the current run is desired or not by the CI_RUN variable and if so, it replaces the cloud-inits placeholder config parts with the actual variables defined in step 4 above.
+Afterwards it initializes a virtual python environment and installs all required external tools needed to run the phoenix_ci.py script. The script is ultimately invoked with all parameters given by the schedule variables.
+
+# License and Contributions
+We distribute the whole Phoenix-CI project under the [MIT license](LICENSE). All contributions are welcome.

cloud-config/docker_runner.yml

@@ -0,0 +1,20 @@
+#cloud-config
+runcmd:
+  - echo 'CI_MASTER_SSHKEY' >> /root/.ssh/authorized_keys
+  - curl -L https://get.docker.com | bash
+  - curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash
+  - apt-get install -y gitlab-runner
+  - export CI_SERVER_URL='CI_REGISTRATION_URL'
+  - export REGISTRATION_TOKEN=CI_REGISTRATION_TOKEN
+  - export REGISTER_NON_INTERACTIVE=true
+  - export RUNNER_EXECUTOR=docker
+  - export RUNNER_TAG_LIST=docker
+  - export RUNNER_ENV='DOCKER_TLS_CERTDIR='
+  - export REGISTER_LOCKED=false
+  - export REGISTER_RUN_UNTAGGED=true
+  - export DOCKER_IMAGE='docker:latest'
+  - export DOCKER_CPUS=2
+  - export DOCKER_PRIVILEGED=true
+  - systemctl -q is-active docker && export DOCKER=1 || systemctl restart docker
+  - systemctl -q is-active docker && export DOCKER=1
+  - if test ${DOCKER} = 1; then gitlab-runner register; fi

cloud-config/shell_runner.yml

@@ -0,0 +1,24 @@
+#cloud-config
+runcmd:
+  - echo 'CI_MASTER_SSHKEY' >> /root/.ssh/authorized_keys
+  - curl -L https://get.docker.com | bash
+  - curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+  - chmod +x /usr/local/bin/docker-compose
+  - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible_ubuntu.list
+  - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
+  - curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash
+  - apt-get update && apt-get -y install unzip python-bashate python3-bashate shellcheck gitlab-runner ansible
+  - wget https://releases.hashicorp.com/packer/1.7.0/packer_1.7.0_linux_amd64.zip -O /tmp/packer.zip
+  - unzip /tmp/packer.zip -d /usr/local/bin/
+  - usermod -a -G docker gitlab-runner
+  - export CI_SERVER_URL='CI_REGISTRATION_URL'
+  - export REGISTRATION_TOKEN=CI_REGISTRATION_TOKEN
+  - export REGISTER_NON_INTERACTIVE=true
+  - export RUNNER_EXECUTOR=shell
+  - export RUNNER_TAG_LIST=shell
+  - export RUNNER_ENV='DOCKER_TLS_CERTDIR='
+  - export REGISTER_LOCKED=false
+  - systemctl -q is-active docker && export DOCKER=1 || systemctl restart docker
+  - systemctl -q is-active docker && export DOCKER=1
+  - if test ${DOCKER} = 1; then gitlab-runner register; fi
+  - rm /home/gitlab-runner/.bash_logout # fixes gitlab-runner issue id 4449