diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a81c8ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..c1d5b1c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +--- +test: + only: + variables: + - $CI_RUN != "1" + script: + - echo "Here we can run some actual CI tests..." + +scale: + only: + variables: + - $CI_RUN == "1" + before_script: + - echo "Preparing the cloud-init config files for runner scaling" + - sed -i "s,CI_MASTER_SSHKEY,${CI_MASTER_SSHKEY},g" cloud-config/*.yml + - sed -i "s,CI_REGISTRATION_URL,${CI_REGISTRATION_URL},g" cloud-config/*.yml + - sed -i "s,CI_REGISTRATION_TOKEN,${CI_REGISTRATION_TOKEN},g" cloud-config/*.yml + - echo "Initialize Python virtual environment" + - virtualenv -q -p python3 .venv + - source .venv/bin/activate + - echo "Installing Python modules" + - pip -q install -r requirements.txt + script: + - python phoenix_ci.py -t ${HCLOUD_TOKEN} -d ${CI_DOCKER_RUNNER} -s ${CI_SHELL_RUNNER} --servertype=${CI_SERVER_TYPE} --docker-userdata=cloud-config/docker_runner.yml --shell-userdata=cloud-config/shell_runner.yml diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..39a1c98 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +# Change Log +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](http://semver.org/). + +## [1.0.5] - 2021-03-14 +- first public release of Phoenix-CI +- renamed all occurences of "worker" to "runner" to be more consistent +- fixed using default servertype if none is given in a CI run + +## [1.0.4] - 2020-01-24 (Internal release) +- added documentation and code comments for better understanding + +## [1.0.3] - 2019-09-12 (Internal release) +- added a shell-runner fix for [debian buster issue](https://gitlab.com/gitlab-org/gitlab-runner/issues/4449) + +## [1.0.2] - 2019-09-11 (Internal release) +- Changed default OS to Debian 10 Buster + +## [1.0.1] - 2019-07-23 (Internal release) +- Reduced verbose output when running scale-up/down for pip/virtualenv +- Added feature to check for a running docker-daemon before registering runner +- Fixed issue with newest docker:dind image and automatic TLS generation +- Fixed smaller bugs + +## [1.0.0] - 2019-06-05 (Internal release) +- Rewrite of worker creation/deletion without static numbering + Now UUIDs are being used, so it's possible to delete any worker + without having Phoenix-CI to fail deleting the others afterwards + +## [1.0.0-beta2] - 2019-06-01 (Internal release) +- Renamed Gitlab-HCloud-CI to Phoenix-CI +- Removed unused hcloud imports +- Removed support for Python 2.x due to upcoming EOL +- Fixed reading cloud-config data without using CLI tools +- Merged both worker scaling methods to a single generic one +- Refactoring of the print methods +- Renamed cloud-config yaml to yml +- Added packer 1.4.1 installation for shell workers +- Added docker & docker-compose installation for shell workers +- Added shellcheck & bashate installation for shell workers +- Added ansible installation for shell workers +- Made shell workers only run jobs with "shell" tag + +## [1.0.0-beta1] - 2019-05-12 (Internal release) +- Initial release / MVP diff --git a/LICENSE b/LICENSE index e8bd545..641d49b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 Barzahlen / viacash +Copyright (C) 2019-2021 viafintech GmbH Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 7bb4b52..2b1b70d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,59 @@ -# phoenix-ci -Phoenix-CI automates the management of Gitlab-CI nodes on the Hetzner Cloud. +![phoenix_icon](phoenix_icon.png) +# Phoenix-CI +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![GitHub release (latest by date)](https://img.shields.io/github/v/release/viafintech/phoenix-ci) ![GitHub repo size](https://img.shields.io/github/repo-size/viafintech/phoenix-ci) + +Phoenix-CI is a small python-based tool to automate the creation and removal of Gitlab CI runners on the Hetzner Cloud. +Phoenix-CI was originally developed in May 2019 and has been used since then in production by viafintech GmbH for almost all of their CI jobs - except some that require explicit virtualization for Qemu/VirtualBox. + +Phoenix-CI helped us to reduce our monthly CI costs by about 45%* while increasing ability to run parallel jobs and thus also increasing speed for each job. By default one CI job will run on one runner at a time to also reduce interference and fight for resources. + +\*We compared costs from a single Hetzner EX41S-SSD (4C/8T Core i7-6700, 64GB, 500GB SSD) to 9 CX21 Cloud servers + +Read more about Phoenix-CI in our [dedicated blogpost here](https://www.sysorchestra.com/introducing-phoenix-ci-for-gitlab-for/). + +# How does it work? +Phoenix-CI is a simple python script that utilizes the Hetzner Cloud API to dynamically spawn dedicated gitlab-runner instances for your Gitlab CI without the need and operational costs of Kubernetes. +For example, it runs in the morning on business days to spawn up fresh runners for the day by a simple Gitlab CI Schedule. At the end of the day another Schedule runs to remove the runners again and deleting the cloud servers to save money. While removing runners in the evening you can also define how many runners you expect to be there, so you can have some spare servers available during the night in case you need to run emergency CI jobs quickly. + +Once a week you can also run a full cleanup jobs where you can tell Phoenix-CI to reduce the number of runners to zero. So the next schedule on a monday will spawn fresh runners in the morning. + +# What does it support? +Phoenix-CI currently supports spawning docker (DinD) runners and shell runners but it can easily be extended using own cloud-init configs. +It also supports defining which types of cloud servers you want to spawn (default is CX21), in which Hetzner Cloud location (default is Falkenstein) and with which operating system (default is Debian 10). + +# Requirements +Phoenix-CI is built to run from two or more Gitlab Schedule Pipelines. One project-specific gitlab-runner is needed to run Phoenix-CI. As a best practice you could run that single gitlab-runner also directly on the Gitlab instance itself and limit it to only run the Phoenix-CI repository. +This runner must be able to run python3 and needs python-virtualenv and pip to install it's required modules on each run. + +# Steps to configure Phoenix-CI +Follow the steps below to install Phoenix-CI to your on-premise Gitlab. There is currently no official support for hosted Gitlab. +For a more in-depth explanation of the steps including pictures, please read [this Blogpost for a setup on Debian 10](https://sysorchestra.com/). + +- Make sure that you have at least one gitlab-runner "shell" instance running for this project, for example on the Gitlab main instance with the following requirements + - Install `python3`, `python3-virtualenv` and `virtualenv` package (tested on Debian) + - Become `gitlab-runner` user and generate an ed25519 keypair using `ssh-keygen -t ed25519` and just press enter when asked for location and password +- Clone this repository into your Gitlab instance and configure the gitlab-runner to only run this project for scaling new gitlab-runner instances on the Hetzner Cloud as well as disable "Shared Runners" in the project's CI configuration. +- Edit the cloud-config files depending on your needs or create a new one. The 2 examples will work just fine though. +- Go to project settings -> CI/CD -> Variables and create the following variables needed to properly run Phoenix-CI + - CI_MASTER_SSHKEY: The master ssh key used by Phoenix-CI to login to a machine and unregister it from Gitlab + - Insert the `id_ed25519.pub` contents here that you generated in step 1 + - CI_REGISTRATION_TOKEN: Gitlabs CIs runner registration token which can be found in the Admin area under Runners + - CI_REGISTRATION_URL: Gitlab CIs runner registration URL which also can be found in the Admin area under Runners + - HCLOUD_TOKEN: The Hetzner Cloud token to be used to create the Hetzner Cloud handle/session + - Create a separate Hetzner Cloud project for these runners, e.g. "Phoenix-CI" + - Then see [here](https://docs.hetzner.cloud/#overview-getting-started) for details how to create an API ke for this project +- Go to CI/CD -> Schedules and create a schedule to scale up runners (Docker in this example) + - Define a time when they should be spawned/deleted as a cron-time + - Create at least 2 variables named `CI_DOCKER_RUNNER` and `CI_SHELL_RUNNER` with the amount of desired runners (can be zero though!) and CI_RUN with the value 1 + - The CI_RUN variable is checked if a scheduled run is intentionally initiated or not +- Create another Schedule to downscale runners with the same variables but a lower desired amount of runners or zero to completely remove all runners. If you want to hold back runners for emergency runs just use values of 1 or 2 to scale the amount down. + +You can also define more schedules depending on your actual needs. You can also define individual schedules for work days and weekends or for times during the day when there should be no Cloud VMs running at all to save money. + +# .gitlab-ci.yml + +The default .gitlab-ci.yml is automatically used to run a scheduled job. It will check if the current run is desired or not by the CI_RUN variable and if so, it replaces the cloud-inits placeholder config parts with the actual variables defined in step 4 above. +Afterwards it initializes a virtual python environment and installs all required external tools needed to run the phoenix_ci.py script. The script is ultimately invoked with all parameters given by the schedule variables. + +# License and Contributions +We distribute the whole Phoenix-CI project under the [MIT license](LICENSE). All contributions are welcome. diff --git a/cloud-config/docker_runner.yml b/cloud-config/docker_runner.yml new file mode 100644 index 0000000..e75bdd6 --- /dev/null +++ b/cloud-config/docker_runner.yml @@ -0,0 +1,20 @@ +#cloud-config +runcmd: + - echo 'CI_MASTER_SSHKEY' >> /root/.ssh/authorized_keys + - curl -L https://get.docker.com | bash + - curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash + - apt-get install -y gitlab-runner + - export CI_SERVER_URL='CI_REGISTRATION_URL' + - export REGISTRATION_TOKEN=CI_REGISTRATION_TOKEN + - export REGISTER_NON_INTERACTIVE=true + - export RUNNER_EXECUTOR=docker + - export RUNNER_TAG_LIST=docker + - export RUNNER_ENV='DOCKER_TLS_CERTDIR=' + - export REGISTER_LOCKED=false + - export REGISTER_RUN_UNTAGGED=true + - export DOCKER_IMAGE='docker:latest' + - export DOCKER_CPUS=2 + - export DOCKER_PRIVILEGED=true + - systemctl -q is-active docker && export DOCKER=1 || systemctl restart docker + - systemctl -q is-active docker && export DOCKER=1 + - if test ${DOCKER} = 1; then gitlab-runner register; fi diff --git a/cloud-config/shell_runner.yml b/cloud-config/shell_runner.yml new file mode 100644 index 0000000..cead1d7 --- /dev/null +++ b/cloud-config/shell_runner.yml @@ -0,0 +1,24 @@ +#cloud-config +runcmd: + - echo 'CI_MASTER_SSHKEY' >> /root/.ssh/authorized_keys + - curl -L https://get.docker.com | bash + - curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + - chmod +x /usr/local/bin/docker-compose + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible_ubuntu.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash + - apt-get update && apt-get -y install unzip python-bashate python3-bashate shellcheck gitlab-runner ansible + - wget https://releases.hashicorp.com/packer/1.7.0/packer_1.7.0_linux_amd64.zip -O /tmp/packer.zip + - unzip /tmp/packer.zip -d /usr/local/bin/ + - usermod -a -G docker gitlab-runner + - export CI_SERVER_URL='CI_REGISTRATION_URL' + - export REGISTRATION_TOKEN=CI_REGISTRATION_TOKEN + - export REGISTER_NON_INTERACTIVE=true + - export RUNNER_EXECUTOR=shell + - export RUNNER_TAG_LIST=shell + - export RUNNER_ENV='DOCKER_TLS_CERTDIR=' + - export REGISTER_LOCKED=false + - systemctl -q is-active docker && export DOCKER=1 || systemctl restart docker + - systemctl -q is-active docker && export DOCKER=1 + - if test ${DOCKER} = 1; then gitlab-runner register; fi + - rm /home/gitlab-runner/.bash_logout # fixes gitlab-runner issue id 4449 diff --git a/phoenix_ci.py b/phoenix_ci.py new file mode 100755 index 0000000..06f6f9d --- /dev/null +++ b/phoenix_ci.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# # -*- coding: utf-8 -*- + +""" +Phoenix-CI provides easy management of Gitlab CI nodes +on the Hetzner Cloud. +""" + +# Import python modules needed for this tool to run properly. +import sys + +try: + import argparse + import subprocess + import uuid + import hcloud + +except ImportError as e: + print("Missing python module: {}".format(e.message)) + sys.exit(255) + + +__author__ = 'Martin Seener' +__copyright__ = 'Copyright (C) 2019-2021 viafintech GmbH' +__license__ = 'MIT' +__version__ = '1.0.5' +__maintainer__ = 'Martin Seener' +__email__ = 'martin.seener@viafintech.com' +__status__ = 'Production' + + +def remove_runner(hc, server): + """ + The remove_runner function takes a hetzner cloud handle (session) and a server definition, + logs into the server via SSH to unregister this runner from the Gitlab instance. + After a successful unregister the runner is destroyed (VM deleted from the Cloud). + """ + serverip = server.public_net.ipv4.ip + remove_runner = True + # Login via SSH and try to unregister this runner from Gitlab + resp = subprocess.run( + 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@' + + serverip + + ' -C "gitlab-runner unregister --all-runners"', + shell=True, + stderr=subprocess.DEVNULL + ) + if resp.returncode != 0: + remove_runner = False + + # Remove the cloud server/runner entirely (delete) + response = hc.servers.delete(server) + + return(response.status, remove_runner) + + +def create_runner(hc, type, datacenter, servertype, image, userdata): + """ + The create_runner function takes a hetzner cloud handle (session) and some runner definitions to + create a new cloud server to be used as a Gitlab CI runner. + + Parameters: + datacenter: The datacenter where the new Cloud VM should be created + name: The name of the new Cloud VM (default: type and a UUID, for ex. docker-2288393hd92dh9hd23d8h92d) + server_type: The Cloud VM type. This defines the amount of CPU, RAM, Disk of the VM + image: The Operating System Base image to be used for spawning the new Cloud VM (e.g. Debian) + ssh_keys: A set of SSH Public keys to be installed into the new Cloud VM for remove management + user_data: Takes a cloud-init config for further configuration of the new Cloud VM after spawning (e.g. install additional tools) + labels: Adds a label depending on the "type" parameter that is true. This is used to easily find corresponding Cloud VMs easier for + later removal + """ + response = hc.servers.create( + datacenter=hc.datacenters.get_by_name(name=datacenter), + name='{}-{}'.format(type, str(uuid.uuid4())), + server_type=hc.server_types.get_by_name(name=servertype), + image=hc.images.get_by_name(name=image), + ssh_keys=hc.ssh_keys.get_all(), + user_data=userdata, + labels={type: "true"} + ) + + return(response.action.status) + + +def scale_runner(hc, type, amount, datacenter, servertype, image, userdata): + """ + The scale_runner function is the tools main function which is responsible to call create_runner or remove_runner functions + depending of the number of desired runner it got through the tools parameters (e.g. spawn new servers or remove runners). + + Parameters: + hc: A valid Hetzner Cloud handle (session) + type: The type of Cloud runner to be created. Currently only docker or shell is supported + amount: The total desired amount of "type" Cloud runners to have available in the Cloud. Depending on the actual amount currently of + currently available runners, the runners are scaled up or down in the cloud + datacenter: The datacenter where the new Cloud VM should be created + server_type: The Cloud VM type. This defines the amount of CPU, RAM, Disk of the VM + image: The Operating System Base image to be used for spawning the new Cloud VM (e.g. Debian) + userdata: Takes a cloud-init config for further configuration of the new Cloud VM after spawning (e.g. install additional tools) + """ + # Get all currently available Cloud VMs of a specific type. + server_list = hc.servers.get_all(label_selector=type) + # Read the cloud-init config files contents of a specific type. + with open(userdata, 'r') as file: + runner_userdata = file.read() + + # Cound the number of currently running Cloud VMs of the specified type. + count = len(server_list) + """ + If the available amount of running servers is larger than the desired amount, remove Cloud VM runners one by one until desired number is reached + by calling the remove_runner function for each server to be removed. + + If the available amount if running servers is lower than the desired amount, spawn a new Cloud VM runner of the desired type until the desired + number of running Cloud VMs is reached. + """ + if count > amount: + for i in range(count - amount): + print( + "Deleting {}: ".format(server_list[i].name), + end='', + flush=True + ) + resp, rmr = remove_runner(hc, server_list[i]) + print("{} (Runner unregistered: {})".format(resp, rmr)) + elif count < amount: + for i in range(amount - count): + print( + "Creating new {}-Runner: ".format(type.title()), + end='', + flush=True + ) + resp = create_runner( + hc, + type, + datacenter, + servertype, + image, + runner_userdata + ) + print(resp) + else: + # If the number of running Cloud VMs is equal to the amount of desired runners, do nothing but return the current state of running/desired runners. + print("{}-Runner: {}/{} up".format(type.title(), count, amount)) + + +def main(args): + """ + The main function is called before anything else. In the beginning it will collect all parameters given by the user when the tool is started. + Some arguments have defaults defined if no arguments are given. + After gathering of all arguments, the arguments are checked for validity, the Hetzner Cloud handle is being created (new API session) + and all arguments are then handed over to the scale_runner function for further processing. + """ + parser = argparse.ArgumentParser( + description='\ + Phoenix-CI provides easy management of\ + Gitlab CI nodes on the Hetzner Cloud. You can\ + manage gitlab-runner nodes with it.', + ) + + parser.add_argument( + '-t', + '--token', + type=str, + help='Enter the API Token of your Hetzner Cloud project.', + ) + parser.add_argument( + '-d', + '--docker-runner', + dest='docker_runner', + type=int, + default=2, + help='Enter the amount of CI Runners with the "Docker" executor\ + that you want to have as a positive integer (default: 2).', + ) + parser.add_argument( + '-s', + '--shell-runner', + dest='shell_runner', + type=int, + default=1, + help='Enter the amount of CI Runners with the "Shell" executor\ + that you want to have as a positive integer (default: 1).', + ) + """ + Optional arguments for fine tuning creation + """ + parser.add_argument( + '--datacenter', + type=str, + default='fsn1-dc14', + help='Enter the name of the desired Hetzner Cloud datacenter\ + to be used for server creation (default: fsn1-dc14)', + ) + parser.add_argument( + '--servertype', + type=str, + help='Enter the server type to be used\ + for server creation (default: cx21)', + ) + parser.add_argument( + '--image', + type=str, + default='debian-10', + help='Enter the server image to be used\ + for server creation (default: debian-10)', + ) + parser.add_argument( + '--docker-userdata', + dest='dockerdata', + type=str, + default='/dev/null', + help='A file containing valid cloud-init userdata\ + for configuring docker runners.', + ) + parser.add_argument( + '--shell-userdata', + dest='shelldata', + type=str, + default='/dev/null', + help='A file containing valid cloud-init userdata\ + for configuring shell runners.', + ) + + """ + Parse all arguments then check validity of some arguments. Afterwards a new Hetzner Cloud handle (Session) is established + and the arguments are passed over to the scale_runner function for further processing. + """ + args = parser.parse_args() + if isinstance(args.token, str) and isinstance(args.docker_runner, int)\ + and isinstance(args.shell_runner, int): + print('Running Phoenix-CI ' + __version__) + hc = hcloud.Client(token=args.token) + amount = { + 'docker': args.docker_runner, + 'shell': args.shell_runner + } + userdata = { + 'docker': args.dockerdata, + 'shell': args.shelldata + } + if args.servertype == "": + # If empty type is given, assume default + args.servertype = "cx21" + for type in ['docker', 'shell']: + scale_runner( + hc, + type, + amount[type], + args.datacenter, + args.servertype, + args.image, + userdata[type] + ) + else: + # If no or invalid arguments are given, the tools help is printed. + parser.print_help() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/phoenix_icon.png b/phoenix_icon.png new file mode 100644 index 0000000..53f71fe Binary files /dev/null and b/phoenix_icon.png differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..410662a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +certifi==2020.12.5 +chardet==4.0.0 +future==0.18.2 +hcloud==1.11.0 +idna==2.10 +python-dateutil==2.8.1 +requests==2.25.1 +six==1.15.0 +urllib3==1.26.3