From 3cf0da126f62d1d1c6399476c6d3e924a2bcb17f Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 28 Nov 2023 15:53:22 +0100 Subject: [PATCH 01/52] initial commit --- build.py | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 build.py diff --git a/build.py b/build.py new file mode 100644 index 0000000..c9e3ff7 --- /dev/null +++ b/build.py @@ -0,0 +1,181 @@ +# Create a commandline script using argparse that +# has two main functionalities: create a conda environment and build an image with packer +# The script should have two subcommands: env and build +# the env subcommand can either install, check or delete the conda environment +# install creates an envornment called "vgcn" with ansible and packer installed +# the env subcommand utilizes conda python api +# check checks if the environment exists and if the packages are installed +# delete deletes the environment + +import argparse +import os +import pathlib +import subprocess +import sys +import time +import datetime +import yaml +from conda.cli.python_api import Commands, run_command + + +class Build: + def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path(), packer_path: pathlib.Path(), provisioning: list[str]): + self.OPENSTACK = openstack + self.TEMPLATE = template + self.OS = "-".join(template.split("-", 2)[:2]) + self.CONDA_ENV = conda_env + self.provisioning = provisioning + self.PACKER_PATH = conda_env + \ + "/bin/packer" if conda_env != None else str(packer_path) + + def dry_run(self): + print(self.assemble_packer_command) + + def assemble_packer_command(self): + prov_str = ','.join(f'"{x}"' for x in provisioning) + cmd = self.CONDA_ENV + cmd = cmd + "/bin/packer" + cmd = "packer build " +\ + "-only=" + self.TEMPLATE +\ + '-var="headless=true"' +\ + "-var='groups=[" + prov_str + "]'" +\ + "templates" + return cmd + + def assemble_name(self): + name = "vgcn~" + pv = self.provisioning + if "generic" in self.provisioning: + name = name + "+".join(self.provisioning.remove("generic")) + else: + name = name + "!generic" + "+".join(self.provisioning) + name = name + self.OS + name = name + self.assemble_timestamp() + + # function that creates a string from the local date + seconds from midnight in the format YYYYMMDD-"seconds from midnight" + def assemble_timestamp(self): + today = datetime.date.today() + seconds_since_midnight = time.time() - time.mktime(today.timetuple()) + return today.strftime("%Y%m%d") + "-" + str(int(seconds_since_midnight)) + # return string with the current seconds from midnight + + def assemble_seconds(self): + + + def execute_cmd(self): + return subprocess.run(self.assemble_packer_command(), capture_output=True) + + def build_image(self): + result = self.execute_cmd() + if result.returncode != 0: + raise ValueError(result.stderr) + else: + print(result.stdout) + + +# Create the parser +my_parser = argparse.ArgumentParser(prog='build', + description='Create a conda environment and build an image with packer') + +# Add the arguments + + +# Create subparsers + +subparsers = my_parser.add_subparsers( + help='sub-command help', dest='subparser_name') + +# Create the parser for the "env" command +parser_env = subparsers.add_parser('env', help='env help') +parser_env.add_argument( + 'action', choices=['install', 'check', 'delete'], help='action help') + +# Create the parser for the "install" command +parser_install = subparsers.add_parser('install', help='install help') +# the install subcommand takes optional arguments for the conda environment name and the conda environment file +parser_install.add_argument('action', choices=['install'], help='action help') +parser_install.add_argument('-n', '--name', type=str, help='name help') +parser_install.add_argument('-f', '--file', type=str, help='file help') + +# Create the parser for the "check" command +parser_check = subparsers.add_parser('check', help='check help') +parser_check.add_argument('action', choices=['check'], help='action help') + +# Create the parser for the "delete" command +parser_delete = subparsers.add_parser('delete', help='delete help') +parser_delete.add_argument('action', choices=['delete'], help='action help') + + +# Create the parser for the "build" command + +parser_build = subparsers.add_parser('build', help='build help') +parser_build.add_argument('action', choices=['build'], help='action help') +# the image argument is required and one of the automatically detected images in the templates folder +# this script uses a function to automatically detect *.cfg files and strips the "-anaconda-ks.cfg" to make it a choice for the parser +parser_build.add_argument('image', choices=[x[:-16] for x in os.listdir( + 'templates') if x.endswith('-anaconda-ks.cfg')], help='image help') +# another required positional argument is the provisioning. This are the ansible playbooks, which are located in the ansible folder +# and are automatically detected by this script, the options are the file basenames +parser_build.add_argument('provisioning', choices=[x[:-3] for x in os.listdir( + 'ansible') if x.endswith('.yml')], help='provisioning help') +# the --openstack option specifies if the image should be uploaded to openstack or not +parser_build.add_argument('--openstack', action='store_true', + help='openstack help') +# another option is to publish the image to /static/vgcn via scp +parser_build.add_argument('--publish', action='store_true', + help='publish help') +# with the --dry-run option the script will only print the commands that would be executed +# and the resulting image file name according to the naming scheme +parser_build.add_argument('--dry-run', action='store_true', + help='dry-run help') + +# Execute the parse_args() method + +args = my_parser.parse_args() + +# Create a function to install the conda environment + + +def install_conda_env(name, file): + # check if the environment exists + if Commands().envs()['envs'] != []: + # if the environment exists, check if the packages are installed + if Commands().list()['packages'] == ['ansible', 'packer']: + print('The conda environment is already installed') + else: + # if the environment exists but the packages are not installed, install the packages + run_command(Commands().install, 'ansible', 'packer') + else: + # if the environment does not exist, create the environment and install the packages + run_command(Commands().create, '-n', name, '--file', file) + run_command(Commands().install, 'ansible', 'packer') + +# Create a function to check if the conda environment exists and if the packages are installed + + +def check_conda_env(): + # check if the environment exists + if Commands().envs()['envs'] != []: + # if the environment exists, check if the packages are installed + if Commands().list()['packages'] == ['ansible', 'packer']: + print('The conda environment is installed') + else: + # if the environment exists but the packages are not installed, install the packages + run_command(Commands().install, 'ansible', 'packer') + else: + # if the environment does not exist, print a message + print('The conda environment is not installed') + +# Create a function to delete the conda environment + + +def delete_conda_env(): + # check if the environment exists + if Commands().envs()['envs'] != []: + # if the environment exists, delete the environment + run_command(Commands().remove, '-n', 'vgcn', '--all') + else: + # if the environment does not exist, print a message + print('The conda environment is not installed') + +# Create a function to build the image From 57f08718a4e543fd8100535fb0de03b0c846d3b4 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 28 Nov 2023 17:27:01 +0100 Subject: [PATCH 02/52] dry run working --- build.py | 110 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 30 deletions(-) diff --git a/build.py b/build.py index c9e3ff7..892aee3 100644 --- a/build.py +++ b/build.py @@ -11,57 +11,66 @@ import os import pathlib import subprocess -import sys import time import datetime -import yaml from conda.cli.python_api import Commands, run_command class Build: - def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path(), packer_path: pathlib.Path(), provisioning: list[str]): + def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path(), packer_path: pathlib.Path(), provisioning: [str], comment: str, publish: bool): self.OPENSTACK = openstack self.TEMPLATE = template self.OS = "-".join(template.split("-", 2)[:2]) + self.comment = comment + self.PUBLISH = publish self.CONDA_ENV = conda_env self.provisioning = provisioning - self.PACKER_PATH = conda_env + \ + self.PACKER_PATH = str(conda_env) + \ "/bin/packer" if conda_env != None else str(packer_path) def dry_run(self): - print(self.assemble_packer_command) + print(self.assemble_packer_command()) + print(self.assemble_name()) def assemble_packer_command(self): - prov_str = ','.join(f'"{x}"' for x in provisioning) - cmd = self.CONDA_ENV - cmd = cmd + "/bin/packer" - cmd = "packer build " +\ - "-only=" + self.TEMPLATE +\ - '-var="headless=true"' +\ - "-var='groups=[" + prov_str + "]'" +\ - "templates" - return cmd + prov_str = ','.join(['"' + x + '"' for x in self.provisioning]) + cmd = [str(self.PACKER_PATH)] + cmd += ["build"] + cmd += ["-var='conda_env=" + + str(self.CONDA_ENV) + + "'"] + cmd += ["-only=" + + self.TEMPLATE] + cmd += ['-var="headless=true"' + + "-var='groups=[" + prov_str + "]'"] + cmd += ["templates"] + return " ".join(cmd) def assemble_name(self): - name = "vgcn~" - pv = self.provisioning + name = ["vgcn"] if "generic" in self.provisioning: - name = name + "+".join(self.provisioning.remove("generic")) + prv = self.provisioning + prv.remove("generic") + name += ["+" + x for x in prv] else: - name = name + "!generic" + "+".join(self.provisioning) - name = name + self.OS - name = name + self.assemble_timestamp() + name += ["!generic+" + "+".join(self.provisioning)] + name += [self.OS] + name += [self.assemble_timestamp()] + name += [subprocess.check_output(['git', 'rev-parse', + '--abbrev-ref', 'HEAD']).decode('ascii').strip()] + name += [subprocess.check_output(['git', 'rev-parse', + '--short', 'HEAD']).decode('ascii').strip()] + if self.comment != None: + name += [self.comment] + return "~".join(name) # function that creates a string from the local date + seconds from midnight in the format YYYYMMDD-"seconds from midnight" def assemble_timestamp(self): today = datetime.date.today() seconds_since_midnight = time.time() - time.mktime(today.timetuple()) - return today.strftime("%Y%m%d") + "-" + str(int(seconds_since_midnight)) + return today.strftime("%Y%m%d") + "~" + str(int(seconds_since_midnight)) # return string with the current seconds from midnight - def assemble_seconds(self): - - def execute_cmd(self): return subprocess.run(self.assemble_packer_command(), capture_output=True) @@ -109,15 +118,13 @@ def build_image(self): # Create the parser for the "build" command parser_build = subparsers.add_parser('build', help='build help') -parser_build.add_argument('action', choices=['build'], help='action help') -# the image argument is required and one of the automatically detected images in the templates folder -# this script uses a function to automatically detect *.cfg files and strips the "-anaconda-ks.cfg" to make it a choice for the parser -parser_build.add_argument('image', choices=[x[:-16] for x in os.listdir( + +parser_build.add_argument('image', choices=["-".join(x.split("-", 3)[:3]) for x in os.listdir( 'templates') if x.endswith('-anaconda-ks.cfg')], help='image help') # another required positional argument is the provisioning. This are the ansible playbooks, which are located in the ansible folder # and are automatically detected by this script, the options are the file basenames -parser_build.add_argument('provisioning', choices=[x[:-3] for x in os.listdir( - 'ansible') if x.endswith('.yml')], help='provisioning help') +parser_build.add_argument('provisioning', choices=[x.split(".", 1)[0] for x in os.listdir( + 'ansible') if x.endswith('.yml')], help='provisioning help', nargs='+') # the --openstack option specifies if the image should be uploaded to openstack or not parser_build.add_argument('--openstack', action='store_true', help='openstack help') @@ -128,11 +135,43 @@ def build_image(self): # and the resulting image file name according to the naming scheme parser_build.add_argument('--dry-run', action='store_true', help='dry-run help') +# The user has to specify either --conda-env or --packer-path +# --conda-env specifies the conda environment to use +parser_build.add_argument('--conda-env', type=pathlib.Path, + help='conda-env help') +# --packer-path specifies the path to the packer binary +parser_build.add_argument('--packer-path', type=pathlib.Path, + help='packer-path help') +# --comment is an optional argument to add a comment to the image name +parser_build.add_argument('--comment', type=str, help='comment help') # Execute the parse_args() method args = my_parser.parse_args() + +def main(): + if args.subparser_name == 'env': + if args.action == 'install': + install_conda_env(args.name, args.file) + elif args.action == 'check': + check_conda_env() + elif args.action == 'delete': + delete_conda_env() + elif args.subparser_name == 'build': + if args.dry_run: + Build(args.openstack, args.image, args.conda_env, + args.packer_path, args.provisioning, args.comment, args.publish).dry_run() + else: + Build(args.openstack, args.image, args.conda_env, + args.packer_path, args.provisioning, args.comment, args.publish).build_image() + else: + print('No action specified') + + +if __name__ == '__main__': + main() + # Create a function to install the conda environment @@ -169,6 +208,17 @@ def check_conda_env(): # Create a function to delete the conda environment +def get_active_branch_name(): + + head_dir = Path(".") / ".git" / "HEAD" + with head_dir.open("r") as f: + content = f.read().splitlines() + + for line in content: + if line[0:4] == "ref:": + return line.partition("refs/heads/")[2] + + def delete_conda_env(): # check if the environment exists if Commands().envs()['envs'] != []: From 7f612a52beb6b4c2c0bcf0b31b08f7647705c1bf Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 1 Dec 2023 14:53:08 +0100 Subject: [PATCH 03/52] update gitignore --- .gitignore | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 006a1d2..54f42dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,17 @@ qemu/ virtualbox-iso/ vmware-iso/ - +packer_plugins/ +ansible/collections +ansible/roles/galaxyproject* +ansible/roles/geerlingguy* +ansible/roles/influxdata* +ansible/roles/usegalaxy-eu.a* +ansible/roles/usegalaxy_eu.* +ansible/roles/usegalaxy-eu.c* +ansible/roles/usegalaxy-eu.d* +ansible/roles/usegalaxy-eu.f* +ansible/roles/usegalaxy-eu.t* # Created by https://www.gitignore.io/api/packer ### Packer ### From fb4fd86bad90cf4754d552e0947e529d30312774 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 13 Dec 2023 12:25:20 +0100 Subject: [PATCH 04/52] dependency handling --- build.py | 342 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 198 insertions(+), 144 deletions(-) mode change 100644 => 100755 build.py diff --git a/build.py b/build.py old mode 100644 new mode 100755 index 892aee3..a777449 --- a/build.py +++ b/build.py @@ -1,8 +1,9 @@ +#!/usr/bin/env python # Create a commandline script using argparse that # has two main functionalities: create a conda environment and build an image with packer # The script should have two subcommands: env and build # the env subcommand can either install, check or delete the conda environment -# install creates an envornment called "vgcn" with ansible and packer installed +# install creates an environment called "vgcn" with ansible and packer installed # the env subcommand utilizes conda python api # check checks if the environment exists and if the packages are installed # delete deletes the environment @@ -12,74 +13,20 @@ import pathlib import subprocess import time +import sys +import shutil import datetime -from conda.cli.python_api import Commands, run_command - - -class Build: - def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path(), packer_path: pathlib.Path(), provisioning: [str], comment: str, publish: bool): - self.OPENSTACK = openstack - self.TEMPLATE = template - self.OS = "-".join(template.split("-", 2)[:2]) - self.comment = comment - self.PUBLISH = publish - self.CONDA_ENV = conda_env - self.provisioning = provisioning - self.PACKER_PATH = str(conda_env) + \ - "/bin/packer" if conda_env != None else str(packer_path) +import signal +from keystoneauth1 import loading - def dry_run(self): - print(self.assemble_packer_command()) - print(self.assemble_name()) - - def assemble_packer_command(self): - prov_str = ','.join(['"' + x + '"' for x in self.provisioning]) - cmd = [str(self.PACKER_PATH)] - cmd += ["build"] - cmd += ["-var='conda_env=" - + str(self.CONDA_ENV) - + "'"] - cmd += ["-only=" - + self.TEMPLATE] - cmd += ['-var="headless=true"' + - "-var='groups=[" + prov_str + "]'"] - cmd += ["templates"] - return " ".join(cmd) +from keystoneauth1 import session - def assemble_name(self): - name = ["vgcn"] - if "generic" in self.provisioning: - prv = self.provisioning - prv.remove("generic") - name += ["+" + x for x in prv] - else: - name += ["!generic+" + "+".join(self.provisioning)] - name += [self.OS] - name += [self.assemble_timestamp()] - name += [subprocess.check_output(['git', 'rev-parse', - '--abbrev-ref', 'HEAD']).decode('ascii').strip()] - name += [subprocess.check_output(['git', 'rev-parse', - '--short', 'HEAD']).decode('ascii').strip()] - if self.comment != None: - name += [self.comment] - return "~".join(name) - - # function that creates a string from the local date + seconds from midnight in the format YYYYMMDD-"seconds from midnight" - def assemble_timestamp(self): - today = datetime.date.today() - seconds_since_midnight = time.time() - time.mktime(today.timetuple()) - return today.strftime("%Y%m%d") + "~" + str(int(seconds_since_midnight)) - # return string with the current seconds from midnight +from cinderclient import client +from conda.cli.python_api import Commands, run_command +import conda.exceptions - def execute_cmd(self): - return subprocess.run(self.assemble_packer_command(), capture_output=True) - def build_image(self): - result = self.execute_cmd() - if result.returncode != 0: - raise ValueError(result.stderr) - else: - print(result.stdout) +DIR_PATH = os.path.dirname(os.path.realpath(__file__)) # Create the parser @@ -95,26 +42,16 @@ def build_image(self): help='sub-command help', dest='subparser_name') # Create the parser for the "env" command -parser_env = subparsers.add_parser('env', help='env help') -parser_env.add_argument( - 'action', choices=['install', 'check', 'delete'], help='action help') - -# Create the parser for the "install" command -parser_install = subparsers.add_parser('install', help='install help') -# the install subcommand takes optional arguments for the conda environment name and the conda environment file -parser_install.add_argument('action', choices=['install'], help='action help') -parser_install.add_argument('-n', '--name', type=str, help='name help') -parser_install.add_argument('-f', '--file', type=str, help='file help') - -# Create the parser for the "check" command -parser_check = subparsers.add_parser('check', help='check help') -parser_check.add_argument('action', choices=['check'], help='action help') - -# Create the parser for the "delete" command -parser_delete = subparsers.add_parser('delete', help='delete help') -parser_delete.add_argument('action', choices=['delete'], help='action help') +parser_env = subparsers.add_parser('deps', help='env help') +parser_env.add_argument( + 'path', type=pathlib.Path, help='Absolute path to the conda environment. e.g. /home/foo/miniconda3/envs/vgcn \ + If "install" is chosen, the script will assume the last part of the path as name.') +parser_env.add_argument( + 'env_action', choices=['install', 'delete'], help='\ + "create will install packages specified in requirements.txt if it \ + exists and create environment first otherwise') # Create the parser for the "build" command parser_build = subparsers.add_parser('build', help='build help') @@ -125,12 +62,15 @@ def build_image(self): # and are automatically detected by this script, the options are the file basenames parser_build.add_argument('provisioning', choices=[x.split(".", 1)[0] for x in os.listdir( 'ansible') if x.endswith('.yml')], help='provisioning help', nargs='+') +# --ansible-args lets the user pass additional args to the packer ansible provisioner, useful e.g. in case of ssh problems +parser_build.add_argument('--ansible-args', type=str, + help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora') # the --openstack option specifies if the image should be uploaded to openstack or not parser_build.add_argument('--openstack', action='store_true', help='openstack help') # another option is to publish the image to /static/vgcn via scp -parser_build.add_argument('--publish', action='store_true', - help='publish help') +parser_build.add_argument('--publish', type=pathlib.Path, + help='specify the path to your ssh key for sn06') # with the --dry-run option the script will only print the commands that would be executed # and the resulting image file name according to the naming scheme parser_build.add_argument('--dry-run', action='store_true', @@ -150,82 +90,196 @@ def build_image(self): args = my_parser.parse_args() -def main(): - if args.subparser_name == 'env': - if args.action == 'install': - install_conda_env(args.name, args.file) - elif args.action == 'check': - check_conda_env() - elif args.action == 'delete': - delete_conda_env() - elif args.subparser_name == 'build': - if args.dry_run: - Build(args.openstack, args.image, args.conda_env, - args.packer_path, args.provisioning, args.comment, args.publish).dry_run() - else: - Build(args.openstack, args.image, args.conda_env, - args.packer_path, args.provisioning, args.comment, args.publish).build_image() - else: - print('No action specified') +# Create a function to install the conda environment -if __name__ == '__main__': - main() +def execute_cmd(name: str, proc: subprocess.Popen): + # Open a subprocess and redirect stdout and stderr to Python + try: + p = None + # Register handler to pass keyboard interrupt to the subprocess + + def handler(sig, frame): + print(f"===================== { + name} ABORTED BY USER =========================") + if p: + + p.send_signal(signal.SIGINT) + else: + raise KeyboardInterrupt.add_note() + signal.signal(signal.SIGINT, handler) + + with proc as p: + # Loop through the readable streams in real - time + for line in iter(p.stdout.readline, b''): + # Print the line to the console + sys.stdout.buffer.write(line) + for line in iter(p.stderr.readline, b''): + # Print the error output to the console + sys.stderr.buffer.write(line) + if p.wait(): + raise Exception(f"===================== { + name} FAILED =========================") + else: + print(f"===================== { + name} SUCCESSFUL =========================") + finally: + signal.signal(signal.SIGINT, signal.SIG_DFL) -# Create a function to install the conda environment +def get_active_branch_name(): -def install_conda_env(name, file): - # check if the environment exists - if Commands().envs()['envs'] != []: - # if the environment exists, check if the packages are installed - if Commands().list()['packages'] == ['ansible', 'packer']: - print('The conda environment is already installed') + head_dir = Path(".") / ".git" / "HEAD" + with head_dir.open("r") as f: + content = f.read().splitlines() + + for line in content: + if line[0:4] == "ref:": + return line.partition("refs/heads/")[2] + + +class CondaEnv: + def __init__(self, path: pathlib.Path): + if path.is_file(): + raise FileExistsError( + "The path you specified is a file, not a directory") else: - # if the environment exists but the packages are not installed, install the packages - run_command(Commands().install, 'ansible', 'packer') - else: - # if the environment does not exist, create the environment and install the packages - run_command(Commands().create, '-n', name, '--file', file) - run_command(Commands().install, 'ansible', 'packer') + self.env_path = path + + def install(self): + install_cmd = ["-p", str(self.env_path), "--file", + "requirements.txt", "-c", "conda-forge"] + try: + run_command(Commands.INSTALL, install_cmd) + + except conda.exceptions.EnvironmentLocationNotFound: + run_command(Commands.CREATE, install_cmd) + + def delete(self): + try: + run_command(Commands.REMOVE, "-p", str(self.env_path), "--all") + except conda.exceptions.EnvironmentLocationNotFound: + print(f"Environment {self.env_path} not found") + + +# Create a function to build the image + + +class Build: + def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, packer_path: pathlib.Path, provisioning: [str], comment: str, publish: pathlib.Path, ansible_args: str): + self.openstack = openstack + self.template = template + self.os = "-".join(template.split("-", 2)[:2]) + self.comment = comment + self.publish = publish + self.conda_env = conda_env + self.provisioning = provisioning + self.ansible_args = ansible_args + self.PACKER_PATH = str(conda_env) + \ + "/bin/packer" if conda_env != None else str(packer_path) + self.image_name = self.assemble_name() + + def dry_run(self): + print(self.assemble_packer_envs()) + print(self.assemble_packer_build_command()) + print(self.assemble_name()) -# Create a function to check if the conda environment exists and if the packages are installed + def assemble_packer_init(self): + cmd = str(self.PACKER_PATH) + cmd += " init " + cmd += DIR_PATH + "/templates" + return cmd + def assemble_packer_build_command(self): + cmd = [str(self.PACKER_PATH), "build"] + cmd.append("-only=qemu." + self.template) + cmd.append(DIR_PATH + "/templates") + return " ".join(cmd) + + def assemble_convert_command(self) -> str: + cmd = [str(shutil.which("qemu-img"))] + cmd.append("convert") + cmd.append("-O raw") + cmd.append(self.template) + cmd.append(DIR_PATH + "/images/" + self.assemble_name() + ".raw") + return cmd + + def assemble_packer_envs(self): + env = os.environ.copy() + env["PACKER_PLUGIN_PATH"] = DIR_PATH + "/packer_plugins" + env["PKR_VAR_groups"] = "[" + \ + ','.join(["\"" + x + "\"" for x in self.provisioning]) + "]" + env["PKR_VAR_headless"] = 'true' + if self.ansible_args != None: + env["PKR_VAR_ansible_extra_args"] = self.ansible_args + return env -def check_conda_env(): - # check if the environment exists - if Commands().envs()['envs'] != []: - # if the environment exists, check if the packages are installed - if Commands().list()['packages'] == ['ansible', 'packer']: - print('The conda environment is installed') + def assemble_name(self): + name = ["vgcn"] + if "generic" in self.provisioning: + prv = self.provisioning + prv.remove("generic") + name += ["+" + x for x in prv] else: - # if the environment exists but the packages are not installed, install the packages - run_command(Commands().install, 'ansible', 'packer') - else: - # if the environment does not exist, print a message - print('The conda environment is not installed') + name += ["!generic+" + "+".join(self.provisioning)] + name += [self.os] + name += [self.assemble_timestamp()] + name += [subprocess.check_output(['git', 'rev-parse', + '--abbrev-ref', 'HEAD']).decode('ascii').strip()] + name += [subprocess.check_output(['git', 'rev-parse', + '--short', 'HEAD']).decode('ascii').strip()] + if self.comment != None: + name += [self.comment] + return "~".join(name) -# Create a function to delete the conda environment + # function that creates a string from the local date + seconds from midnight in the format YYYYMMDD-"seconds from midnight" + def assemble_timestamp(self): + today = datetime.date.today() + seconds_since_midnight = time.time() - time.mktime(today.timetuple()) + return today.strftime("%Y%m%d") + "~" + str(int(seconds_since_midnight)) + # return string with the current seconds from midnight + def convert(self): + execute_cmd(name="CONVERT", proc=subprocess.Popen(self.assemble_convert_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) -def get_active_branch_name(): + def build(self): + execute_cmd("BUILD", subprocess.Popen(self.assemble_packer_build_command(), env=self.assemble_packer_envs(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) - head_dir = Path(".") / ".git" / "HEAD" - with head_dir.open("r") as f: - content = f.read().splitlines() + def upload_to_OS(self): + pass - for line in content: - if line[0:4] == "ref:": - return line.partition("refs/heads/")[2] + def publish(self): + scp_cmd = ["scp", "images/" + self.image_name, "usegalaxy"] + execute_cmd("PUBLISH", subprocess.Popen(scp_cmd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) -def delete_conda_env(): - # check if the environment exists - if Commands().envs()['envs'] != []: - # if the environment exists, delete the environment - run_command(Commands().remove, '-n', 'vgcn', '--all') +def main(): + if args.subparser_name == 'deps': + env = CondaEnv(args.path) + if args.env_action == 'install': + env.install() + elif args.env_action == 'delete': + env.delete() + elif args.subparser_name == 'build': + if args.dry_run: + Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, + packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, publish=args.publish).dry_run() + else: + image = Build(args.openstack, args.image, args.conda_env, + args.packer_path, args.provisioning, args.comment, args.publish, ansible_args=args.ansible_args) + image.build() + image.convert() + if args.openstack: + image.upload_to_OS() + if args.publish: + image.publish() + else: - # if the environment does not exist, print a message - print('The conda environment is not installed') + print('No action specified') -# Create a function to build the image + +if __name__ == '__main__': + main() From 6918ad844c9732c9148deb669790e91fcfe92080 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 13 Dec 2023 12:31:34 +0100 Subject: [PATCH 05/52] use conda command directly, remove wrapper --- build.py | 109 +++++++++++++------------------------------------------ 1 file changed, 26 insertions(+), 83 deletions(-) diff --git a/build.py b/build.py index a777449..8e1f4bf 100755 --- a/build.py +++ b/build.py @@ -34,56 +34,34 @@ description='Create a conda environment and build an image with packer') # Add the arguments - - -# Create subparsers - -subparsers = my_parser.add_subparsers( - help='sub-command help', dest='subparser_name') - -# Create the parser for the "env" command -parser_env = subparsers.add_parser('deps', help='env help') - -parser_env.add_argument( - 'path', type=pathlib.Path, help='Absolute path to the conda environment. e.g. /home/foo/miniconda3/envs/vgcn \ - If "install" is chosen, the script will assume the last part of the path as name.') - -parser_env.add_argument( - 'env_action', choices=['install', 'delete'], help='\ - "create will install packages specified in requirements.txt if it \ - exists and create environment first otherwise') -# Create the parser for the "build" command - -parser_build = subparsers.add_parser('build', help='build help') - -parser_build.add_argument('image', choices=["-".join(x.split("-", 3)[:3]) for x in os.listdir( +my_parser.add_argument('image', choices=["-".join(x.split("-", 3)[:3]) for x in os.listdir( 'templates') if x.endswith('-anaconda-ks.cfg')], help='image help') # another required positional argument is the provisioning. This are the ansible playbooks, which are located in the ansible folder # and are automatically detected by this script, the options are the file basenames -parser_build.add_argument('provisioning', choices=[x.split(".", 1)[0] for x in os.listdir( +my_parser.add_argument('provisioning', choices=[x.split(".", 1)[0] for x in os.listdir( 'ansible') if x.endswith('.yml')], help='provisioning help', nargs='+') # --ansible-args lets the user pass additional args to the packer ansible provisioner, useful e.g. in case of ssh problems -parser_build.add_argument('--ansible-args', type=str, - help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora') +my_parser.add_argument('--ansible-args', type=str, + help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora') # the --openstack option specifies if the image should be uploaded to openstack or not -parser_build.add_argument('--openstack', action='store_true', - help='openstack help') +my_parser.add_argument('--openstack', action='store_true', + help='openstack help') # another option is to publish the image to /static/vgcn via scp -parser_build.add_argument('--publish', type=pathlib.Path, - help='specify the path to your ssh key for sn06') +my_parser.add_argument('--publish', type=pathlib.Path, + help='specify the path to your ssh key for sn06') # with the --dry-run option the script will only print the commands that would be executed # and the resulting image file name according to the naming scheme -parser_build.add_argument('--dry-run', action='store_true', - help='dry-run help') +my_parser.add_argument('--dry-run', action='store_true', + help='dry-run help') # The user has to specify either --conda-env or --packer-path # --conda-env specifies the conda environment to use -parser_build.add_argument('--conda-env', type=pathlib.Path, - help='conda-env help') +my_parser.add_argument('--conda-env', type=pathlib.Path, + help='conda-env help') # --packer-path specifies the path to the packer binary -parser_build.add_argument('--packer-path', type=pathlib.Path, - help='packer-path help') +my_parser.add_argument('--packer-path', type=pathlib.Path, + help='packer-path help') # --comment is an optional argument to add a comment to the image name -parser_build.add_argument('--comment', type=str, help='comment help') +my_parser.add_argument('--comment', type=str, help='comment help') # Execute the parse_args() method @@ -137,31 +115,6 @@ def get_active_branch_name(): if line[0:4] == "ref:": return line.partition("refs/heads/")[2] - -class CondaEnv: - def __init__(self, path: pathlib.Path): - if path.is_file(): - raise FileExistsError( - "The path you specified is a file, not a directory") - else: - self.env_path = path - - def install(self): - install_cmd = ["-p", str(self.env_path), "--file", - "requirements.txt", "-c", "conda-forge"] - try: - run_command(Commands.INSTALL, install_cmd) - - except conda.exceptions.EnvironmentLocationNotFound: - run_command(Commands.CREATE, install_cmd) - - def delete(self): - try: - run_command(Commands.REMOVE, "-p", str(self.env_path), "--all") - except conda.exceptions.EnvironmentLocationNotFound: - print(f"Environment {self.env_path} not found") - - # Create a function to build the image @@ -257,28 +210,18 @@ def publish(self): def main(): - if args.subparser_name == 'deps': - env = CondaEnv(args.path) - if args.env_action == 'install': - env.install() - elif args.env_action == 'delete': - env.delete() - elif args.subparser_name == 'build': - if args.dry_run: - Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, - packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, publish=args.publish).dry_run() - else: - image = Build(args.openstack, args.image, args.conda_env, - args.packer_path, args.provisioning, args.comment, args.publish, ansible_args=args.ansible_args) - image.build() - image.convert() - if args.openstack: - image.upload_to_OS() - if args.publish: - image.publish() - + if args.dry_run: + Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, + packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, publish=args.publish).dry_run() else: - print('No action specified') + image = Build(args.openstack, args.image, args.conda_env, + args.packer_path, args.provisioning, args.comment, args.publish, ansible_args=args.ansible_args) + image.build() + image.convert() + if args.openstack: + image.upload_to_OS() + if args.publish: + image.publish() if __name__ == '__main__': From 4686df83f6b8d4bc8d5d2b88edb518a062ae06ea Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 14 Dec 2023 15:49:46 +0100 Subject: [PATCH 06/52] add publish method --- build.py | 148 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 109 insertions(+), 39 deletions(-) diff --git a/build.py b/build.py index 8e1f4bf..b6e270f 100755 --- a/build.py +++ b/build.py @@ -17,13 +17,11 @@ import shutil import datetime import signal -from keystoneauth1 import loading +import threading +from keystoneauth1.identity import v3 from keystoneauth1 import session - -from cinderclient import client -from conda.cli.python_api import Commands, run_command -import conda.exceptions +from glanceclient import client DIR_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -70,6 +68,42 @@ # Create a function to install the conda environment +# Create a function to build the image + + +class Spinner: + busy = False + delay = 0.1 + + @staticmethod + def spinning_cursor(): + while 1: + for cursor in '|/-\\': + yield cursor + + def __init__(self, delay=None): + self.spinner_generator = self.spinning_cursor() + if delay and float(delay): + self.delay = delay + + def spinner_task(self): + while self.busy: + sys.stdout.write(next(self.spinner_generator)) + sys.stdout.flush() + time.sleep(self.delay) + sys.stdout.write('\b') + sys.stdout.flush() + + def __enter__(self): + self.busy = True + threading.Thread(target=self.spinner_task).start() + + def __exit__(self, exception, value, tb): + self.busy = False + time.sleep(self.delay) + if exception is not None: + return False + def execute_cmd(name: str, proc: subprocess.Popen): # Open a subprocess and redirect stdout and stderr to Python @@ -86,28 +120,28 @@ def handler(sig, frame): else: raise KeyboardInterrupt.add_note() signal.signal(signal.SIGINT, handler) - - with proc as p: - # Loop through the readable streams in real - time - for line in iter(p.stdout.readline, b''): - # Print the line to the console - sys.stdout.buffer.write(line) - for line in iter(p.stderr.readline, b''): - # Print the error output to the console - sys.stderr.buffer.write(line) - if p.wait(): - raise Exception(f"===================== { - name} FAILED =========================") - else: - print(f"===================== { - name} SUCCESSFUL =========================") + with Spinner(): + print(f"{name}ing...") + with proc as p: + # Loop through the readable streams in real - time + for line in iter(p.stdout.readline, b''): + # Print the line to the console + sys.stdout.buffer.write(line) + for line in iter(p.stderr.readline, b''): + # Print the error output to the console + sys.stderr.buffer.write(line) + if p.wait(): + raise Exception(f"===================== { + name} FAILED =========================") + else: + print(f"===================== { + name} SUCCESSFUL =========================") finally: signal.signal(signal.SIGINT, signal.SIG_DFL) def get_active_branch_name(): - - head_dir = Path(".") / ".git" / "HEAD" + head_dir = pathlib.Path(".") / ".git" / "HEAD" with head_dir.open("r") as f: content = f.read().splitlines() @@ -115,8 +149,6 @@ def get_active_branch_name(): if line[0:4] == "ref:": return line.partition("refs/heads/")[2] -# Create a function to build the image - class Build: def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, packer_path: pathlib.Path, provisioning: [str], comment: str, publish: pathlib.Path, ansible_args: str): @@ -131,11 +163,14 @@ def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, pack self.PACKER_PATH = str(conda_env) + \ "/bin/packer" if conda_env != None else str(packer_path) self.image_name = self.assemble_name() + self.image_path = pathlib.Path( + DIR_PATH + "/" + self.image_name + '.raw') def dry_run(self): print(self.assemble_packer_envs()) print(self.assemble_packer_build_command()) - print(self.assemble_name()) + print(self.image_name) + print(self.assemble_convert_command()) def assemble_packer_init(self): cmd = str(self.PACKER_PATH) @@ -149,13 +184,14 @@ def assemble_packer_build_command(self): cmd.append(DIR_PATH + "/templates") return " ".join(cmd) - def assemble_convert_command(self) -> str: + def assemble_convert_command(self): cmd = [str(shutil.which("qemu-img"))] cmd.append("convert") - cmd.append("-O raw") - cmd.append(self.template) - cmd.append(DIR_PATH + "/images/" + self.assemble_name() + ".raw") - return cmd + cmd.append("-O") + cmd.append("raw") + cmd.append("./images/" + self.template) + cmd.append(str(self.image_path)) + return " ".join(cmd) def assemble_packer_envs(self): env = os.environ.copy() @@ -197,26 +233,60 @@ def convert(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) def build(self): - execute_cmd("BUILD", subprocess.Popen(self.assemble_packer_build_command(), env=self.assemble_packer_envs(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) + try: + self.clean_image_dir() + execute_cmd("BUILD", subprocess.Popen(self.assemble_packer_build_command(), env=self.assemble_packer_envs(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) + except BaseException: + self.clean_image_dir() + raise Exception("Cleaned faulty or unfinished images") + + def clean_image_dir(self): + if os.path.exists("./images"): + shutil.rmtree("./images") def upload_to_OS(self): - pass + with Spinner(): + auth = v3.ApplicationCredential( + application_credential_secret=os.environ[ + 'OS_APPLICATION_CREDENTIAL_SECRET'], + application_credential_id=os.environ[ + 'OS_APPLICATION_CREDENTIAL_ID'], + auth_url=os.environ['OS_AUTH_URL'] + + ) + + sess = session.Session(auth=auth) + print("CREATing image...") + glance = client.Client('2', session=sess) + image = glance.images.create(name=self.image_name, is_public='False', + disk_format="raw", container_format="bare", data=os.path.basename(self.image_path)) + print(f"CREATEd image with ID {image.id}") + print(f"UPLOADing image...") + res = glance.images.upload( + image.id, open(self.image_path, 'rb')) + if res == None: + raise Exception( + f"===================== UPLOAD FAILED =========================") + else: + print( + f"===================== UPLOAD SUCCESSFUL =========================") def publish(self): - scp_cmd = ["scp", "images/" + self.image_name, "usegalaxy"] + scp_cmd = ["scp", self.image_path, + "sn06.usegalaxy.eu:/data/dnb01/vgcn/" + os.path.basename(self.image_path)] execute_cmd("PUBLISH", subprocess.Popen(scp_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) def main(): + image = Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, + packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, + ansible_args=args.ansible_args, publish=args.publish) if args.dry_run: - Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, - packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, publish=args.publish).dry_run() + image.dry_run() else: - image = Build(args.openstack, args.image, args.conda_env, - args.packer_path, args.provisioning, args.comment, args.publish, ansible_args=args.ansible_args) - image.build() + # image.build() image.convert() if args.openstack: image.upload_to_OS() From ef590cdae604f1b1d6f360df9fc6e134d739172e Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 14 Dec 2023 16:29:34 +0100 Subject: [PATCH 07/52] auth error handling --- build.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/build.py b/build.py index b6e270f..0b554cc 100755 --- a/build.py +++ b/build.py @@ -247,30 +247,28 @@ def clean_image_dir(self): def upload_to_OS(self): with Spinner(): - auth = v3.ApplicationCredential( - application_credential_secret=os.environ[ - 'OS_APPLICATION_CREDENTIAL_SECRET'], - application_credential_id=os.environ[ - 'OS_APPLICATION_CREDENTIAL_ID'], - auth_url=os.environ['OS_AUTH_URL'] - - ) + try: + auth = v3.ApplicationCredential( + application_credential_secret=os.environ[ + 'OS_APPLICATION_CREDENTIAL_SECRET'], + application_credential_id=os.environ[ + 'OS_APPLICATION_CREDENTIAL_ID'], + auth_url=os.environ['OS_AUTH_URL'] + ) + except KeyError: + raise Exception( + f"Please source OpenStack Application Credentials file first") sess = session.Session(auth=auth) - print("CREATing image...") + print("CREATing OpenStack image...") glance = client.Client('2', session=sess) image = glance.images.create(name=self.image_name, is_public='False', disk_format="raw", container_format="bare", data=os.path.basename(self.image_path)) print(f"CREATEd image with ID {image.id}") - print(f"UPLOADing image...") - res = glance.images.upload( + print(f"UPLOADing image to OpenStack...") + glance.images.upload( image.id, open(self.image_path, 'rb')) - if res == None: - raise Exception( - f"===================== UPLOAD FAILED =========================") - else: - print( - f"===================== UPLOAD SUCCESSFUL =========================") + print(f"===================== UPLOAD SUCCESSFUL =========================") def publish(self): scp_cmd = ["scp", self.image_path, From 9199d441c66c9031f01596d7b3d033f960561ddb Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 14 Dec 2023 17:01:37 +0100 Subject: [PATCH 08/52] use subprocess --- build.py | 70 +++++++++++++++++++++++++------------------------------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/build.py b/build.py index 0b554cc..f1bf999 100755 --- a/build.py +++ b/build.py @@ -19,10 +19,6 @@ import signal import threading -from keystoneauth1.identity import v3 -from keystoneauth1 import session -from glanceclient import client - DIR_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -70,7 +66,7 @@ # Create a function to build the image - +# Spinner thanks to stackoverflow user victor-moyseenko class Spinner: busy = False delay = 0.1 @@ -151,12 +147,12 @@ def get_active_branch_name(): class Build: - def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, packer_path: pathlib.Path, provisioning: [str], comment: str, publish: pathlib.Path, ansible_args: str): + def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, packer_path: pathlib.Path, provisioning: [str], comment: str, pvt_key: pathlib.Path, ansible_args: str): self.openstack = openstack self.template = template self.os = "-".join(template.split("-", 2)[:2]) self.comment = comment - self.publish = publish + self.pvt_key = pvt_key self.conda_env = conda_env self.provisioning = provisioning self.ansible_args = ansible_args @@ -193,6 +189,10 @@ def assemble_convert_command(self): cmd.append(str(self.image_path)) return " ".join(cmd) + def assemble_os_command(self): + return ["openstack", "image", "create", "--file", + str(self.image_path), self.image_name] + def assemble_packer_envs(self): env = os.environ.copy() env["PACKER_PLUGIN_PATH"] = DIR_PATH + "/packer_plugins" @@ -221,7 +221,20 @@ def assemble_name(self): name += [self.comment] return "~".join(name) - # function that creates a string from the local date + seconds from midnight in the format YYYYMMDD-"seconds from midnight" + def assemble_scp_command(self): + return ["scp", self.image_path, + "sn06.usegalaxy.eu:/data/dnb01/vgcn/" + os.path.basename(self.image_path)] + + def assemble_ssh_command(self): + cmd = ["ssh"] + cmd.append("-i") + cmd.append(str(self.pvt_key)) + cmd.append("sn06.galaxyproject.eu") + cmd.append("chmod") + cmd.append("ugo+r") + cmd.append("/data/dnb01/vgcn/" + os.path.basename(self.image_path)) + return cmd + def assemble_timestamp(self): today = datetime.date.today() seconds_since_midnight = time.time() - time.mktime(today.timetuple()) @@ -246,50 +259,29 @@ def clean_image_dir(self): shutil.rmtree("./images") def upload_to_OS(self): - with Spinner(): - try: - auth = v3.ApplicationCredential( - application_credential_secret=os.environ[ - 'OS_APPLICATION_CREDENTIAL_SECRET'], - application_credential_id=os.environ[ - 'OS_APPLICATION_CREDENTIAL_ID'], - auth_url=os.environ['OS_AUTH_URL'] - ) - except KeyError: - raise Exception( - f"Please source OpenStack Application Credentials file first") - - sess = session.Session(auth=auth) - print("CREATing OpenStack image...") - glance = client.Client('2', session=sess) - image = glance.images.create(name=self.image_name, is_public='False', - disk_format="raw", container_format="bare", data=os.path.basename(self.image_path)) - print(f"CREATEd image with ID {image.id}") - print(f"UPLOADing image to OpenStack...") - glance.images.upload( - image.id, open(self.image_path, 'rb')) - print(f"===================== UPLOAD SUCCESSFUL =========================") - - def publish(self): - scp_cmd = ["scp", self.image_path, - "sn06.usegalaxy.eu:/data/dnb01/vgcn/" + os.path.basename(self.image_path)] - execute_cmd("PUBLISH", subprocess.Popen(scp_cmd, + execute_cmd("OPENSTACK IMAGE CREATE", subprocess.Popen(self.assemble_os_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) + + def pvt_key(self): + execute_cmd("PUBLISH", subprocess.Popen(self.assemble_scp_command(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) + execute_cmd("PERMISSION CHANGE", subprocess.Popen(self.assemble_ssh_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) def main(): image = Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, - ansible_args=args.ansible_args, publish=args.publish) + ansible_args=args.ansible_args, pvt_key=args.publish) if args.dry_run: image.dry_run() else: - # image.build() + image.build() image.convert() if args.openstack: image.upload_to_OS() if args.publish: - image.publish() + image.pvt_key() if __name__ == '__main__': From a64c3b6c08ae22dd9f2d0ad63bad92b2a1a4f96f Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 13:35:13 +0100 Subject: [PATCH 09/52] mostly documentation --- build.py | 124 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/build.py b/build.py index f1bf999..859894e 100755 --- a/build.py +++ b/build.py @@ -1,12 +1,18 @@ #!/usr/bin/env python -# Create a commandline script using argparse that -# has two main functionalities: create a conda environment and build an image with packer -# The script should have two subcommands: env and build -# the env subcommand can either install, check or delete the conda environment -# install creates an environment called "vgcn" with ansible and packer installed -# the env subcommand utilizes conda python api -# check checks if the environment exists and if the packages are installed -# delete deletes the environment +# A commandline script using argparse that builds a vgcn image with packer +# CAUTION: If a directory called 'images' exists in this directory, it will be deleted! +# Required arguments are the template, which are named like the anaconda-ks.cfg files without the +# anaconda-ks.cfg suffix and the provisioning, separated by space and named like the ansible-playbooks +# in the ansible directory without the .yml suffix. +# Optionally you can specify the path to the Packer binary you want to use (--packer-path) +# the path to the conda env you want to use (--conda-env) +# the path to your ssh private key for copying the images to sn06 and +# make them publicly available (--publish) +# you can also source your OpenStack application credentials first and set the (`--openstack`) flag +# to create a raw image in your openstack tenant +# If you have trouble with the ansible provider on your setup, you can specify additional +# --ansible-args="..." to e.g. solve the issues with scp on some distros + import argparse import os @@ -23,51 +29,43 @@ DIR_PATH = os.path.dirname(os.path.realpath(__file__)) -# Create the parser my_parser = argparse.ArgumentParser(prog='build', - description='Create a conda environment and build an image with packer') + description='Build a VGCN image with Packer and the Ansible provisioner') -# Add the arguments my_parser.add_argument('image', choices=["-".join(x.split("-", 3)[:3]) for x in os.listdir( 'templates') if x.endswith('-anaconda-ks.cfg')], help='image help') -# another required positional argument is the provisioning. This are the ansible playbooks, which are located in the ansible folder -# and are automatically detected by this script, the options are the file basenames my_parser.add_argument('provisioning', choices=[x.split(".", 1)[0] for x in os.listdir( - 'ansible') if x.endswith('.yml')], help='provisioning help', nargs='+') -# --ansible-args lets the user pass additional args to the packer ansible provisioner, useful e.g. in case of ssh problems + 'ansible') if x.endswith('.yml')], help=''' + The playbooks you want to provision. + The playbook files are located in the ansible folder + and are automatically detected by this script, the options are the filenames, without .yml suffix + ''', nargs='+') my_parser.add_argument('--ansible-args', type=str, help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora') -# the --openstack option specifies if the image should be uploaded to openstack or not my_parser.add_argument('--openstack', action='store_true', - help='openstack help') -# another option is to publish the image to /static/vgcn via scp -my_parser.add_argument('--publish', type=pathlib.Path, + help='Create an image in your OpenStack tenant and upload it. Make sure to source your credentials first') +my_parser.add_argument('--publish', type=pathlib.Path, metavar='PVT_KEY', help='specify the path to your ssh key for sn06') -# with the --dry-run option the script will only print the commands that would be executed -# and the resulting image file name according to the naming scheme my_parser.add_argument('--dry-run', action='store_true', - help='dry-run help') -# The user has to specify either --conda-env or --packer-path -# --conda-env specifies the conda environment to use + help='just print the commands without executing anything') my_parser.add_argument('--conda-env', type=pathlib.Path, - help='conda-env help') -# --packer-path specifies the path to the packer binary + help='specifies the path to the conda environment to use') my_parser.add_argument('--packer-path', type=pathlib.Path, - help='packer-path help') -# --comment is an optional argument to add a comment to the image name -my_parser.add_argument('--comment', type=str, help='comment help') + help='specifies the path to the packer binary') +my_parser.add_argument('--comment', type=str, + help='add a comment to the image name') -# Execute the parse_args() method args = my_parser.parse_args() -# Create a function to install the conda environment - -# Create a function to build the image - # Spinner thanks to stackoverflow user victor-moyseenko class Spinner: + """ + Creates a spinning cursor while the command runs.\n + Indicates the user that the screen did not freeze or similar.\n + Especially useful during the image upload, which can take several minutes.\n + """ busy = False delay = 0.1 @@ -101,8 +99,12 @@ def __exit__(self, exception, value, tb): return False -def execute_cmd(name: str, proc: subprocess.Popen): - # Open a subprocess and redirect stdout and stderr to Python +def run_subprocess_with_spinner(name: str, proc: subprocess.Popen): + """ + Opens a subprocess and redirect stdout and stderr to Python.\n + Shows a spinning Cursor while the command runs.\n + Exits with returncode of subprocess if not equals 0.\n + """ try: p = None # Register handler to pass keyboard interrupt to the subprocess @@ -111,15 +113,13 @@ def handler(sig, frame): print(f"===================== { name} ABORTED BY USER =========================") if p: - p.send_signal(signal.SIGINT) else: raise KeyboardInterrupt.add_note() signal.signal(signal.SIGINT, handler) with Spinner(): - print(f"{name}ing...") + print(f"{name.rstrip('Ee')}ing...") with proc as p: - # Loop through the readable streams in real - time for line in iter(p.stdout.readline, b''): # Print the line to the console sys.stdout.buffer.write(line) @@ -137,7 +137,7 @@ def handler(sig, frame): def get_active_branch_name(): - head_dir = pathlib.Path(".") / ".git" / "HEAD" + head_dir = pathlib.Path(DIR_PATH + "/.git/HEAD") with head_dir.open("r") as f: content = f.read().splitlines() @@ -167,6 +167,11 @@ def dry_run(self): print(self.assemble_packer_build_command()) print(self.image_name) print(self.assemble_convert_command()) + if self.openstack != None: + print(self.assemble_os_command()) + if self.pvt_key != None: + print(self.assemble_scp_command()) + print(self.assemble_ssh_command()) def assemble_packer_init(self): cmd = str(self.PACKER_PATH) @@ -204,6 +209,10 @@ def assemble_packer_envs(self): return env def assemble_name(self): + """ + Uses a naming scheme described in\n + https://github.com/usegalaxy-eu/vgcn/issues/78 + """ name = ["vgcn"] if "generic" in self.provisioning: prv = self.provisioning @@ -239,34 +248,29 @@ def assemble_timestamp(self): today = datetime.date.today() seconds_since_midnight = time.time() - time.mktime(today.timetuple()) return today.strftime("%Y%m%d") + "~" + str(int(seconds_since_midnight)) - # return string with the current seconds from midnight - - def convert(self): - execute_cmd(name="CONVERT", proc=subprocess.Popen(self.assemble_convert_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) def build(self): - try: - self.clean_image_dir() - execute_cmd("BUILD", subprocess.Popen(self.assemble_packer_build_command(), env=self.assemble_packer_envs(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) - except BaseException: - self.clean_image_dir() - raise Exception("Cleaned faulty or unfinished images") + self.clean_image_dir() + run_subprocess_with_spinner("BUILD", subprocess.Popen(self.assemble_packer_build_command(), env=self.assemble_packer_envs(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) + + def convert(self): + run_subprocess_with_spinner(name="CONVERT", proc=subprocess.Popen(self.assemble_convert_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) def clean_image_dir(self): - if os.path.exists("./images"): - shutil.rmtree("./images") + if os.path.exists(DIR_PATH + "/images"): + shutil.rmtree(DIR_PATH + "/images") def upload_to_OS(self): - execute_cmd("OPENSTACK IMAGE CREATE", subprocess.Popen(self.assemble_os_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) + run_subprocess_with_spinner("OPENSTACK IMAGE CREATE", subprocess.Popen(self.assemble_os_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) def pvt_key(self): - execute_cmd("PUBLISH", subprocess.Popen(self.assemble_scp_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) - execute_cmd("PERMISSION CHANGE", subprocess.Popen(self.assemble_ssh_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)) + run_subprocess_with_spinner("PUBLISH", subprocess.Popen(self.assemble_scp_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) + run_subprocess_with_spinner("PERMISSION CHANGE", subprocess.Popen(self.assemble_ssh_command(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) def main(): From 6d344b8c621998c893ed722b4986d8bd971bf71d Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 13:36:47 +0100 Subject: [PATCH 10/52] use sys.exit instead of exception --- build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 859894e..963cfd0 100755 --- a/build.py +++ b/build.py @@ -126,9 +126,11 @@ def handler(sig, frame): for line in iter(p.stderr.readline, b''): # Print the error output to the console sys.stderr.buffer.write(line) - if p.wait(): - raise Exception(f"===================== { + returncode = p.wait() + if returncode: + print(f"===================== { name} FAILED =========================") + sys.exit(returncode) else: print(f"===================== { name} SUCCESSFUL =========================") From 8cfaaa999e4633981eb6dbb1d161c5ed2fa055a7 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 15:46:19 +0100 Subject: [PATCH 11/52] add conda env file --- environment.yml | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..7d676fa --- /dev/null +++ b/environment.yml @@ -0,0 +1,60 @@ +name: vgcn-build +channels: + - tsnyder + - anaconda + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - ansible=8.5.0=pyhd8ed1ab_0 + - ansible-core=2.15.4=pyh707e725_0 + - bzip2=1.0.8=hd590300_5 + - c-ares=1.23.0=hd590300_0 + - ca-certificates=2023.11.17=hbcca054_0 + - cffi=1.16.0=py312hf06ca03_0 + - cryptography=41.0.7=py312h4742d6a_1 + - gettext=0.21.1=h27087fc_0 + - jinja2=3.1.2=pyhd8ed1ab_1 + - krb5=1.20.1=h143b758_1 + - ld_impl_linux-64=2.40=h41732ed_0 + - libcurl=7.88.1=hdc1c0ab_1 + - libedit=3.1.20221030=h5eee18b_0 + - libev=4.33=hd590300_2 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=13.2.0=h807b86a_3 + - libglib=2.70.2=h174f98d_4 + - libgomp=13.2.0=h807b86a_3 + - libiconv=1.17=hd590300_1 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libsqlite=3.44.2=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libuuid=2.38.1=h0b41bf4_0 + - libzlib=1.2.13=hd590300_5 + - lz4-c=1.9.3=h9c3ff4c_1 + - markupsafe=2.1.3=py312h98912ed_1 + - ncurses=6.4=h59595ed_2 + - openssl=3.2.0=hd590300_1 + - packaging=23.2=pyhd8ed1ab_0 + - packer=1.10.0=ha8f183a_0 + - pcre=8.45=h9c3ff4c_0 + - pip=23.3.1=pyhd8ed1ab_0 + - pixman=0.40.0=h36c2ea0_0 + - pycparser=2.21=pyhd8ed1ab_0 + - python=3.12.0=hab00c5b_0_cpython + - python_abi=3.12=4_cp312 + - pyyaml=6.0.1=py312h98912ed_1 + - qemu=5.0.0=hc4641f2_0 + - readline=8.2=h8228510_1 + - resolvelib=1.0.1=pyhd8ed1ab_0 + - setuptools=68.2.2=pyhd8ed1ab_0 + - tk=8.6.13=noxft_h4845f30_101 + - tzdata=2023c=h71feb2d_0 + - wheel=0.42.0=pyhd8ed1ab_0 + - xz=5.2.6=h166bdaf_0 + - yaml=0.2.5=h7f98852_2 + - zlib=1.2.13=hd590300_5 + - zstd=1.5.2=h8a70e8d_1 From 04f2a40b34fc7928b80efb3edc4f219dfd429eaf Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 15:52:43 +0100 Subject: [PATCH 12/52] add openstack to env file --- environment.yml | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/environment.yml b/environment.yml index 7d676fa..6360aa7 100644 --- a/environment.yml +++ b/environment.yml @@ -9,13 +9,32 @@ dependencies: - _openmp_mutex=4.5=2_gnu - ansible=8.5.0=pyhd8ed1ab_0 - ansible-core=2.15.4=pyh707e725_0 + - appdirs=1.4.4=pyh9f0ad1d_0 + - attrs=23.1.0=pyh71513ae_1 + - autopage=0.5.2=pyhd8ed1ab_0 + - brotlipy=0.7.0=py312h98912ed_1006 - bzip2=1.0.8=hd590300_5 - c-ares=1.23.0=hd590300_0 - ca-certificates=2023.11.17=hbcca054_0 + - certifi=2023.11.17=pyhd8ed1ab_0 - cffi=1.16.0=py312hf06ca03_0 + - charset-normalizer=3.3.2=pyhd8ed1ab_0 + - cliff=4.4.0=pyhd8ed1ab_0 + - cmd2=2.4.3=py312h7900ff3_1 - cryptography=41.0.7=py312h4742d6a_1 + - debtcollector=2.5.0=pyhd8ed1ab_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - dogpile.cache=1.2.2=pyhd8ed1ab_1 - gettext=0.21.1=h27087fc_0 + - idna=3.6=pyhd8ed1ab_0 + - importlib-metadata=7.0.0=pyha770c72_0 + - importlib_metadata=7.0.0=hd8ed1ab_0 + - iso8601=2.1.0=pyhd8ed1ab_0 - jinja2=3.1.2=pyhd8ed1ab_1 + - jmespath=1.0.1=pyhd8ed1ab_0 + - jsonpatch=1.33=pyhd8ed1ab_0 + - jsonpointer=2.4=py312h7900ff3_3 + - keystoneauth1=5.4.0=pyhd8ed1ab_0 - krb5=1.20.1=h143b758_1 - ld_impl_linux-64=2.40=h41732ed_0 - libcurl=7.88.1=hdc1c0ab_1 @@ -36,25 +55,59 @@ dependencies: - libzlib=1.2.13=hd590300_5 - lz4-c=1.9.3=h9c3ff4c_1 - markupsafe=2.1.3=py312h98912ed_1 + - msgpack-python=1.0.3=py312hdb19cb5_0 - ncurses=6.4=h59595ed_2 + - netaddr=0.7.19=py_0 + - netifaces=0.11.0=py312h98912ed_2 - openssl=3.2.0=hd590300_1 + - openstacksdk=2.0.0=pyhd8ed1ab_0 + - os-service-types=1.7.0=pyh9f0ad1d_0 + - osc-lib=2.9.0=pyhd8ed1ab_0 + - oslo.config=9.2.0=pyhd8ed1ab_0 + - oslo.i18n=6.2.0=pyhd8ed1ab_0 + - oslo.serialization=5.2.0=pyhd8ed1ab_0 + - oslo.utils=6.3.0=pyhd8ed1ab_0 - packaging=23.2=pyhd8ed1ab_0 - packer=1.10.0=ha8f183a_0 + - pbr=6.0.0=pyhd8ed1ab_0 - pcre=8.45=h9c3ff4c_0 - pip=23.3.1=pyhd8ed1ab_0 - pixman=0.40.0=h36c2ea0_0 + - platformdirs=4.1.0=pyhd8ed1ab_0 + - prettytable=3.9.0=pyhd8ed1ab_0 - pycparser=2.21=pyhd8ed1ab_0 + - pyopenssl=23.3.0=pyhd8ed1ab_0 + - pyparsing=3.1.1=pyhd8ed1ab_0 + - pyperclip=1.8.2=pyhd8ed1ab_2 + - pysocks=1.7.1=pyha2e5f31_6 - python=3.12.0=hab00c5b_0_cpython + - python-cinderclient=9.4.0=pyhd8ed1ab_0 + - python-keystoneclient=5.2.0=pyhd8ed1ab_0 + - python-novaclient=18.3.0=pyhd8ed1ab_0 + - python-openstackclient=6.3.0=pyhd8ed1ab_0 + - python-tzdata=2023.3=pyhd8ed1ab_0 - python_abi=3.12=4_cp312 + - pytz=2023.3.post1=pyhd8ed1ab_0 - pyyaml=6.0.1=py312h98912ed_1 - qemu=5.0.0=hc4641f2_0 - readline=8.2=h8228510_1 + - requests=2.31.0=pyhd8ed1ab_0 + - requestsexceptions=1.4.0=py_0 - resolvelib=1.0.1=pyhd8ed1ab_0 + - rfc3986=2.0.0=pyhd8ed1ab_0 - setuptools=68.2.2=pyhd8ed1ab_0 + - simplejson=3.19.2=py312h98912ed_0 + - six=1.16.0=pyh6c4a22f_0 + - stevedore=5.1.0=pyhd8ed1ab_0 - tk=8.6.13=noxft_h4845f30_101 + - typing_extensions=4.9.0=pyha770c72_0 - tzdata=2023c=h71feb2d_0 + - urllib3=1.26.15=pyhd8ed1ab_0 + - wcwidth=0.2.12=pyhd8ed1ab_0 - wheel=0.42.0=pyhd8ed1ab_0 + - wrapt=1.16.0=py312h98912ed_0 - xz=5.2.6=h166bdaf_0 - yaml=0.2.5=h7f98852_2 + - zipp=3.17.0=pyhd8ed1ab_0 - zlib=1.2.13=hd590300_5 - zstd=1.5.2=h8a70e8d_1 From 39c1d1baae26221cd7564d606f2840e7bef8ac96 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 15:57:15 +0100 Subject: [PATCH 13/52] allow build.py to set ansible extra vars --- templates/build.pkr.hcl | 5 +---- templates/variables.pkr.hcl | 11 +++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/templates/build.pkr.hcl b/templates/build.pkr.hcl index db96222..e06dcf5 100644 --- a/templates/build.pkr.hcl +++ b/templates/build.pkr.hcl @@ -97,10 +97,7 @@ build { "ANSIBLE_HOST_KEY_CHECKING=False", "ANSIBLE_SCP_EXTRA_ARGS = '-0'", ] - extra_arguments = [ - "--vault-password-file=.vault_password", - # "--ssh-extra-args=-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=ssh-rsa", - ] + extra_arguments = "${ local.internal ? [var.vault_file, var.ansible_extra_args] : [var.ansible_extra_args]}" groups = var.groups } diff --git a/templates/variables.pkr.hcl b/templates/variables.pkr.hcl index ae20ce5..e9c70e6 100644 --- a/templates/variables.pkr.hcl +++ b/templates/variables.pkr.hcl @@ -26,3 +26,14 @@ variable "headless" { type = string default = "true" } +locals { + internal = contains(var.groups, "internal") +} +variable "vault_file" { + type = string + default = "--vault-password-file=.vault_password" +} +variable "ansible_extra_args" { + type = string + default = "" +} \ No newline at end of file From 78843e2aca7050195b1bcccd97aae7c0fa180943 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 15:59:10 +0100 Subject: [PATCH 14/52] no fail if vault password is missing --- ansible/internal.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ansible/internal.yml b/ansible/internal.yml index 8b103be..c3480be 100644 --- a/ansible/internal.yml +++ b/ansible/internal.yml @@ -5,17 +5,19 @@ vars_files: - "group_vars/all.yml" - "group_vars/condor.yml" - - "secret_group_vars/internal.yml" pre_tasks: + - name: Include secret vars + ansible.builtin.include_vars: "secret_group_vars/internal.yml" + when: inventory_hostname in group.internal - name: Copy server key into VM temporarily - copy: + ansible.builtin.copy: src: server_ca dest: /tmp/server_ca owner: root group: root - mode: 0600 + mode: "0600" - name: Add HostCertificate options - lineinfile: + ansible.builtin.lineinfile: path: /etc/ssh/sshd_config regexp: "^HostKey /etc/ssh/ssh_host_{{ item }}_key" line: "HostKey /etc/ssh/ssh_host_{{ item }}_key\nHostCertificate /etc/ssh/ssh_host_{{ item }}_key-cert.pub" @@ -29,6 +31,7 @@ limit_type: hard limit_item: core value: 0 + roles: - usegalaxy_eu.htcondor - lock-root From ad8085e64daba4a8f53b7fe785649800bc07f0d4 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 16:00:46 +0100 Subject: [PATCH 15/52] nl --- templates/variables.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/variables.pkr.hcl b/templates/variables.pkr.hcl index e9c70e6..973141f 100644 --- a/templates/variables.pkr.hcl +++ b/templates/variables.pkr.hcl @@ -36,4 +36,4 @@ variable "vault_file" { variable "ansible_extra_args" { type = string default = "" -} \ No newline at end of file +} From dd6b84f16789a9be2ed29aca06abceaa28578bd2 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Fri, 15 Dec 2023 17:14:43 +0100 Subject: [PATCH 16/52] better binary path management --- build.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/build.py b/build.py index 963cfd0..0f974b5 100755 --- a/build.py +++ b/build.py @@ -158,8 +158,14 @@ def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, pack self.conda_env = conda_env self.provisioning = provisioning self.ansible_args = ansible_args - self.PACKER_PATH = str(conda_env) + \ - "/bin/packer" if conda_env != None else str(packer_path) + self.packer_path = str(conda_env) + \ + "/bin/packer" if conda_env != None else str( + packer_path) | shutil.which("qemu-img") + self.qemu_path = str(conda_env) + \ + "/bin/qemu-img" if conda_env != None else shutil.which("qemu-img") + self.openstack_path = str(conda_env) + \ + "/bin/openstack" if conda_env != None else shutil.which( + "openstack") self.image_name = self.assemble_name() self.image_path = pathlib.Path( DIR_PATH + "/" + self.image_name + '.raw') @@ -176,19 +182,19 @@ def dry_run(self): print(self.assemble_ssh_command()) def assemble_packer_init(self): - cmd = str(self.PACKER_PATH) + cmd = str(self.packer_path) cmd += " init " cmd += DIR_PATH + "/templates" return cmd def assemble_packer_build_command(self): - cmd = [str(self.PACKER_PATH), "build"] + cmd = [str(self.packer_path), "build"] cmd.append("-only=qemu." + self.template) cmd.append(DIR_PATH + "/templates") return " ".join(cmd) def assemble_convert_command(self): - cmd = [str(shutil.which("qemu-img"))] + cmd = [str(self.qemu_path)] cmd.append("convert") cmd.append("-O") cmd.append("raw") @@ -197,7 +203,7 @@ def assemble_convert_command(self): return " ".join(cmd) def assemble_os_command(self): - return ["openstack", "image", "create", "--file", + return [str(self.openstack_path), "image", "create", "--file", str(self.image_path), self.image_name] def assemble_packer_envs(self): @@ -233,7 +239,7 @@ def assemble_name(self): return "~".join(name) def assemble_scp_command(self): - return ["scp", self.image_path, + return ["scp", str(self.image_path), "sn06.usegalaxy.eu:/data/dnb01/vgcn/" + os.path.basename(self.image_path)] def assemble_ssh_command(self): @@ -265,7 +271,7 @@ def clean_image_dir(self): shutil.rmtree(DIR_PATH + "/images") def upload_to_OS(self): - run_subprocess_with_spinner("OPENSTACK IMAGE CREATE", subprocess.Popen(self.assemble_os_command(), + run_subprocess_with_spinner("OPENSTACK IMAGE CREATE", subprocess.Popen(self.assemble_os_command(), env=os.environ.copy(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) def pvt_key(self): From b4379457a880334b2b35eb01b2c4b1b1a44a2fec Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 19 Dec 2023 11:42:28 +0100 Subject: [PATCH 17/52] sort imports --- build.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index 0f974b5..12a17c8 100755 --- a/build.py +++ b/build.py @@ -15,16 +15,15 @@ import argparse +import datetime import os import pathlib -import subprocess -import time -import sys import shutil -import datetime import signal +import subprocess +import sys import threading - +import time DIR_PATH = os.path.dirname(os.path.realpath(__file__)) From 4fc5aa6e51426c1eaab3649e9fdacada508c5323 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 19 Dec 2023 11:47:16 +0100 Subject: [PATCH 18/52] use black, use fstrings, remove unnecessary function --- build.py | 354 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 236 insertions(+), 118 deletions(-) diff --git a/build.py b/build.py index 12a17c8..33e3263 100755 --- a/build.py +++ b/build.py @@ -27,32 +27,64 @@ DIR_PATH = os.path.dirname(os.path.realpath(__file__)) - -my_parser = argparse.ArgumentParser(prog='build', - description='Build a VGCN image with Packer and the Ansible provisioner') - -my_parser.add_argument('image', choices=["-".join(x.split("-", 3)[:3]) for x in os.listdir( - 'templates') if x.endswith('-anaconda-ks.cfg')], help='image help') -my_parser.add_argument('provisioning', choices=[x.split(".", 1)[0] for x in os.listdir( - 'ansible') if x.endswith('.yml')], help=''' +STATIC_DIR = "/data/dnb01/vgcn/" + +SSH_HOST = "sn06.galaxyproject.eu" + +SSH_USER = "root" + + +my_parser = argparse.ArgumentParser( + prog="build", + description="Build a VGCN image with Packer and the Ansible provisioner", +) + +my_parser.add_argument( + "image", + choices=[ + "-".join(x.split("-", 3)[:3]) + for x in os.listdir("templates") + if x.endswith("-anaconda-ks.cfg") + ], + help="image help", +) +my_parser.add_argument( + "provisioning", + choices=[x.split(".", 1)[0] for x in os.listdir("ansible") if x.endswith(".yml")], + help=""" The playbooks you want to provision. The playbook files are located in the ansible folder and are automatically detected by this script, the options are the filenames, without .yml suffix - ''', nargs='+') -my_parser.add_argument('--ansible-args', type=str, - help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora') -my_parser.add_argument('--openstack', action='store_true', - help='Create an image in your OpenStack tenant and upload it. Make sure to source your credentials first') -my_parser.add_argument('--publish', type=pathlib.Path, metavar='PVT_KEY', - help='specify the path to your ssh key for sn06') -my_parser.add_argument('--dry-run', action='store_true', - help='just print the commands without executing anything') -my_parser.add_argument('--conda-env', type=pathlib.Path, - help='specifies the path to the conda environment to use') -my_parser.add_argument('--packer-path', type=pathlib.Path, - help='specifies the path to the packer binary') -my_parser.add_argument('--comment', type=str, - help='add a comment to the image name') + """, + nargs="+", +) +my_parser.add_argument( + "--ansible-args", + type=str, + help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora', +) +my_parser.add_argument( + "--openstack", + action="store_true", + help="Create an image in your OpenStack tenant and upload it. Make sure to source your credentials first", +) +my_parser.add_argument( + "--publish", + type=pathlib.Path, + metavar="PVT_KEY", + help="specify the path to your ssh key for sn06", +) +my_parser.add_argument( + "--dry-run", + action="store_true", + help="just print the commands without executing anything", +) +my_parser.add_argument( + "--conda-env", + type=pathlib.Path, + help="specifies the path to the conda environment to use", +) +my_parser.add_argument("--comment", type=str, help="add a comment to the image name") args = my_parser.parse_args() @@ -65,13 +97,14 @@ class Spinner: Indicates the user that the screen did not freeze or similar.\n Especially useful during the image upload, which can take several minutes.\n """ + busy = False delay = 0.1 @staticmethod def spinning_cursor(): while 1: - for cursor in '|/-\\': + for cursor in "|/-\\": yield cursor def __init__(self, delay=None): @@ -84,7 +117,7 @@ def spinner_task(self): sys.stdout.write(next(self.spinner_generator)) sys.stdout.flush() time.sleep(self.delay) - sys.stdout.write('\b') + sys.stdout.write("\b") sys.stdout.flush() def __enter__(self): @@ -109,109 +142,124 @@ def run_subprocess_with_spinner(name: str, proc: subprocess.Popen): # Register handler to pass keyboard interrupt to the subprocess def handler(sig, frame): - print(f"===================== { - name} ABORTED BY USER =========================") + print(f"================= {name} ABORTED BY USER ====================") if p: p.send_signal(signal.SIGINT) else: raise KeyboardInterrupt.add_note() + signal.signal(signal.SIGINT, handler) with Spinner(): print(f"{name.rstrip('Ee')}ing...") with proc as p: - for line in iter(p.stdout.readline, b''): + for line in iter(p.stdout.readline, b""): # Print the line to the console sys.stdout.buffer.write(line) - for line in iter(p.stderr.readline, b''): + for line in iter(p.stderr.readline, b""): # Print the error output to the console sys.stderr.buffer.write(line) returncode = p.wait() if returncode: - print(f"===================== { - name} FAILED =========================") + print( + f"===================== {name} FAILED =========================" + ) sys.exit(returncode) else: - print(f"===================== { - name} SUCCESSFUL =========================") + print( + f"===================== {name} SUCCESSFUL =====================" + ) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) -def get_active_branch_name(): - head_dir = pathlib.Path(DIR_PATH + "/.git/HEAD") - with head_dir.open("r") as f: - content = f.read().splitlines() - - for line in content: - if line[0:4] == "ref:": - return line.partition("refs/heads/")[2] - - class Build: - def __init__(self, openstack: bool, template: str, conda_env: pathlib.Path, packer_path: pathlib.Path, provisioning: [str], comment: str, pvt_key: pathlib.Path, ansible_args: str): + def __init__( + self, + openstack: bool, + template: str, + conda_env: pathlib.Path, + provisioning: [str], + comment: str, + pvt_key: pathlib.Path, + ansible_args: str, + ): self.openstack = openstack self.template = template - self.os = "-".join(template.split("-", 2)[:2]) self.comment = comment self.pvt_key = pvt_key - self.conda_env = conda_env self.provisioning = provisioning self.ansible_args = ansible_args - self.packer_path = str(conda_env) + \ - "/bin/packer" if conda_env != None else str( - packer_path) | shutil.which("qemu-img") - self.qemu_path = str(conda_env) + \ - "/bin/qemu-img" if conda_env != None else shutil.which("qemu-img") - self.openstack_path = str(conda_env) + \ - "/bin/openstack" if conda_env != None else shutil.which( - "openstack") self.image_name = self.assemble_name() - self.image_path = pathlib.Path( - DIR_PATH + "/" + self.image_name + '.raw') + self.image_path = pathlib.Path(DIR_PATH + "/" + self.image_name + ".raw") + if conda_env: + self.qemu_path = f"{conda_env}/bin/qemu-img" + self.openstack_path = f"{conda_env}/bin/openstack" + self.packer_path = f"{conda_env}/bin/packer" + else: + self.packer_path = shutil.which("packer") + self.openstack_path = shutil.which("openstack") + self.qemu_path = shutil.which("qemu-img") def dry_run(self): print(self.assemble_packer_envs()) print(self.assemble_packer_build_command()) print(self.image_name) print(self.assemble_convert_command()) - if self.openstack != None: + if self.openstack: print(self.assemble_os_command()) - if self.pvt_key != None: + if self.pvt_key: print(self.assemble_scp_command()) print(self.assemble_ssh_command()) def assemble_packer_init(self): - cmd = str(self.packer_path) - cmd += " init " - cmd += DIR_PATH + "/templates" - return cmd + return [ + f"self.packer_path", + f"init", + f"{DIR_PATH}/templates", + ] def assemble_packer_build_command(self): - cmd = [str(self.packer_path), "build"] - cmd.append("-only=qemu." + self.template) - cmd.append(DIR_PATH + "/templates") - return " ".join(cmd) + return " ".join( + [ + f"{self.packer_path}", + f"build", + f"-only=qemu.{self.template}", + f"{DIR_PATH}/templates", + ] + ) def assemble_convert_command(self): - cmd = [str(self.qemu_path)] - cmd.append("convert") - cmd.append("-O") - cmd.append("raw") - cmd.append("./images/" + self.template) - cmd.append(str(self.image_path)) - return " ".join(cmd) + return " ".join( + [ + f"{self.qemu_path}", + f"convert", + f"-O", + f"raw", + f"./images/{self.template}", + f"{self.image_path}", + ] + ) def assemble_os_command(self): - return [str(self.openstack_path), "image", "create", "--file", - str(self.image_path), self.image_name] + return " ".join( + [ + f"{self.openstack_path}", + f"image", + f"create", + f"--file", + f"{self.image_path}", + f"{self.image_name}", + ] + ) def assemble_packer_envs(self): env = os.environ.copy() - env["PACKER_PLUGIN_PATH"] = DIR_PATH + "/packer_plugins" - env["PKR_VAR_groups"] = "[" + \ - ','.join(["\"" + x + "\"" for x in self.provisioning]) + "]" - env["PKR_VAR_headless"] = 'true' - if self.ansible_args != None: + env["PACKER_PLUGIN_PATH"] = f"{DIR_PATH}/packer_plugins" + env[ + "PKR_VAR_groups" + ] = f"[{','.join('\"' + x + '\"' for x in self.provisioning)}]" + env["PKR_VAR_headless"] = "true" + if self.ansible_args: env["PKR_VAR_ansible_extra_args"] = self.ansible_args return env @@ -222,34 +270,48 @@ def assemble_name(self): """ name = ["vgcn"] if "generic" in self.provisioning: - prv = self.provisioning - prv.remove("generic") - name += ["+" + x for x in prv] + provisioning = self.provisioning + provisioning.remove("generic") + name += [f"+{'+'.join(provisioning)}"] else: - name += ["!generic+" + "+".join(self.provisioning)] - name += [self.os] - name += [self.assemble_timestamp()] - name += [subprocess.check_output(['git', 'rev-parse', - '--abbrev-ref', 'HEAD']).decode('ascii').strip()] - name += [subprocess.check_output(['git', 'rev-parse', - '--short', 'HEAD']).decode('ascii').strip()] - if self.comment != None: + name += [f"!generic+{'+'.join(self.provisioning)}"] + name += [ + "-".join(self.template.split("-", 2)[:2]), + self.assemble_timestamp(), + subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + .decode("ascii") + .strip(), + subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]) + .decode("ascii") + .strip(), + ] + if self.comment: name += [self.comment] return "~".join(name) def assemble_scp_command(self): - return ["scp", str(self.image_path), - "sn06.usegalaxy.eu:/data/dnb01/vgcn/" + os.path.basename(self.image_path)] + return " ".join( + [ + f"scp", + f"-i", + f"{self.pvt_key}", + f"{self.image_path}", + f"{SSH_USER}@{SSH_HOST}:{STATIC_DIR}/{os.path.basename(self.image_path)}", + ] + ) def assemble_ssh_command(self): - cmd = ["ssh"] - cmd.append("-i") - cmd.append(str(self.pvt_key)) - cmd.append("sn06.galaxyproject.eu") - cmd.append("chmod") - cmd.append("ugo+r") - cmd.append("/data/dnb01/vgcn/" + os.path.basename(self.image_path)) - return cmd + return " ".join( + [ + f"ssh", + f"-i", + f"{self.pvt_key}", + f"{SSH_USER}@{SSH_HOST}", + f"chmod", + f"ugo+r", + f"{STATIC_DIR}{os.path.basename(self.image_path)}", + ] + ) def assemble_timestamp(self): today = datetime.date.today() @@ -258,32 +320,88 @@ def assemble_timestamp(self): def build(self): self.clean_image_dir() - run_subprocess_with_spinner("BUILD", subprocess.Popen(self.assemble_packer_build_command(), env=self.assemble_packer_envs(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) + run_subprocess_with_spinner( + "BUILD", + subprocess.Popen( + self.assemble_packer_build_command(), + env=self.assemble_packer_envs(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + shell=True, + ), + ) def convert(self): - run_subprocess_with_spinner(name="CONVERT", proc=subprocess.Popen(self.assemble_convert_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) + run_subprocess_with_spinner( + name="CONVERT", + proc=subprocess.Popen( + self.assemble_convert_command(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + shell=True, + ), + ) def clean_image_dir(self): if os.path.exists(DIR_PATH + "/images"): shutil.rmtree(DIR_PATH + "/images") def upload_to_OS(self): - run_subprocess_with_spinner("OPENSTACK IMAGE CREATE", subprocess.Popen(self.assemble_os_command(), env=os.environ.copy(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) - - def pvt_key(self): - run_subprocess_with_spinner("PUBLISH", subprocess.Popen(self.assemble_scp_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) - run_subprocess_with_spinner("PERMISSION CHANGE", subprocess.Popen(self.assemble_ssh_command(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, shell=True)) + env = os.environ.copy() + if not ( + "OS_AUTH_URL" in env + and "OS_APPLICATION_CREDENTIAL_ID" in env + and "OS_APPLICATION_CREDENTIAL_SECRET" in env + ): + print("OS credentials missing in environment vars") + sys.exit(1) + run_subprocess_with_spinner( + "OPENSTACK IMAGE CREATE", + subprocess.Popen( + self.assemble_os_command(), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + shell=True, + ), + ) + + def publish(self): + run_subprocess_with_spinner( + "PUBLISH", + subprocess.Popen( + self.assemble_scp_command(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + shell=True, + ), + ) + run_subprocess_with_spinner( + "PERMISSION CHANGE", + subprocess.Popen( + self.assemble_ssh_command(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + shell=True, + ), + ) def main(): - image = Build(openstack=args.openstack, template=args.image, conda_env=args.conda_env, - packer_path=args.packer_path, provisioning=args.provisioning, comment=args.comment, - ansible_args=args.ansible_args, pvt_key=args.publish) + image = Build( + openstack=args.openstack, + template=args.image, + conda_env=args.conda_env, + provisioning=args.provisioning, + comment=args.comment, + ansible_args=args.ansible_args, + pvt_key=args.publish, + ) if args.dry_run: image.dry_run() else: @@ -292,8 +410,8 @@ def main(): if args.openstack: image.upload_to_OS() if args.publish: - image.pvt_key() + image.publish() -if __name__ == '__main__': +if __name__ == "__main__": main() From d1d6b46b665e8f2db646bed6ca3fd786be64897a Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 19 Dec 2023 11:48:43 +0100 Subject: [PATCH 19/52] comment --- build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build.py b/build.py index 33e3263..56f2ee5 100755 --- a/build.py +++ b/build.py @@ -349,6 +349,7 @@ def clean_image_dir(self): shutil.rmtree(DIR_PATH + "/images") def upload_to_OS(self): + # Checking this, because OS is failing silently env = os.environ.copy() if not ( "OS_AUTH_URL" in env From d953c6677ebfa3c8f13923b710ff94e7db9ecadc Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:45:16 +0100 Subject: [PATCH 20/52] remove unnecessary linebreaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 56f2ee5..52e873f 100755 --- a/build.py +++ b/build.py @@ -133,9 +133,9 @@ def __exit__(self, exception, value, tb): def run_subprocess_with_spinner(name: str, proc: subprocess.Popen): """ - Opens a subprocess and redirect stdout and stderr to Python.\n - Shows a spinning Cursor while the command runs.\n - Exits with returncode of subprocess if not equals 0.\n + Opens a subprocess and redirect stdout and stderr to Python. + Shows a spinning Cursor while the command runs. + Exits with returncode of subprocess if not equals 0. """ try: p = None From b898debbbbde77b6c07c4df7c35af8c451e43406 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:48:06 +0100 Subject: [PATCH 21/52] Update .gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- .gitignore | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitignore b/.gitignore index 54f42dc..55b0ed6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,15 +3,6 @@ virtualbox-iso/ vmware-iso/ packer_plugins/ ansible/collections -ansible/roles/galaxyproject* -ansible/roles/geerlingguy* -ansible/roles/influxdata* -ansible/roles/usegalaxy-eu.a* -ansible/roles/usegalaxy_eu.* -ansible/roles/usegalaxy-eu.c* -ansible/roles/usegalaxy-eu.d* -ansible/roles/usegalaxy-eu.f* -ansible/roles/usegalaxy-eu.t* # Created by https://www.gitignore.io/api/packer ### Packer ### From 2d3a60c06ba00cdbcfc6d59eafd988cc192bc626 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 19 Dec 2023 18:03:42 +0100 Subject: [PATCH 22/52] make parser as function --- requirements.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.yml b/requirements.yml index 081adc0..6ce96d8 100644 --- a/requirements.yml +++ b/requirements.yml @@ -19,7 +19,7 @@ roles: version: 6.1.0 - src: https://github.com/usegalaxy-eu/ansible-chrony name: influxdata.chrony - version: 0.1.0 + version: 0.1.1 - src: galaxyproject.pulsar version: 1.0.10 - src: https://github.com/usegalaxy-eu/ansible-dynmotd @@ -46,4 +46,6 @@ roles: - name: usegalaxy-eu.telegraf src: https://github.com/usegalaxy-eu/ansible-telegraf version: 0.14.1 - + - name: artis3n.tailscale + src: https://github.com/artis3n/ansible-role-tailscale + version: v4.2.3 From d6107de6f6012f4b97a30bcf14ca55d357600511 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:07:14 +0100 Subject: [PATCH 23/52] add license to spinner class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 52e873f..c0ec77d 100755 --- a/build.py +++ b/build.py @@ -90,7 +90,7 @@ args = my_parser.parse_args() -# Spinner thanks to stackoverflow user victor-moyseenko +# Spinner class from https://stackoverflow.com/a/39504463 by Victor Moyseenko, subject to CC BY-SA 4.0 license. class Spinner: """ Creates a spinning cursor while the command runs.\n From 3c5ce6913cb576445be662e50bebd242033d6ad2 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:08:07 +0100 Subject: [PATCH 24/52] remove unnecessary linebreaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index c0ec77d..806c2c6 100755 --- a/build.py +++ b/build.py @@ -93,9 +93,9 @@ # Spinner class from https://stackoverflow.com/a/39504463 by Victor Moyseenko, subject to CC BY-SA 4.0 license. class Spinner: """ - Creates a spinning cursor while the command runs.\n - Indicates the user that the screen did not freeze or similar.\n - Especially useful during the image upload, which can take several minutes.\n + Creates a spinning cursor while the command runs. + Indicates the user that the screen did not freeze or similar. + Especially useful during the image upload, which can take several minutes. """ busy = False From 222716206f75f13316e8ebff72d930c9797ad47a Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:10:00 +0100 Subject: [PATCH 25/52] remove unnecessary linebreaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 806c2c6..ca25428 100755 --- a/build.py +++ b/build.py @@ -265,7 +265,7 @@ def assemble_packer_envs(self): def assemble_name(self): """ - Uses a naming scheme described in\n + Uses a naming scheme described in https://github.com/usegalaxy-eu/vgcn/issues/78 """ name = ["vgcn"] From 6b97b23b1a063d068bfbe9124f0fba4d62cdfc73 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:11:17 +0100 Subject: [PATCH 26/52] Update doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index ca25428..5b2e6e7 100755 --- a/build.py +++ b/build.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# A commandline script using argparse that builds a vgcn image with packer +# A command line script that builds a VGCN image with Packer. # CAUTION: If a directory called 'images' exists in this directory, it will be deleted! # Required arguments are the template, which are named like the anaconda-ks.cfg files without the # anaconda-ks.cfg suffix and the provisioning, separated by space and named like the ansible-playbooks From 16164550deebcc1279a4719db8a327858f177930 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:24:46 +0100 Subject: [PATCH 27/52] no append MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 5b2e6e7..fbfbf9d 100755 --- a/build.py +++ b/build.py @@ -275,7 +275,8 @@ def assemble_name(self): name += [f"+{'+'.join(provisioning)}"] else: name += [f"!generic+{'+'.join(self.provisioning)}"] - name += [ + name = [ + "vgcn", "-".join(self.template.split("-", 2)[:2]), self.assemble_timestamp(), subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) From 07cd13ac215b498a83f2f2e7c9f1d04bf584fa80 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:36:11 +0100 Subject: [PATCH 28/52] remove choices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build.py b/build.py index fbfbf9d..242db75 100755 --- a/build.py +++ b/build.py @@ -41,11 +41,6 @@ my_parser.add_argument( "image", - choices=[ - "-".join(x.split("-", 3)[:3]) - for x in os.listdir("templates") - if x.endswith("-anaconda-ks.cfg") - ], help="image help", ) my_parser.add_argument( From c071715b47e052c30f592959a6a891786ecfe151 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Tue, 19 Dec 2023 18:58:02 +0100 Subject: [PATCH 29/52] wrap argparse in function --- build.py | 103 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/build.py b/build.py index 242db75..0bcc1e3 100755 --- a/build.py +++ b/build.py @@ -34,55 +34,58 @@ SSH_USER = "root" -my_parser = argparse.ArgumentParser( - prog="build", - description="Build a VGCN image with Packer and the Ansible provisioner", -) - -my_parser.add_argument( - "image", - help="image help", -) -my_parser.add_argument( - "provisioning", - choices=[x.split(".", 1)[0] for x in os.listdir("ansible") if x.endswith(".yml")], - help=""" - The playbooks you want to provision. - The playbook files are located in the ansible folder - and are automatically detected by this script, the options are the filenames, without .yml suffix - """, - nargs="+", -) -my_parser.add_argument( - "--ansible-args", - type=str, - help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora', -) -my_parser.add_argument( - "--openstack", - action="store_true", - help="Create an image in your OpenStack tenant and upload it. Make sure to source your credentials first", -) -my_parser.add_argument( - "--publish", - type=pathlib.Path, - metavar="PVT_KEY", - help="specify the path to your ssh key for sn06", -) -my_parser.add_argument( - "--dry-run", - action="store_true", - help="just print the commands without executing anything", -) -my_parser.add_argument( - "--conda-env", - type=pathlib.Path, - help="specifies the path to the conda environment to use", -) -my_parser.add_argument("--comment", type=str, help="add a comment to the image name") - - -args = my_parser.parse_args() +def make_parser() -> argparse.ArgumentParser: + my_parser = argparse.ArgumentParser( + prog="build", + description="Build a VGCN image with Packer and the Ansible provisioner", + ) + + my_parser.add_argument( + "image", + help="image help", + ) + my_parser.add_argument( + "provisioning", + choices=[ + x.split(".", 1)[0] for x in os.listdir("ansible") if x.endswith(".yml") + ], + help=""" + The playbooks you want to provision. + The playbook files are located in the ansible folder + and are automatically detected by this script, the options are the filenames, without .yml suffix + """, + nargs="+", + ) + my_parser.add_argument( + "--ansible-args", + type=str, + help='e.g. --ansible-args="--scp-extra-args=-O" which activates SCP compatibility mode and might be needed on Fedora', + ) + my_parser.add_argument( + "--openstack", + action="store_true", + help="Create an image in your OpenStack tenant and upload it. Make sure to source your credentials first", + ) + my_parser.add_argument( + "--publish", + type=pathlib.Path, + metavar="PVT_KEY", + help="specify the path to your ssh key for sn06", + ) + my_parser.add_argument( + "--dry-run", + action="store_true", + help="just print the commands without executing anything", + ) + my_parser.add_argument( + "--conda-env", + type=pathlib.Path, + help="specifies the path to the conda environment to use", + ) + my_parser.add_argument( + "--comment", type=str, help="add a comment to the image name" + ) + return my_parser # Spinner class from https://stackoverflow.com/a/39504463 by Victor Moyseenko, subject to CC BY-SA 4.0 license. @@ -390,6 +393,8 @@ def publish(self): def main(): + my_parser = make_parser() + args = my_parser.parse_args() image = Build( openstack=args.openstack, template=args.image, From 1cedd6d5ccbe20a85980b724b22a821d27bb21ff Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:02:54 +0100 Subject: [PATCH 30/52] pathlib.path constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 0bcc1e3..4b304b3 100755 --- a/build.py +++ b/build.py @@ -25,9 +25,9 @@ import threading import time -DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +DIR_PATH = pathlib.Path(__file__).parent.absolute() -STATIC_DIR = "/data/dnb01/vgcn/" +STATIC_DIR = pathlib.Path("/data/dnb01/vgcn/").absolute() SSH_HOST = "sn06.galaxyproject.eu" From 39be50e15cb057c8cbc7a8da86ead4c3119f5513 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:03:23 +0100 Subject: [PATCH 31/52] remove anaconda from docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 4b304b3..77753b7 100755 --- a/build.py +++ b/build.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # A command line script that builds a VGCN image with Packer. # CAUTION: If a directory called 'images' exists in this directory, it will be deleted! -# Required arguments are the template, which are named like the anaconda-ks.cfg files without the -# anaconda-ks.cfg suffix and the provisioning, separated by space and named like the ansible-playbooks -# in the ansible directory without the .yml suffix. +# Required arguments are a Packer build (builds are defined in templates/build.pkr.hcl) +# and the provisioning Ansible playbooks (located in the ansible directory, passed +# without the .yml suffix). # Optionally you can specify the path to the Packer binary you want to use (--packer-path) # the path to the conda env you want to use (--conda-env) # the path to your ssh private key for copying the images to sn06 and From 45468b47dad89f7f606e5fcf4f45c64179d47b6d Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:04:01 +0100 Subject: [PATCH 32/52] pathlib objects look better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 77753b7..0bbf665 100755 --- a/build.py +++ b/build.py @@ -188,7 +188,7 @@ def __init__( self.provisioning = provisioning self.ansible_args = ansible_args self.image_name = self.assemble_name() - self.image_path = pathlib.Path(DIR_PATH + "/" + self.image_name + ".raw") + self.image_path = DIR_PATH / f"{self.image_name}.raw" if conda_env: self.qemu_path = f"{conda_env}/bin/qemu-img" self.openstack_path = f"{conda_env}/bin/openstack" From 0392ffc2a33661c9626d4c54250b5368264ff3af Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:04:34 +0100 Subject: [PATCH 33/52] another simple path join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 0bbf665..177f097 100755 --- a/build.py +++ b/build.py @@ -213,7 +213,7 @@ def assemble_packer_init(self): return [ f"self.packer_path", f"init", - f"{DIR_PATH}/templates", + f"{DIR_PATH / 'templates'}", ] def assemble_packer_build_command(self): From 505ef7c9bc92d603e611ad92dda4da139d660134 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:04:56 +0100 Subject: [PATCH 34/52] and one more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 177f097..a16a747 100755 --- a/build.py +++ b/build.py @@ -222,7 +222,7 @@ def assemble_packer_build_command(self): f"{self.packer_path}", f"build", f"-only=qemu.{self.template}", - f"{DIR_PATH}/templates", + f"{DIR_PATH / 'templates'}", ] ) From 36f041cc7621e76daeef14f5405971e3290e2208 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:05:21 +0100 Subject: [PATCH 35/52] that should be a variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index a16a747..259ec8f 100755 --- a/build.py +++ b/build.py @@ -211,7 +211,7 @@ def dry_run(self): def assemble_packer_init(self): return [ - f"self.packer_path", + f"{self.packer_path}", f"init", f"{DIR_PATH / 'templates'}", ] From 9c8c9038d27dad6ba7c1cc209dbcb2bc454e3afd Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:07:32 +0100 Subject: [PATCH 36/52] fix order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build.py b/build.py index 259ec8f..5432566 100755 --- a/build.py +++ b/build.py @@ -276,6 +276,7 @@ def assemble_name(self): name = [ "vgcn", "-".join(self.template.split("-", 2)[:2]), + f"+{'+'.join(provisioning)}", self.assemble_timestamp(), subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) .decode("ascii") From 60ee6d6f5cb14cdcff33b064c6e3b709d2c1c209 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:08:00 +0100 Subject: [PATCH 37/52] more paths to join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 5432566..a73f8a1 100755 --- a/build.py +++ b/build.py @@ -296,7 +296,7 @@ def assemble_scp_command(self): f"-i", f"{self.pvt_key}", f"{self.image_path}", - f"{SSH_USER}@{SSH_HOST}:{STATIC_DIR}/{os.path.basename(self.image_path)}", + f"{SSH_USER}@{SSH_HOST}:{STATIC_DIR / self.image_path.basename()}", ] ) From 3b6a4b0e3e1c76317a8f8129dc4baa63d8bfcdc0 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:12:30 +0100 Subject: [PATCH 38/52] use pathlib join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index a73f8a1..bb0df0b 100755 --- a/build.py +++ b/build.py @@ -309,7 +309,7 @@ def assemble_ssh_command(self): f"{SSH_USER}@{SSH_HOST}", f"chmod", f"ugo+r", - f"{STATIC_DIR}{os.path.basename(self.image_path)}", + f"{STATIC_DIR / self.image_path.basename()}", ] ) From 8b363fa2a1ba12c47823feb19449055ef196370d Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 19:59:11 +0100 Subject: [PATCH 39/52] copy list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index bb0df0b..9e9ef5d 100755 --- a/build.py +++ b/build.py @@ -267,12 +267,9 @@ def assemble_name(self): https://github.com/usegalaxy-eu/vgcn/issues/78 """ name = ["vgcn"] - if "generic" in self.provisioning: - provisioning = self.provisioning - provisioning.remove("generic") - name += [f"+{'+'.join(provisioning)}"] - else: - name += [f"!generic+{'+'.join(self.provisioning)}"] + provisioning = self.provisioning.copy() + if "generic" not in self.provisioning: + provisioning.insert(0, "!generic") name = [ "vgcn", "-".join(self.template.split("-", 2)[:2]), From 1f153c5cb017913375f4f84fc69b69b1f46b8162 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:00:20 +0100 Subject: [PATCH 40/52] add architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 9e9ef5d..de28c61 100755 --- a/build.py +++ b/build.py @@ -272,7 +272,7 @@ def assemble_name(self): provisioning.insert(0, "!generic") name = [ "vgcn", - "-".join(self.template.split("-", 2)[:2]), + self.template, f"+{'+'.join(provisioning)}", self.assemble_timestamp(), subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) From e1c6d934916eadb0221b042072219b5339405c98 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:01:05 +0100 Subject: [PATCH 41/52] reorder assemble name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build.py b/build.py index de28c61..e08dda0 100755 --- a/build.py +++ b/build.py @@ -266,7 +266,6 @@ def assemble_name(self): Uses a naming scheme described in https://github.com/usegalaxy-eu/vgcn/issues/78 """ - name = ["vgcn"] provisioning = self.provisioning.copy() if "generic" not in self.provisioning: provisioning.insert(0, "!generic") From ef2e646bb82657463eb924508bcabba9d7526cfa Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:01:35 +0100 Subject: [PATCH 42/52] use pathlibs exist method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Manuel Domínguez <43052541+kysrpex@users.noreply.github.com> --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index e08dda0..02cab41 100755 --- a/build.py +++ b/build.py @@ -341,8 +341,8 @@ def convert(self): ) def clean_image_dir(self): - if os.path.exists(DIR_PATH + "/images"): - shutil.rmtree(DIR_PATH + "/images") + if (DIR_PATH / "images").exists(): + shutil.rmtree(DIR_PATH / "images") def upload_to_OS(self): # Checking this, because OS is failing silently From 56b9e2f5d00cc4513a4a59141a14eedd3e40952c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= <43052541+kysrpex@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:29:55 +0100 Subject: [PATCH 43/52] Compute commit time in `assemble_timestamp` Compute commit time in `assemble_timestamp` instead of build time as agreed in https://github.com/usegalaxy-eu/vgcn/issues/78. --- build.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 02cab41..bc3fd8e 100755 --- a/build.py +++ b/build.py @@ -310,9 +310,25 @@ def assemble_ssh_command(self): ) def assemble_timestamp(self): - today = datetime.date.today() - seconds_since_midnight = time.time() - time.mktime(today.timetuple()) - return today.strftime("%Y%m%d") + "~" + str(int(seconds_since_midnight)) + commit_time = subprocess.check_output( + ["git", "show", "--no-patch", "--format=%ct"] + ).decode("ascii").strip() + commit_time = int(commit_time) + commit_time = datetime.datetime.fromtimestamp(commit_time) + + commit_date = commit_time.date() + commit_date_midnight = datetime.datetime( + commit_date.year, commit_time.month, commit_time.day + ) + + seconds_since_midnight = int( + (commit_time - commit_date_midnight).total_seconds() + ) + + return ( + f"{commit_time.date().strftime('%Y%m%d')}" + f"~{seconds_since_midnight}" + ) def build(self): self.clean_image_dir() From 72db2e4ce4a1f33dc283a290b44d0bfb905710ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Wed, 20 Dec 2023 16:35:08 +0100 Subject: [PATCH 44/52] Workaround for including double quotes in `PKR_VAR_groups` env variable Prevents "SyntaxError: f-string expression part cannot include a backslash". ``` In [1]: f"[{','.join('\"' + x + '\"' for x in self.provisioning)}]" Cell In[1], line 1 f"[{','.join('\"' + x + '\"' for x in self.provisioning)}]" ^ SyntaxError: f-string expression part cannot include a backslash ``` --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index bc3fd8e..7e920c6 100755 --- a/build.py +++ b/build.py @@ -255,7 +255,7 @@ def assemble_packer_envs(self): env["PACKER_PLUGIN_PATH"] = f"{DIR_PATH}/packer_plugins" env[ "PKR_VAR_groups" - ] = f"[{','.join('\"' + x + '\"' for x in self.provisioning)}]" + ] = f"""[{','.join('"' + x + '"' for x in self.provisioning)}]""" env["PKR_VAR_headless"] = "true" if self.ansible_args: env["PKR_VAR_ansible_extra_args"] = self.ansible_args From b543b85af1b1bb2b6d19a9b35a2229f3ac1cb44d Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 21 Dec 2023 10:26:12 +0100 Subject: [PATCH 45/52] miners in cloudinit --- ansible/roles/cloudinit/tasks/main.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ansible/roles/cloudinit/tasks/main.yml b/ansible/roles/cloudinit/tasks/main.yml index 397b3c1..a1f7dc9 100644 --- a/ansible/roles/cloudinit/tasks/main.yml +++ b/ansible/roles/cloudinit/tasks/main.yml @@ -87,3 +87,23 @@ - systemctl restart telegraf - systemctl restart docker when: internal | bool + + - name: cloud-init config 6 (block mining pool IPs) + blockinfile: + marker: "# {mark} ANSIBLE MANAGED BLOCK 6" + dest: /etc/cloud/cloud.cfg + block: | + write_files: + - path: /run/scripts/test-script.sh + content: | + #! /bin/sh + + + curl -sL 'https://minerstat.com/mining-pool-whitelist.txt' \ + | awk '!/^\s*#/ {print $1}' \ + | sort -u \ + | xargs -n1 -P10 -i firewall-cmd --direct --add-rule ipv4 filter OUTPUT 0 -d "{}" -j DROP + + firewall-cmd --runtime-to-permanent + runcmd: + - [ sh, "/run/scripts/test-script.sh" ] From 6b7ff245bac36933671372042125e2c99014673a Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 21 Dec 2023 11:25:05 +0100 Subject: [PATCH 46/52] move to role --- ansible/roles/block-ips/defaults/main.yml | 2 ++ ansible/roles/block-ips/tasks/main.yml | 21 +++++++++++++++++++++ ansible/roles/cloudinit/tasks/main.yml | 20 -------------------- 3 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 ansible/roles/block-ips/defaults/main.yml create mode 100644 ansible/roles/block-ips/tasks/main.yml diff --git a/ansible/roles/block-ips/defaults/main.yml b/ansible/roles/block-ips/defaults/main.yml new file mode 100644 index 0000000..9a6bf67 --- /dev/null +++ b/ansible/roles/block-ips/defaults/main.yml @@ -0,0 +1,2 @@ +--- +block_ips_url: https://minerstat.com/mining-pool-whitelist.txt \ No newline at end of file diff --git a/ansible/roles/block-ips/tasks/main.yml b/ansible/roles/block-ips/tasks/main.yml new file mode 100644 index 0000000..1a3f175 --- /dev/null +++ b/ansible/roles/block-ips/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Get IPs + ansible.builtin.uri: + url: "{{ block_ips_url }}" + return_content: true + status_code: 200 + body_format: raw + register: ips_raw + +- name: Set IPs as facts + ansible.builtin.set_fact: + ips_to_block: "{{ ips_raw.content | regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}" + +- name: Set firewall rules + ansible.posix.firewalld: + zone: public + state: present + permanent: true + immediate: true + rich_rule: 'rule family="ipv4" destination address="{{ item }}" DROP' + with_items: "{{ ips_to_block }}" diff --git a/ansible/roles/cloudinit/tasks/main.yml b/ansible/roles/cloudinit/tasks/main.yml index a1f7dc9..397b3c1 100644 --- a/ansible/roles/cloudinit/tasks/main.yml +++ b/ansible/roles/cloudinit/tasks/main.yml @@ -87,23 +87,3 @@ - systemctl restart telegraf - systemctl restart docker when: internal | bool - - - name: cloud-init config 6 (block mining pool IPs) - blockinfile: - marker: "# {mark} ANSIBLE MANAGED BLOCK 6" - dest: /etc/cloud/cloud.cfg - block: | - write_files: - - path: /run/scripts/test-script.sh - content: | - #! /bin/sh - - - curl -sL 'https://minerstat.com/mining-pool-whitelist.txt' \ - | awk '!/^\s*#/ {print $1}' \ - | sort -u \ - | xargs -n1 -P10 -i firewall-cmd --direct --add-rule ipv4 filter OUTPUT 0 -d "{}" -j DROP - - firewall-cmd --runtime-to-permanent - runcmd: - - [ sh, "/run/scripts/test-script.sh" ] From 70d816ad6f277db46e84abbeeb5813d4c9011255 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 21 Dec 2023 11:25:30 +0100 Subject: [PATCH 47/52] add it to workers --- ansible/workers.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/workers.yml b/ansible/workers.yml index 848ef31..c2865c7 100644 --- a/ansible/workers.yml +++ b/ansible/workers.yml @@ -161,3 +161,5 @@ ulimit_fsize_unit: "condor.service" ulimit_fsize_soft: 268435456000 ulimit_fsize_hard: 1073741824000 + + - role: block-ips From f402692c54261885b857f7c44264e3cdba0e9a13 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 21 Dec 2023 14:52:44 +0100 Subject: [PATCH 48/52] add block-ips role --- ansible/roles/block-ips/defaults/main.yml | 3 +- ansible/roles/block-ips/tasks/main.yml | 37 ++++++++++--------- .../templates/block-outbound.service | 15 ++++++++ .../block-ips/templates/block-outbound.sh.j2 | 7 ++++ 4 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 ansible/roles/block-ips/templates/block-outbound.service create mode 100644 ansible/roles/block-ips/templates/block-outbound.sh.j2 diff --git a/ansible/roles/block-ips/defaults/main.yml b/ansible/roles/block-ips/defaults/main.yml index 9a6bf67..ec8a6b1 100644 --- a/ansible/roles/block-ips/defaults/main.yml +++ b/ansible/roles/block-ips/defaults/main.yml @@ -1,2 +1,3 @@ --- -block_ips_url: https://minerstat.com/mining-pool-whitelist.txt \ No newline at end of file +block_ips_urls: + - https://minerstat.com/mining-pool-whitelist.txt \ No newline at end of file diff --git a/ansible/roles/block-ips/tasks/main.yml b/ansible/roles/block-ips/tasks/main.yml index 1a3f175..55b9d71 100644 --- a/ansible/roles/block-ips/tasks/main.yml +++ b/ansible/roles/block-ips/tasks/main.yml @@ -1,21 +1,22 @@ --- -- name: Get IPs - ansible.builtin.uri: - url: "{{ block_ips_url }}" - return_content: true - status_code: 200 - body_format: raw - register: ips_raw +- name: Template bash script (block-outbound) + ansible.builtin.template: + src: block-outbound.sh.j2 + dest: /usr/local/bin/block-outbound.sh + owner: root + group: root + mode: '0755' -- name: Set IPs as facts - ansible.builtin.set_fact: - ips_to_block: "{{ ips_raw.content | regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}" +- name: Add block-outbound service + ansible.builtin.template: + src: block-outbound.service.j2 + dest: /etc/systemd/system/block-outbound.service + owner: root + group: root + mode: '0644' -- name: Set firewall rules - ansible.posix.firewalld: - zone: public - state: present - permanent: true - immediate: true - rich_rule: 'rule family="ipv4" destination address="{{ item }}" DROP' - with_items: "{{ ips_to_block }}" +- name: Enable block-outbound service + ansible.builtin.service: + name: block-outbound + state: reloaded + enabled: true diff --git a/ansible/roles/block-ips/templates/block-outbound.service b/ansible/roles/block-ips/templates/block-outbound.service new file mode 100644 index 0000000..95231a1 --- /dev/null +++ b/ansible/roles/block-ips/templates/block-outbound.service @@ -0,0 +1,15 @@ +[Unit] +Description=Script that fetches IP lists from URLs and blocks outbound packets to them +After=network.target + +[Service] +UMask=022 +Type=simple +User=root +Group=root +ExecStart=/usr/local/bin/block-outbound.sh +MemoryLimit=1G +Restart=no + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/roles/block-ips/templates/block-outbound.sh.j2 b/ansible/roles/block-ips/templates/block-outbound.sh.j2 new file mode 100644 index 0000000..b54b340 --- /dev/null +++ b/ansible/roles/block-ips/templates/block-outbound.sh.j2 @@ -0,0 +1,7 @@ +#! /bin/sh + +{% for url in block_ips_urls %} +curl -sL '{url}' \ +| awk '!/^\s*#/ {print $1}' \ +| xargs -n1 -P10 -i firewall-cmd --direct --add-rule ipv4 filter OUTPUT 0 -d "{}" -j DROP +{% endfor %} From 522cd03021ba0b4f41ac5fe55a8dd249cd257aca Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:43:54 +0100 Subject: [PATCH 49/52] special var is called group_names --- ansible/internal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/internal.yml b/ansible/internal.yml index c3480be..b806e3a 100644 --- a/ansible/internal.yml +++ b/ansible/internal.yml @@ -8,7 +8,7 @@ pre_tasks: - name: Include secret vars ansible.builtin.include_vars: "secret_group_vars/internal.yml" - when: inventory_hostname in group.internal + when: inventory_hostname in group_names - name: Copy server key into VM temporarily ansible.builtin.copy: src: server_ca From c54ea61dc3d668e67a8a97210d49bc48a64fb5f7 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:45:27 +0100 Subject: [PATCH 50/52] not hostname but internal --- ansible/internal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/internal.yml b/ansible/internal.yml index b806e3a..d0ad82e 100644 --- a/ansible/internal.yml +++ b/ansible/internal.yml @@ -8,7 +8,7 @@ pre_tasks: - name: Include secret vars ansible.builtin.include_vars: "secret_group_vars/internal.yml" - when: inventory_hostname in group_names + when: "internal" in group_names - name: Copy server key into VM temporarily ansible.builtin.copy: src: server_ca From 99750446d7f4fa49849b7a6df7c7585f05709cb5 Mon Sep 17 00:00:00 2001 From: Mira <86979912+mira-miracoli@users.noreply.github.com> Date: Thu, 21 Dec 2023 15:52:17 +0100 Subject: [PATCH 51/52] and the quotes --- ansible/internal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/internal.yml b/ansible/internal.yml index d0ad82e..5331a27 100644 --- a/ansible/internal.yml +++ b/ansible/internal.yml @@ -8,7 +8,7 @@ pre_tasks: - name: Include secret vars ansible.builtin.include_vars: "secret_group_vars/internal.yml" - when: "internal" in group_names + when: "'internal' in group_names" - name: Copy server key into VM temporarily ansible.builtin.copy: src: server_ca From 4e2c9e4011eb1722df6155a758d7b7e761ccf7fe Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Thu, 21 Dec 2023 17:04:47 +0100 Subject: [PATCH 52/52] use hostnames only --- ansible/roles/block-ips/defaults/main.yml | 3 - ansible/roles/block-ips/files/hosts-to-block | 1460 +++++++++++++++++ ansible/roles/block-ips/tasks/main.yml | 25 +- .../templates/block-outbound.service | 15 - .../block-ips/templates/block-outbound.sh.j2 | 7 - 5 files changed, 1464 insertions(+), 46 deletions(-) delete mode 100644 ansible/roles/block-ips/defaults/main.yml create mode 100644 ansible/roles/block-ips/files/hosts-to-block delete mode 100644 ansible/roles/block-ips/templates/block-outbound.service delete mode 100644 ansible/roles/block-ips/templates/block-outbound.sh.j2 diff --git a/ansible/roles/block-ips/defaults/main.yml b/ansible/roles/block-ips/defaults/main.yml deleted file mode 100644 index ec8a6b1..0000000 --- a/ansible/roles/block-ips/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -block_ips_urls: - - https://minerstat.com/mining-pool-whitelist.txt \ No newline at end of file diff --git a/ansible/roles/block-ips/files/hosts-to-block b/ansible/roles/block-ips/files/hosts-to-block new file mode 100644 index 0000000..e319c12 --- /dev/null +++ b/ansible/roles/block-ips/files/hosts-to-block @@ -0,0 +1,1460 @@ +127.0.0.1 2miners.com +127.0.0.1 51pool.online +127.0.0.1 ae.2miners.com +127.0.0.1 aeon.miner.rocks +127.0.0.1 aeon.minercountry.com +127.0.0.1 aergo.eu.mine.zpool.ca +127.0.0.1 aergo.jp.mine.zpool.ca +127.0.0.1 aergo.na.mine.zpool.ca +127.0.0.1 ahamay.net +127.0.0.1 aikapool.com +127.0.0.1 aionmine.org +127.0.0.1 aka.2miners.com +127.0.0.1 akroma.wattpool.net +127.0.0.1 allium.asia.mine.zergpool.com +127.0.0.1 allium.eu.mine.zpool.ca +127.0.0.1 allium.jp.mine.zpool.ca +127.0.0.1 allium.mine.zergpool.com +127.0.0.1 allium.na.mine.zpool.ca +127.0.0.1 americas.blake2s.mining-dutch.nl +127.0.0.1 americas.equihash.mining-dutch.nl +127.0.0.1 americas.groestl.mining-dutch.nl +127.0.0.1 americas.kawpow.mining-dutch.nl +127.0.0.1 americas.keccak.mining-dutch.nl +127.0.0.1 americas.lyra2rev2.mining-dutch.nl +127.0.0.1 americas.myrgro.mining-dutch.nl +127.0.0.1 americas.neoscrypt.mining-dutch.nl +127.0.0.1 americas.quark.mining-dutch.nl +127.0.0.1 americas.qubit.mining-dutch.nl +127.0.0.1 americas.scrypt.mining-dutch.nl +127.0.0.1 americas.sha256.mining-dutch.nl +127.0.0.1 americas.skein.mining-dutch.nl +127.0.0.1 americas.tribus.mining-dutch.nl +127.0.0.1 americas.x11.mining-dutch.nl +127.0.0.1 americas.x11gost.mining-dutch.nl +127.0.0.1 americas.yescrypt.mining-dutch.nl +127.0.0.1 americas.yescryptr16.mining-dutch.nl +127.0.0.1 americas.yescryptr32.mining-dutch.nl +127.0.0.1 americas.yespower.mining-dutch.nl +127.0.0.1 americas.yespowerr16.mining-dutch.nl +127.0.0.1 anon.2miners.com +127.0.0.1 anon.altpool.pro +127.0.0.1 ap-01.miningrigrentals.com +127.0.0.1 ap-nodes.mainnet.beam.mw +127.0.0.1 ap.luckpool.net +127.0.0.1 ap.nimpool.io +127.0.0.1 api.bsod.pw +127.0.0.1 arctichash.na.mine.zpool.ca +127.0.0.1 argon2.minercountry.com +127.0.0.1 argon2d-dyn.asia.mine.zergpool.com +127.0.0.1 argon2d-dyn.eu.mine.zergpool.com +127.0.0.1 argon2d-dyn.eu.mine.zpool.ca +127.0.0.1 argon2d-dyn.jp.mine.zpool.ca +127.0.0.1 argon2d-dyn.mine.zergpool.com +127.0.0.1 argon2d-dyn.na.mine.zergpool.com +127.0.0.1 argon2d-dyn.na.mine.zpool.ca +127.0.0.1 argon2d250.asia.mine.zergpool.com +127.0.0.1 argon2d250.eu.mine.zergpool.com +127.0.0.1 argon2d250.mine.zergpool.com +127.0.0.1 argon2d4096.asia.mine.zergpool.com +127.0.0.1 argon2d4096.eu.mine.zergpool.com +127.0.0.1 argon2d4096.eu.mine.zpool.ca +127.0.0.1 argon2d4096.mine.zergpool.com +127.0.0.1 argon2d4096.na.mine.zpool.ca +127.0.0.1 argon2d500.eu.mine.zpool.ca +127.0.0.1 argon2d500.na.mine.zpool.ca +127.0.0.1 arms01.p2poolmining.us +127.0.0.1 arq.pool.gntl.co.uk +127.0.0.1 arrow.dapool.io +127.0.0.1 arrr-eu.luxor.tech +127.0.0.1 arrr-us.luxor.tech +127.0.0.1 arrr.luxor.tech +127.0.0.1 arw.snowmining.com +127.0.0.1 as1-zil.shardpool.io +127.0.0.1 asia-ae.2miners.com +127.0.0.1 asia-beam.2miners.com +127.0.0.1 asia-btg.2miners.com +127.0.0.1 asia-ckb.2miners.com +127.0.0.1 asia-etc.2miners.com +127.0.0.1 asia-grin.2miners.com +127.0.0.1 asia-rvn.2miners.com +127.0.0.1 asia-solo-ae.2miners.com +127.0.0.1 asia-solo-btg.2miners.com +127.0.0.1 asia-solo-etc.2miners.com +127.0.0.1 asia-solo-rvn.2miners.com +127.0.0.1 asia-xzc.2miners.com +127.0.0.1 asia-zec.2miners.com +127.0.0.1 asia.aionpool.tech +127.0.0.1 asia.blake2s.mining-dutch.nl +127.0.0.1 asia.bsod.pw +127.0.0.1 asia.dbixmine.pro +127.0.0.1 asia.equihash.mining-dutch.nl +127.0.0.1 asia.equihub.pro +127.0.0.1 asia.ethash-hub.miningpoolhub.com +127.0.0.1 asia.ezil.me +127.0.0.1 asia.frostypool.com +127.0.0.1 asia.groestl.mining-dutch.nl +127.0.0.1 asia.kawpow-hub.miningpoolhub.com +127.0.0.1 asia.keccak.mining-dutch.nl +127.0.0.1 asia.lyra2rev2.mining-dutch.nl +127.0.0.1 asia.mecrypto.club +127.0.0.1 asia.miner.tokyo +127.0.0.1 asia.myrgro.mining-dutch.nl +127.0.0.1 asia.neoscrypt.mining-dutch.nl +127.0.0.1 asia.quark.mining-dutch.nl +127.0.0.1 asia.qubit.mining-dutch.nl +127.0.0.1 asia.randomx-hub.miningpoolhub.com +127.0.0.1 asia.ravenminer.com +127.0.0.1 asia.scrypt.mining-dutch.nl +127.0.0.1 asia.sha256.mining-dutch.nl +127.0.0.1 asia.siamining.com +127.0.0.1 asia.skein.mining-dutch.nl +127.0.0.1 asia.tribus.mining-dutch.nl +127.0.0.1 asia.x11.mining-dutch.nl +127.0.0.1 asia.x11gost.mining-dutch.nl +127.0.0.1 asia.yescrypt.mining-dutch.nl +127.0.0.1 asia.yescryptr16.mining-dutch.nl +127.0.0.1 asia.yescryptr32.mining-dutch.nl +127.0.0.1 asia.yespower.mining-dutch.nl +127.0.0.1 asia.yespowerr16.mining-dutch.nl +127.0.0.1 asia1-beam.flypool.org +127.0.0.1 aspac-etc.hiveon.net +127.0.0.1 aspac1-etc.hiveon.net +127.0.0.1 aspac1-eth.hiveon.net +127.0.0.1 astralhash.eu.mine.zpool.ca +127.0.0.1 astralhash.na.mine.zpool.ca +127.0.0.1 atheios.wattpool.net +127.0.0.1 au-01.miningrigrentals.com +127.0.0.1 axe.antpool.com +127.0.0.1 b4c.f2pool.com +127.0.0.1 baikalmine.com +127.0.0.1 bbr.luckypool.io +127.0.0.1 bbscoin.my-mining-pool.de +127.0.0.1 bcd.eu.mine.zpool.ca +127.0.0.1 bcd.jp.mine.zpool.ca +127.0.0.1 bcd.mine.blazepool.com +127.0.0.1 bcd.na.mine.zpool.ca +127.0.0.1 bcd.sea.mine.zpool.ca +127.0.0.1 bch.ukrpool.com +127.0.0.1 bch.viabtc.com +127.0.0.1 bch.viabtc.top +127.0.0.1 bci.suprnova.cc +127.0.0.1 bcn.pool.minergate.com +127.0.0.1 bcx.vvpool.com +127.0.0.1 beam-asia.leafpool.com +127.0.0.1 beam-eu.leafpool.com +127.0.0.1 beam-us.leafpool.com +127.0.0.1 beam.2miners.com +127.0.0.1 beam.acepool.top +127.0.0.1 beam.antpool.com +127.0.0.1 beam.leafpool.com +127.0.0.1 beam.sparkpool.com +127.0.0.1 beam.sunpool.top +127.0.0.1 beam.suprnova.cc +127.0.0.1 beamv3.br.nicehash.com +127.0.0.1 beamv3.eu.nicehash.com +127.0.0.1 beamv3.hk.nicehash.com +127.0.0.1 beamv3.in.nicehash.com +127.0.0.1 beamv3.jp.nicehash.com +127.0.0.1 beamv3.usa.nicehash.com +127.0.0.1 bibop.net +127.0.0.1 binarium-v1.eu.mine.zpool.ca +127.0.0.1 binarium-v1.jp.mine.zpool.ca +127.0.0.1 binarium-v1.na.mine.zpool.ca +127.0.0.1 bitcoin.viabtc.com +127.0.0.1 bitcoin.viabtc.top +127.0.0.1 bitcore.asia.mine.zergpool.com +127.0.0.1 bitcore.eu.mine.zpool.ca +127.0.0.1 bitcore.jp.mine.zpool.ca +127.0.0.1 bitcore.mine.zergpool.com +127.0.0.1 bitcore.na.mine.zpool.ca +127.0.0.1 bittubecash.miner.rocks +127.0.0.1 blake2b.asia.mine.zergpool.com +127.0.0.1 blake2b.eu.mine.zpool.ca +127.0.0.1 blake2b.mine.zergpool.com +127.0.0.1 blake2b.na.mine.zpool.ca +127.0.0.1 blake2s.asia.mine.zergpool.com +127.0.0.1 blake2s.br.nicehash.com +127.0.0.1 blake2s.eu.mine.zpool.ca +127.0.0.1 blake2s.eu.nicehash.com +127.0.0.1 blake2s.hk.nicehash.com +127.0.0.1 blake2s.in.nicehash.com +127.0.0.1 blake2s.jp.mine.zpool.ca +127.0.0.1 blake2s.jp.nicehash.com +127.0.0.1 blake2s.mine.blazepool.com +127.0.0.1 blake2s.mine.zergpool.com +127.0.0.1 blake2s.mining-dutch.nl +127.0.0.1 blake2s.na.mine.zpool.ca +127.0.0.1 blake2s.pool.atomminer.com +127.0.0.1 blake2s.sea.mine.zpool.ca +127.0.0.1 blake2s.usa.nicehash.com +127.0.0.1 blakecoin.eu.mine.zpool.ca +127.0.0.1 blockify.xyz +127.0.0.1 blockmasters.co +127.0.0.1 bmw512.asia.mine.zergpool.com +127.0.0.1 bmw512.eu.mine.zergpool.com +127.0.0.1 bmw512.eu.mine.zpool.ca +127.0.0.1 bmw512.mine.zergpool.com +127.0.0.1 bmw512.na.mine.zpool.ca +127.0.0.1 bs.poolbinance.com +127.0.0.1 bsod.pw +127.0.0.1 bsv.ukrpool.com +127.0.0.1 btc-eu.f2pool.com +127.0.0.1 btc-eu.luxor.tech +127.0.0.1 btc-mow.f2pool.com +127.0.0.1 btc-us.f2pool.com +127.0.0.1 btc-us.luxor.tech +127.0.0.1 btc.blockwarepool.com +127.0.0.1 btc.f2pool.com +127.0.0.1 btc.luxor.tech +127.0.0.1 btc.ss.poolin.com +127.0.0.1 btc.trustpool.ru +127.0.0.1 btc.ukrpool.com +127.0.0.1 btc.viabtc.com +127.0.0.1 btc.viabtc.top +127.0.0.1 btcp.2miners.com +127.0.0.1 btcp.suprnova.cc +127.0.0.1 btcv.gomine.pro +127.0.0.1 btcz.2miners.com +127.0.0.1 btcz.altpool.pro +127.0.0.1 btcz.suprnova.cc +127.0.0.1 btg.2miners.com +127.0.0.1 btg.hashcity.org +127.0.0.1 btg.pool.gold +127.0.0.1 btg.suprnova.cc +127.0.0.1 bth.altpool.pro +127.0.0.1 bze.pcmining.xyz +127.0.0.1 c11.asia.mine.zergpool.com +127.0.0.1 c11.eu.mine.zpool.ca +127.0.0.1 c11.jp.mine.zpool.ca +127.0.0.1 c11.mine.zergpool.com +127.0.0.1 c11.na.mine.zpool.ca +127.0.0.1 ca-qrl.miningocean.org +127.0.0.1 ca-tor01.miningrigrentals.com +127.0.0.1 ca-upx.miningocean.org +127.0.0.1 ca.conceal.herominers.com +127.0.0.1 ca.haven.herominers.com +127.0.0.1 ca.haven.miner.rocks +127.0.0.1 ca.monero.herominers.com +127.0.0.1 ca.rvn.minermore.com +127.0.0.1 ca.scala.herominers.com +127.0.0.1 ca.stratu.ms +127.0.0.1 ca.stratum.slushpool.com +127.0.0.1 cfx-eu.ss.poolflare.net +127.0.0.1 cfx-eu.ss.poolflare.net +127.0.0.1 cfx-us.ss.poolflare.net +127.0.0.1 cfx.ss.poolflare.net +127.0.0.1 cfx.ss.poolflare.net +127.0.0.1 champmine.de +127.0.0.1 chicago01.hashvault.pro +127.0.0.1 ckb.2miners.com +127.0.0.1 ckb.antpool.com +127.0.0.1 ckb.f2pool.com +127.0.0.1 ckb.sparkpool.com +127.0.0.1 ckb.stratum.hashpool.com +127.0.0.1 clo.2miners.com +127.0.0.1 cluster.aionpool.tech +127.0.0.1 cmm-eu.coinblockers.com +127.0.0.1 cmra.miningpool.fun +127.0.0.1 coinotron.com +127.0.0.1 coins4u.cc +127.0.0.1 comining.io +127.0.0.1 conceal.herominers.com +127.0.0.1 conceal.miner.rocks +127.0.0.1 connect.pool.bitcoin.com +127.0.0.1 coolpool.top +127.0.0.1 cpacoin.my-mining-pool.de +127.0.0.1 cpu-pool.com +127.0.0.1 cpupower.asia.mine.zergpool.com +127.0.0.1 cpupower.eu.mine.zergpool.com +127.0.0.1 cpupower.eu.mine.zpool.ca +127.0.0.1 cpupower.mine.zergpool.com +127.0.0.1 crypit.net +127.0.0.1 cryptodredge-ax16r-oeu.bsod.pw +127.0.0.1 cryptonightr.br.nicehash.com +127.0.0.1 cryptonightr.eu.nicehash.com +127.0.0.1 cryptonightr.hk.nicehash.com +127.0.0.1 cryptonightr.in.nicehash.com +127.0.0.1 cryptonightr.jp.nicehash.com +127.0.0.1 cryptonightr.usa.nicehash.com +127.0.0.1 cryptonote.club +127.0.0.1 cryptonote.social +127.0.0.1 ctxc.2miners.com +127.0.0.1 cuckoo.cortexmint.com +127.0.0.1 cuckoocycle.br.nicehash.com +127.0.0.1 cuckoocycle.eu.nicehash.com +127.0.0.1 cuckoocycle.hk.nicehash.com +127.0.0.1 cuckoocycle.in.nicehash.com +127.0.0.1 cuckoocycle.jp.nicehash.com +127.0.0.1 cuckoocycle.usa.nicehash.com +127.0.0.1 daggerhashimoto.br.nicehash.com +127.0.0.1 daggerhashimoto.eu.nicehash.com +127.0.0.1 daggerhashimoto.hk.nicehash.com +127.0.0.1 daggerhashimoto.in.nicehash.com +127.0.0.1 daggerhashimoto.jp.nicehash.com +127.0.0.1 daggerhashimoto.usa.nicehash.com +127.0.0.1 dash-asia.luxor.tech +127.0.0.1 dash-eu.luxor.tech +127.0.0.1 dash.f2pool.com +127.0.0.1 dash.ss.poolin.com +127.0.0.1 dash.suprnova.cc +127.0.0.1 dbix-us.maxhash.org +127.0.0.1 dbix.hashcity.org +127.0.0.1 dbix.pool.sexy +127.0.0.1 dcr-eu.luxor.tech +127.0.0.1 dcr-us.luxor.tech +127.0.0.1 dcr.antpool.com +127.0.0.1 de.ckpool.org +127.0.0.1 de.equihash.pro +127.0.0.1 de.stratu.ms +127.0.0.1 de01.supportxmr.com +127.0.0.1 de02.supportxmr.com +127.0.0.1 decred.eu.mine.zpool.ca +127.0.0.1 decred.jp.mine.zpool.ca +127.0.0.1 decred.na.mine.zpool.ca +127.0.0.1 dedal.eu.mine.zpool.ca +127.0.0.1 dero.antpool.com +127.0.0.1 dero.miner.rocks +127.0.0.1 dero.ss.dxpool.com +127.0.0.1 dgb-odocrypt.f2pool.com +127.0.0.1 dgb-scrypt.f2pool.com +127.0.0.1 dgb-sha256d.f2pool.com +127.0.0.1 dgbo.suprnova.cc +127.0.0.1 dgbq.suprnova.cc +127.0.0.1 dgbs.suprnova.cc +127.0.0.1 dig.wazn.io +127.0.0.1 digihash.co +127.0.0.1 dlinodes.com +127.0.0.1 donate.v2.xmrig.com +127.0.0.1 duckiepool.com +127.0.0.1 dvt.gomine.pro +127.0.0.1 dxpool.com +127.0.0.1 dyn.suprnova.cc +127.0.0.1 eaglesong.eu.nicehash.com +127.0.0.1 eaglesong.hk.nicehash.com +127.0.0.1 eaglesong.usa.nicehash.com +127.0.0.1 east.us.rvn.minermore.com +127.0.0.1 eccak.na.mine.zpool.ca +127.0.0.1 egem-esoteria.icanmining.ru +127.0.0.1 egem.digipools.org +127.0.0.1 ella.2miners.com +127.0.0.1 ella.digipools.org +127.0.0.1 ella.pool.sexy +127.0.0.1 em.huobipool.com +127.0.0.1 emc2.suprnova.cc +127.0.0.1 equihash.asia.mine.zergpool.com +127.0.0.1 equihash.br.nicehash.com +127.0.0.1 equihash.eu.mine.zergpool.com +127.0.0.1 equihash.eu.mine.zpool.ca +127.0.0.1 equihash.eu.nicehash.com +127.0.0.1 equihash.f2pool.com +127.0.0.1 equihash.hk.nicehash.com +127.0.0.1 equihash.in.nicehash.com +127.0.0.1 equihash.jp.mine.zpool.ca +127.0.0.1 equihash.jp.nicehash.com +127.0.0.1 equihash.luxor.tech +127.0.0.1 equihash.mine.zergpool.com +127.0.0.1 equihash.mining-dutch.nl +127.0.0.1 equihash.na.mine.zergpool.com +127.0.0.1 equihash.na.mine.zpool.ca +127.0.0.1 equihash.sea.mine.zpool.ca +127.0.0.1 equihash.usa.nicehash.com +127.0.0.1 equihash125.asia.mine.zergpool.com +127.0.0.1 equihash125.eu.mine.zergpool.com +127.0.0.1 equihash125.mine.zergpool.com +127.0.0.1 equihash125.na.mine.zergpool.com +127.0.0.1 equihash125.na.mine.zpool.ca +127.0.0.1 equihash144.asia.mine.zergpool.com +127.0.0.1 equihash144.eu.mine.zergpool.com +127.0.0.1 equihash144.eu.mine.zpool.ca +127.0.0.1 equihash144.jp.mine.zpool.ca +127.0.0.1 equihash144.mine.zergpool.com +127.0.0.1 equihash144.na.mine.zergpool.com +127.0.0.1 equihash144.na.mine.zpool.ca +127.0.0.1 equihash144.sea.mine.zpool.ca +127.0.0.1 equihash192.asia.mine.zergpool.com +127.0.0.1 equihash192.eu.mine.zergpool.com +127.0.0.1 equihash192.eu.mine.zpool.ca +127.0.0.1 equihash192.jp.mine.zpool.ca +127.0.0.1 equihash192.mine.zergpool.com +127.0.0.1 equihash192.na.mine.zergpool.com +127.0.0.1 equihash192.na.mine.zpool.ca +127.0.0.1 equihash192.sea.mine.zpool.ca +127.0.0.1 equihash96.eu.mine.zpool.ca +127.0.0.1 equihash96.na.mine.zpool.ca +127.0.0.1 es.huobipool.com +127.0.0.1 etc-eu.maxhash.org +127.0.0.1 etc-solo.baikalmine.com +127.0.0.1 etc-solo.wattpool.net +127.0.0.1 etc-us.maxhash.org +127.0.0.1 etc.2miners.com +127.0.0.1 etc.altpool.pro +127.0.0.1 etc.baikalmine.com +127.0.0.1 etc.coolpool.top +127.0.0.1 etc.digipools.org +127.0.0.1 etc.ethashpool.com +127.0.0.1 etc.ethermine.org +127.0.0.1 etc.f2pool.com +127.0.0.1 etc.hashcity.org +127.0.0.1 etc.pool.minergate.com +127.0.0.1 etc.pool.zet-tech.eu +127.0.0.1 etc.solomine.org +127.0.0.1 etcc.mole-pool.net +127.0.0.1 etcturk.com +127.0.0.1 eth-asia1.hellominer.com +127.0.0.1 eth-esoteria.icanmining.ru +127.0.0.1 eth-eu.f2pool.com +127.0.0.1 eth-eu.maxhash.org +127.0.0.1 eth-eu.sparkpool.com +127.0.0.1 eth-eu1.hellominer.com +127.0.0.1 eth-na.f2pool.com +127.0.0.1 eth-solo.wattpool.net +127.0.0.1 eth-us.f2pool.com +127.0.0.1 eth-us.maxhash.org +127.0.0.1 eth-us1.hellominer.com +127.0.0.1 eth.2miners.com +127.0.0.1 eth.altpool.pro +127.0.0.1 eth.digipools.org +127.0.0.1 eth.ethashpool.com +127.0.0.1 eth.f2pool.com +127.0.0.1 eth.hashcity.org +127.0.0.1 eth.hiveon.net +127.0.0.1 eth.miningfrance.io +127.0.0.1 eth.pool-moscow.ru +127.0.0.1 eth.pool.minergate.com +127.0.0.1 eth.pool.zet-tech.eu +127.0.0.1 eth.pool.zhizhu.top +127.0.0.1 eth.ss.poolin.com +127.0.0.1 eth.ukrpool.com +127.0.0.1 eth.viabtc.top +127.0.0.1 ethash-eu.unmineable.com +127.0.0.1 ethash-us.unmineable.com +127.0.0.1 ethash.asia.mine.zergpool.com +127.0.0.1 ethash.eu.mine.zergpool.com +127.0.0.1 ethash.mine.zergpool.com +127.0.0.1 ethash.na.mine.zergpool.com +127.0.0.1 ethash.tron-mining.com +127.0.0.1 ethash.unmineable.com +127.0.0.1 ether1-solo.wattpool.net +127.0.0.1 ether1.weeminepool.com +127.0.0.1 ethereum.trustpool.ru +127.0.0.1 ethermine.org +127.0.0.1 etho.pool.sexy +127.0.0.1 ethv2-eu.f2pool.com +127.0.0.1 etnx.pool-pay.com +127.0.0.1 etp.2miners.com +127.0.0.1 etp.dodopool.com +127.0.0.1 etp.sandpool.org +127.0.0.1 etx.coolpool.top +127.0.0.1 eu-01.miningrigrentals.com +127.0.0.1 eu-bch.ss.btc.com +127.0.0.1 eu-de01.miningrigrentals.com +127.0.0.1 eu-de02.miningrigrentals.com +127.0.0.1 eu-etc.hiveon.net +127.0.0.1 eu-eth.hiveon.net +127.0.0.1 eu-mine.smartcash.cc +127.0.0.1 eu-pool.wienchain.com +127.0.0.1 eu-ru01.miningrigrentals.com +127.0.0.1 eu-solo-zel.zellabs.net +127.0.0.1 eu-west-stratum.grinmint.com +127.0.0.1 eu-xsg.equihub.pro +127.0.0.1 eu-zcl.equihub.pro +127.0.0.1 eu-zel.equihub.pro +127.0.0.1 eu-zel.zellabs.net +127.0.0.1 eu-zer.equihub.pro +127.0.0.1 eu-zero.equihub.pro +127.0.0.1 eu-zil.rustpool.xyz +127.0.0.1 eu.aionpool.tech +127.0.0.1 eu.altpool.eu +127.0.0.1 eu.bsod.pw +127.0.0.1 eu.btgpool.pro +127.0.0.1 eu.crazypool.org +127.0.0.1 eu.emcd.io +127.0.0.1 eu.equihub.pro +127.0.0.1 eu.etc.ezil.me +127.0.0.1 eu.ezil.me +127.0.0.1 eu.frostypool.com +127.0.0.1 eu.gethash.cc +127.0.0.1 eu.hyperpool.tech +127.0.0.1 eu.luckpool.net +127.0.0.1 eu.mecrypto.club +127.0.0.1 eu.nimpool.io +127.0.0.1 eu.pool.ms +127.0.0.1 eu.ravenminer.com +127.0.0.1 eu.rig.tokyo +127.0.0.1 eu.rito.minermore.com +127.0.0.1 eu.rvn.minermore.com +127.0.0.1 eu.rvnsolo.k1pool.com +127.0.0.1 eu.sandbox.pool.ms +127.0.0.1 eu.siamining.com +127.0.0.1 eu.ss.btc.com +127.0.0.1 eu.ss.dpool.top +127.0.0.1 eu.stratum.slushpool.com +127.0.0.1 eu.ubiqpool.io +127.0.0.1 eu1-3g.whalesburg.com +127.0.0.1 eu1-beam.flypool.org +127.0.0.1 eu1-zil.shardpool.io +127.0.0.1 eu1.bsod.pw +127.0.0.1 eu1.equihub.pro +127.0.0.1 eu1.fairpool.pro +127.0.0.1 eu1.hellominer.com +127.0.0.1 eu1.ltc.sigmapool.com +127.0.0.1 eu1.mastermining.net +127.0.0.1 eu1.whalesburg.com +127.0.0.1 eu1.zhash.pro +127.0.0.1 eu2-zil.rustpool.xyz +127.0.0.1 eu2.emcd.io +127.0.0.1 eu2.whalesburg.com +127.0.0.1 eupool.sinovate.io +127.0.0.1 eurocdy.pool.awmlite.com +127.0.0.1 europe.dgb256.online +127.0.0.1 europe.ethash-hub.miningpoolhub.com +127.0.0.1 europe.kawpow-hub.miningpoolhub.com +127.0.0.1 europe.miner.tokyo +127.0.0.1 europe.randomx-hub.miningpoolhub.com +127.0.0.1 exp.2miners.com +127.0.0.1 ezil.me +127.0.0.1 f2pool.com +127.0.0.1 fastepic.eu +127.0.0.1 fastpool.xyz +127.0.0.1 fed.cryptonote.club +127.0.0.1 fenixpool.top +127.0.0.1 fi.haven.herominers.com +127.0.0.1 fr.moneroocean.stream +127.0.0.1 frankfurt01.hashvault.pro +127.0.0.1 freshgarlicblocks.net +127.0.0.1 fullnode.quarkchain.io +127.0.0.1 gap.suprnova.cc +127.0.0.1 gate.emcd.io +127.0.0.1 geo.pool.akroma.eu +127.0.0.1 geo.pool.whalecoin.eu +127.0.0.1 gomine.pro +127.0.0.1 gpuhash.org +127.0.0.1 gpuhot.com +127.0.0.1 graft.ingest.cryptoknight.cc +127.0.0.1 graft.westcoastxmr.ca +127.0.0.1 grimm.sunpool.top +127.0.0.1 grin-pool.org +127.0.0.1 grin.2miners.com +127.0.0.1 grin.antpool.com +127.0.0.1 grin.pool.minergate.com +127.0.0.1 grin.sparkpool.com +127.0.0.1 grincuckatoo32.eu.nicehash.com +127.0.0.1 grincuckatoo32.hk.nicehash.com +127.0.0.1 grincuckatoo32.usa.nicehash.com +127.0.0.1 groestl.asia.mine.zergpool.com +127.0.0.1 groestl.eu.mine.zpool.ca +127.0.0.1 groestl.jp.mine.zpool.ca +127.0.0.1 groestl.mine.zergpool.com +127.0.0.1 groestl.mining-dutch.nl +127.0.0.1 groestl.na.mine.zpool.ca +127.0.0.1 grs.suprnova.cc +127.0.0.1 gs.poolbinance.com +127.0.0.1 gulf.moneroocean.stream +127.0.0.1 handshake-de.6block.com +127.0.0.1 handshake-us.6block.com +127.0.0.1 handshake.6block.com +127.0.0.1 hathor.acc-pool.pw +127.0.0.1 haven.herominers.com +127.0.0.1 haven.miner.rocks +127.0.0.1 hc.antpool.com +127.0.0.1 hex.eu.mine.zpool.ca +127.0.0.1 hex.jp.mine.zpool.ca +127.0.0.1 hex.na.mine.zpool.ca +127.0.0.1 hk.haven.herominers.com +127.0.0.1 hmq1725.asia.mine.zergpool.com +127.0.0.1 hmq1725.eu.mine.zergpool.com +127.0.0.1 hmq1725.eu.mine.zpool.ca +127.0.0.1 hmq1725.jp.mine.zpool.ca +127.0.0.1 hmq1725.mine.blazepool.com +127.0.0.1 hmq1725.mine.zergpool.com +127.0.0.1 hmq1725.na.mine.zpool.ca +127.0.0.1 hmq1725.sea.mine.zpool.ca +127.0.0.1 hns.f2pool.com +127.0.0.1 hns.pool.blackminer.com +127.0.0.1 hns.ss.dxpool.com +127.0.0.1 hodl.suprnova.cc +127.0.0.1 honeycomb.eu.mine.zpool.ca +127.0.0.1 honeycomb.na.mine.zpool.ca +127.0.0.1 hub.miningpoolhub.com +127.0.0.1 hush.2miners.com +127.0.0.1 hush.suprnova.cc +127.0.0.1 icemining.ca +127.0.0.1 in.stratu.ms +127.0.0.1 infinity.ultranote.org +127.0.0.1 italo.ingest.cryptoknight.cc +127.0.0.1 italo.network +127.0.0.1 jeonghash.eu.mine.zpool.ca +127.0.0.1 jp-01.miningrigrentals.com +127.0.0.1 jp.stratum.slushpool.com +127.0.0.1 k12.asia.mine.zergpool.com +127.0.0.1 k12.mine.zergpool.com +127.0.0.1 k12.na.mine.zergpool.com +127.0.0.1 kawpow.asia.mine.zergpool.com +127.0.0.1 kawpow.br.nicehash.com +127.0.0.1 kawpow.eu.mine.zergpool.com +127.0.0.1 kawpow.eu.nicehash.com +127.0.0.1 kawpow.hk.nicehash.com +127.0.0.1 kawpow.in.nicehash.com +127.0.0.1 kawpow.jp.nicehash.com +127.0.0.1 kawpow.mine.zergpool.com +127.0.0.1 kawpow.mining-dutch.nl +127.0.0.1 kawpow.na.mine.zergpool.com +127.0.0.1 kawpow.usa.nicehash.com +127.0.0.1 kda-asia.ss.poolflare.net +127.0.0.1 kda-asia.ss.poolflare.net +127.0.0.1 kda-eu.ss.poolflare.com +127.0.0.1 kda-eu.ss.poolflare.net +127.0.0.1 kda-eu2.ss.poolflare.net +127.0.0.1 kda-us.ss.poolflare.com +127.0.0.1 kda-us.ss.poolflare.net +127.0.0.1 kda.f2pool.com +127.0.0.1 kda.ss.poolflare.com +127.0.0.1 kda.ss.poolflare.net +127.0.0.1 kda.ss.poolflare.net +127.0.0.1 kda.stratum.hashpool.com +127.0.0.1 keccak.asia.mine.zergpool.com +127.0.0.1 keccak.eu.mine.zpool.ca +127.0.0.1 keccak.eu.nicehash.com +127.0.0.1 keccak.hk.nicehash.com +127.0.0.1 keccak.jp.mine.zpool.ca +127.0.0.1 keccak.mine.zergpool.com +127.0.0.1 keccak.mining-dutch.nl +127.0.0.1 keccak.na.mine.zergpool.com +127.0.0.1 keccak.na.mine.zpool.ca +127.0.0.1 keccak.usa.nicehash.com +127.0.0.1 keccakc.asia.mine.zergpool.com +127.0.0.1 keccakc.eu.mine.zpool.ca +127.0.0.1 keccakc.jp.mine.zpool.ca +127.0.0.1 keccakc.mine.zergpool.com +127.0.0.1 keccakc.na.mine.zpool.ca +127.0.0.1 kingsminer.ddnsking.com +127.0.0.1 kp.unmineable.com +127.0.0.1 kreds.suprnova.cc +127.0.0.1 kva.ss.dxpool.com +127.0.0.1 kz.emcd.io +127.0.0.1 lb.geo.egempool.eu +127.0.0.1 lb.geo.pirlpool.eu +127.0.0.1 lb.geo.ubiqpool.org +127.0.0.1 lbc.luxor.tech +127.0.0.1 lbk3.eu.mine.zpool.ca +127.0.0.1 lbk3.jp.mine.zpool.ca +127.0.0.1 lbk3.na.mine.zpool.ca +127.0.0.1 lbry.asia.mine.zergpool.com +127.0.0.1 lbry.eu.mine.zpool.ca +127.0.0.1 lbry.eu.nicehash.com +127.0.0.1 lbry.hk.nicehash.com +127.0.0.1 lbry.jp.mine.zpool.ca +127.0.0.1 lbry.mine.zergpool.com +127.0.0.1 lbry.na.mine.zpool.ca +127.0.0.1 lbry.suprnova.cc +127.0.0.1 lbry.usa.nicehash.com +127.0.0.1 lidonia.com +127.0.0.1 litecoinpool.org +127.0.0.1 ls.huobipool.com +127.0.0.1 ltc-eu.f2pool.com +127.0.0.1 ltc-us.f2pool.com +127.0.0.1 ltc.f2pool.com +127.0.0.1 ltc.pool.minergate.com +127.0.0.1 ltc.ss.poolin.com +127.0.0.1 ltc.viabtc.com +127.0.0.1 ltc.viabtc.top +127.0.0.1 ltz.pcmining.xyz +127.0.0.1 lyra2rev2.eu.nicehash.com +127.0.0.1 lyra2rev2.hk.nicehash.com +127.0.0.1 lyra2rev2.mining-dutch.nl +127.0.0.1 lyra2rev2.usa.nicehash.com +127.0.0.1 lyra2v2.asia.mine.zergpool.com +127.0.0.1 lyra2v2.eu.mine.zpool.ca +127.0.0.1 lyra2v2.jp.mine.zpool.ca +127.0.0.1 lyra2v2.mine.blazepool.com +127.0.0.1 lyra2v2.mine.zergpool.com +127.0.0.1 lyra2v2.na.mine.zergpool.com +127.0.0.1 lyra2v2.na.mine.zpool.ca +127.0.0.1 lyra2v3.eu.mine.zpool.ca +127.0.0.1 lyra2v3.jp.mine.zpool.ca +127.0.0.1 lyra2v3.mine.blazepool.com +127.0.0.1 lyra2v3.na.mine.zpool.ca +127.0.0.1 lyra2z.asia.mine.zergpool.com +127.0.0.1 lyra2z.eu.mine.zpool.ca +127.0.0.1 lyra2z.eu.nicehash.com +127.0.0.1 lyra2z.hk.nicehash.com +127.0.0.1 lyra2z.jp.mine.zpool.ca +127.0.0.1 lyra2z.mine.zergpool.com +127.0.0.1 lyra2z.na.mine.zergpool.com +127.0.0.1 lyra2z.na.mine.zpool.ca +127.0.0.1 lyra2z.usa.nicehash.com +127.0.0.1 lyra2z330.asia.mine.zergpool.com +127.0.0.1 lyra2z330.mine.zergpool.com +127.0.0.1 lyra2z330.na.mine.zergpool.com +127.0.0.1 m.1stminingpool.com +127.0.0.1 m7m.asia.mine.zergpool.com +127.0.0.1 m7m.eu.mine.zergpool.com +127.0.0.1 m7m.eu.mine.zpool.ca +127.0.0.1 m7m.jp.mine.zpool.ca +127.0.0.1 m7m.mine.zergpool.com +127.0.0.1 m7m.na.mine.zpool.ca +127.0.0.1 m7m.sea.mine.zpool.ca +127.0.0.1 mallob-ml.as.neuropool.net +127.0.0.1 mallob-ml.as2.neuropool.net +127.0.0.1 mallob-ml.au.neuropool.net +127.0.0.1 mallob-ml.ca.neuropool.net +127.0.0.1 mallob-ml.eu.neuropool.net +127.0.0.1 mallob-ml.in.neuropool.net +127.0.0.1 mallob-ml.me.neuropool.net +127.0.0.1 mallob-ml.us.neuropool.net +127.0.0.1 masari.miner.rocks +127.0.0.1 mastermine.eu +127.0.0.1 mcmpool.eu +127.0.0.1 megabtx.asia.mine.zergpool.com +127.0.0.1 megabtx.eu.mine.zpool.ca +127.0.0.1 megabtx.mine.zergpool.com +127.0.0.1 megamec.asia.mine.zergpool.com +127.0.0.1 megamec.mine.zergpool.com +127.0.0.1 mine-beam-testnet.leafpool.com +127.0.0.1 mine.arqma.com +127.0.0.1 mine.bittubecash.fairpool.xyz +127.0.0.1 mine.c3pool.com +127.0.0.1 mine.etc.fairpool.xyz +127.0.0.1 mine.loki.fairpool.xyz +127.0.0.1 mine.msr.fairpool.xyz +127.0.0.1 mine.nlpool.nl +127.0.0.1 mine.nuko.fairpool.xyz +127.0.0.1 mine.pgc.fairpool.xyz +127.0.0.1 mine.realpool.eu +127.0.0.1 mine.ryo.fairpool.xyz +127.0.0.1 mine.swap.fairpool.xyz +127.0.0.1 mine.thegrinpool.com +127.0.0.1 mine.wow.fairpool.xyz +127.0.0.1 mine.xhv.fairpool.xyz +127.0.0.1 mine.xmrpool.net +127.0.0.1 mine.xrn.fairpool.xyz +127.0.0.1 mine.zano.fairpool.xyz +127.0.0.1 mine.zergpool.com +127.0.0.1 mine.zpool.ca +127.0.0.1 mineit.io +127.0.0.1 minepool.com +127.0.0.1 minepool.online +127.0.0.1 miner-control.de +127.0.0.1 miner.rocks +127.0.0.1 minerpool.net +127.0.0.1 minimizing.net +127.0.0.1 mining.bittube.app +127.0.0.1 mining.xpoolx.com +127.0.0.1 miningmadness.com +127.0.0.1 miningpoolhub.com +127.0.0.1 mix-solo.wattpool.net +127.0.0.1 mix.minerpool.net +127.0.0.1 mix.wattpool.net +127.0.0.1 mn.pool-pay.com +127.0.0.1 mng.miningpool.fun +127.0.0.1 moac.2miners.com +127.0.0.1 mona.suprnova.cc +127.0.0.1 monero.herominers.com +127.0.0.1 monerohash.com +127.0.0.1 music.2miners.com +127.0.0.1 mwc.2miners.com +127.0.0.1 my-mining-pool.de +127.0.0.1 myr-gr.asia.mine.zergpool.com +127.0.0.1 myr-gr.eu.mine.zpool.ca +127.0.0.1 myr-gr.jp.mine.zpool.ca +127.0.0.1 myr-gr.mine.zergpool.com +127.0.0.1 myr-gr.na.mine.zpool.ca +127.0.0.1 myrgro.mining-dutch.nl +127.0.0.1 na-etc.hiveon.net +127.0.0.1 na.aionpool.tech +127.0.0.1 na.luckpool.net +127.0.0.1 naw-eth.hiveon.net +127.0.0.1 nbr-pool.proxpool.com +127.0.0.1 nbr.pool-pay.com +127.0.0.1 neoscrypt.asia.mine.zergpool.com +127.0.0.1 neoscrypt.br.nicehash.com +127.0.0.1 neoscrypt.eu.mine.zergpool.com +127.0.0.1 neoscrypt.eu.mine.zpool.ca +127.0.0.1 neoscrypt.eu.nicehash.com +127.0.0.1 neoscrypt.eu1.unimining.net +127.0.0.1 neoscrypt.hk.nicehash.com +127.0.0.1 neoscrypt.in.nicehash.com +127.0.0.1 neoscrypt.jp.mine.zpool.ca +127.0.0.1 neoscrypt.jp.nicehash.com +127.0.0.1 neoscrypt.mine.blazepool.com +127.0.0.1 neoscrypt.mine.zergpool.com +127.0.0.1 neoscrypt.mining-dutch.nl +127.0.0.1 neoscrypt.na.mine.zpool.ca +127.0.0.1 neoscrypt.sea.mine.zpool.ca +127.0.0.1 neoscrypt.usa.nicehash.com +127.0.0.1 nicehash.com +127.0.0.1 nilu.minerpool.net +127.0.0.1 nimiq-trm.icemining.ca +127.0.0.1 nimiq.icemining.ca +127.0.0.1 nist5.asia.mine.zergpool.com +127.0.0.1 nist5.eu.mine.zpool.ca +127.0.0.1 nist5.jp.mine.zpool.ca +127.0.0.1 nist5.mine.zergpool.com +127.0.0.1 nist5.na.mine.zpool.ca +127.0.0.1 nl.stratu.ms +127.0.0.1 nlpool.nl +127.0.0.1 noncepool.com +127.0.0.1 nuko.minerpool.net +127.0.0.1 nuko.mofumofu.me +127.0.0.1 odocrypt.asia.mine.zergpool.com +127.0.0.1 odocrypt.eu.mine.zpool.ca +127.0.0.1 odocrypt.mine.zergpool.com +127.0.0.1 odocrypt.mine.zpool.ca +127.0.0.1 odocrypt.mining-dutch.nl +127.0.0.1 omb.pool-pay.com +127.0.0.1 p2p-spb.xyz +127.0.0.1 p2p-usa.xyz +127.0.0.1 padihash.eu.mine.zpool.ca +127.0.0.1 pasl.fairpool.xyz +127.0.0.1 pavhash.mooo.com +127.0.0.1 pawelhash.eu.mine.zpool.ca +127.0.0.1 perkle-pool.esprezzo.io +127.0.0.1 persianmine.com +127.0.0.1 pgc.minerpool.net +127.0.0.1 phi.asia.mine.zergpool.com +127.0.0.1 phi.eu.mine.zpool.ca +127.0.0.1 phi.jp.mine.zpool.ca +127.0.0.1 phi.mine.zergpool.com +127.0.0.1 phi.na.mine.zpool.ca +127.0.0.1 phi2.eu.mine.zpool.ca +127.0.0.1 phi2.jp.mine.zpool.ca +127.0.0.1 phi2.na.mine.zpool.ca +127.0.0.1 pirl-solo.wattpool.net +127.0.0.1 pirl.2miners.com +127.0.0.1 pirl.coinminer.space +127.0.0.1 pirl.pool.sexy +127.0.0.1 plpool.org +127.0.0.1 polytimos.eu.mine.zpool.ca +127.0.0.1 polytimos.jp.mine.zpool.ca +127.0.0.1 polytimos.na.mine.zpool.ca +127.0.0.1 pool-eu.ethosdistro.com +127.0.0.1 pool-usa.ethosdistro.com +127.0.0.1 pool.0cash.org +127.0.0.1 pool.572133.club +127.0.0.1 pool.acemining.co +127.0.0.1 pool.aionmine.org +127.0.0.1 pool.bibop.net +127.0.0.1 pool.bitcoinote.org +127.0.0.1 pool.blocpal.com +127.0.0.1 pool.bsod.pw +127.0.0.1 pool.btcp.network +127.0.0.1 pool.ckpool.org +127.0.0.1 pool.cryptopay.org.za +127.0.0.1 pool.dgb256.online +127.0.0.1 pool.egem.io +127.0.0.1 pool.ehpro.ca +127.0.0.1 pool.electroneropulse.org +127.0.0.1 pool.erstweal.com +127.0.0.1 pool.esn.today +127.0.0.1 pool.ethercore.io +127.0.0.1 pool.expanse.tech +127.0.0.1 pool.hashvault.pro +127.0.0.1 pool.helpico.tech +127.0.0.1 pool.italo.network +127.0.0.1 pool.joys.digital +127.0.0.1 pool.krb.fairhash.org +127.0.0.1 pool.lanacoin.com +127.0.0.1 pool.mangocoin.online +127.0.0.1 pool.minerall.io +127.0.0.1 pool.monerov.online +127.0.0.1 pool.ms +127.0.0.1 pool.nashcash.net +127.0.0.1 pool.peopleland.net +127.0.0.1 pool.pexaproject.com +127.0.0.1 pool.pigeoncoin.org +127.0.0.1 pool.pirl.network +127.0.0.1 pool.poolen.io +127.0.0.1 pool.qpool.net +127.0.0.1 pool.qwertycoin.org +127.0.0.1 pool.rwinfo.club +127.0.0.1 pool.ryo-currency.com +127.0.0.1 pool.sprintpay.net +127.0.0.1 pool.supportxmr.com +127.0.0.1 pool.swampcoin.tech +127.0.0.1 pool.swampthing.net +127.0.0.1 pool.talleo.org +127.0.0.1 pool.tube.fairhash.org +127.0.0.1 pool.wattum.io +127.0.0.1 pool.xmc.fairhash.org +127.0.0.1 pool.zelerius.org +127.0.0.1 pool.zls.fairhash.org +127.0.0.1 poolparty.cryptopay.org.za +127.0.0.1 power2b.asia.mine.zergpool.com +127.0.0.1 power2b.eu.mine.zergpool.com +127.0.0.1 power2b.eu.mine.zpool.ca +127.0.0.1 power2b.mine.zergpool.com +127.0.0.1 powerpool.money +127.0.0.1 pps-etc-eu.adaminers.com +127.0.0.1 pps-eth-eu.adaminers.com +127.0.0.1 profit.pool.bitcoin.com +127.0.0.1 prohashing.com +127.0.0.1 proxy.pool.whalesburg.com +127.0.0.1 proxy.prohashing.com +127.0.0.1 qkc.ontopool.com +127.0.0.1 qrl-us.leafpool.com +127.0.0.1 qrl.herominers.com +127.0.0.1 qrl.miningocean.org +127.0.0.1 quark.asia.mine.zergpool.com +127.0.0.1 quark.eu.mine.zpool.ca +127.0.0.1 quark.eu.nicehash.com +127.0.0.1 quark.hk.nicehash.com +127.0.0.1 quark.jp.mine.zpool.ca +127.0.0.1 quark.mine.zergpool.com +127.0.0.1 quark.mining-dutch.nl +127.0.0.1 quark.na.mine.zpool.ca +127.0.0.1 quark.usa.nicehash.com +127.0.0.1 qubit.asia.mine.zergpool.com +127.0.0.1 qubit.br.nicehash.com +127.0.0.1 qubit.eu.mine.zpool.ca +127.0.0.1 qubit.eu.nicehash.com +127.0.0.1 qubit.hk.nicehash.com +127.0.0.1 qubit.in.nicehash.com +127.0.0.1 qubit.jp.mine.zpool.ca +127.0.0.1 qubit.jp.nicehash.com +127.0.0.1 qubit.mine.zergpool.com +127.0.0.1 qubit.mining-dutch.nl +127.0.0.1 qubit.na.mine.zpool.ca +127.0.0.1 qubit.usa.nicehash.com +127.0.0.1 randomarq.asia.mine.zergpool.com +127.0.0.1 randomarq.mine.zergpool.com +127.0.0.1 randomsfx.asia.mine.zergpool.com +127.0.0.1 randomsfx.eu.mine.zergpool.com +127.0.0.1 randomsfx.mine.zergpool.com +127.0.0.1 randomx.eu.mine.zergpool.com +127.0.0.1 randomx.mine.zergpool.com +127.0.0.1 randomx.na.mine.zergpool.com +127.0.0.1 randomxmonero.br.nicehash.com +127.0.0.1 randomxmonero.eu.nicehash.com +127.0.0.1 randomxmonero.hk.nicehash.com +127.0.0.1 randomxmonero.in.nicehash.com +127.0.0.1 randomxmonero.jp.nicehash.com +127.0.0.1 randomxmonero.usa.nicehash.com +127.0.0.1 raven.f2pool.com +127.0.0.1 ravencoin-us.mintpond.com +127.0.0.1 ravencoin.flypool.org +127.0.0.1 ravencoin.mintpond.com +127.0.0.1 ravenminer.com +127.0.0.1 reosc.digipools.org +127.0.0.1 reosc.wattpool.net +127.0.0.1 rfv2.na.mine.zpool.ca +127.0.0.1 ric.suprnova.cc +127.0.0.1 roi.suprnova.cc +127.0.0.1 rto.pool-pay.com +127.0.0.1 ru-etc.hiveon.net +127.0.0.1 ru-eth.hiveon.net +127.0.0.1 ru.bsod.pw +127.0.0.1 ru.hashrent.pro +127.0.0.1 ru.miner.tokyo +127.0.0.1 rustpool.xyz +127.0.0.1 rvn-us.coinblockers.com +127.0.0.1 rvn.2miners.com +127.0.0.1 rvn.antpool.com +127.0.0.1 rvn.hashcity.org +127.0.0.1 rvn.suprnova.cc +127.0.0.1 rx.unmineable.com +127.0.0.1 ryo-eu.leafpool.com +127.0.0.1 ryo.bohemianpool.com +127.0.0.1 ryo.miner.rocks +127.0.0.1 ryopool.cryptosewer.com +127.0.0.1 s-eu.comining.io +127.0.0.1 s-jp.comining.io +127.0.0.1 s-ru.comining.io +127.0.0.1 s-sg.comining.io +127.0.0.1 s-us.comining.io +127.0.0.1 s.comining.io +127.0.0.1 sandbox.pool.ms +127.0.0.1 sc-eu.luxor.tech +127.0.0.1 sc-us.luxor.tech +127.0.0.1 sc.f2pool.com +127.0.0.1 sc.luxor.tech +127.0.0.1 scala.herominers.com +127.0.0.1 scp-eu.luxor.tech +127.0.0.1 scp-us.luxor.tech +127.0.0.1 scrypt-ld.eu.mine.zpool.ca +127.0.0.1 scrypt-ld.jp.mine.zpool.ca +127.0.0.1 scrypt-ld.na.mine.zpool.ca +127.0.0.1 scrypt.asia.mine.zergpool.com +127.0.0.1 scrypt.br.nicehash.com +127.0.0.1 scrypt.eu.mine.zergpool.com +127.0.0.1 scrypt.eu.mine.zpool.ca +127.0.0.1 scrypt.eu.nicehash.com +127.0.0.1 scrypt.hk.nicehash.com +127.0.0.1 scrypt.in.nicehash.com +127.0.0.1 scrypt.jp.mine.zpool.ca +127.0.0.1 scrypt.jp.nicehash.com +127.0.0.1 scrypt.mine.zergpool.com +127.0.0.1 scrypt.mining-dutch.nl +127.0.0.1 scrypt.na.mine.zergpool.com +127.0.0.1 scrypt.na.mine.zpool.ca +127.0.0.1 scrypt.sea.mine.zpool.ca +127.0.0.1 scrypt.usa.nicehash.com +127.0.0.1 scryptn2.eu.mine.zergpool.com +127.0.0.1 scryptn2.mine.zergpool.com +127.0.0.1 seu-eth.hiveon.net +127.0.0.1 sfx.pool-pay.com +127.0.0.1 sg-zil.rustpool.xyz +127.0.0.1 sg.haven.herominers.com +127.0.0.1 sg.haven.miner.rocks +127.0.0.1 sg.qrl.herominers.com +127.0.0.1 sg.stratu.ms +127.0.0.1 sg.stratum.slushpool.com +127.0.0.1 sha256-ld.eu.mine.zpool.ca +127.0.0.1 sha256-ld.jp.mine.zpool.ca +127.0.0.1 sha256-ld.na.mine.zpool.ca +127.0.0.1 sha256.antpool.com +127.0.0.1 sha256.asia.mine.zergpool.com +127.0.0.1 sha256.br.nicehash.com +127.0.0.1 sha256.eu.mine.zpool.ca +127.0.0.1 sha256.eu.nicehash.com +127.0.0.1 sha256.hk.nicehash.com +127.0.0.1 sha256.in.nicehash.com +127.0.0.1 sha256.jp.mine.zpool.ca +127.0.0.1 sha256.jp.nicehash.com +127.0.0.1 sha256.mine.zergpool.com +127.0.0.1 sha256.mining-dutch.nl +127.0.0.1 sha256.na.mine.zergpool.com +127.0.0.1 sha256.na.mine.zpool.ca +127.0.0.1 sha256.poolbinance.com +127.0.0.1 sha256.usa.nicehash.com +127.0.0.1 sha256asicboost.br.nicehash.com +127.0.0.1 sha256asicboost.eu.nicehash.com +127.0.0.1 sha256asicboost.hk.nicehash.com +127.0.0.1 sha256asicboost.in.nicehash.com +127.0.0.1 sha256asicboost.jp.nicehash.com +127.0.0.1 sha256asicboost.usa.nicehash.com +127.0.0.1 sha256t.eu.mine.zpool.ca +127.0.0.1 sha256t.jp.mine.zpool.ca +127.0.0.1 sha256t.na.mine.zpool.ca +127.0.0.1 sha3d.mine.zergpool.com +127.0.0.1 sha3d.mine.zpool.ca +127.0.0.1 sib.asia.mine.zergpool.com +127.0.0.1 sib.eu.mine.zpool.ca +127.0.0.1 sib.jp.mine.zpool.ca +127.0.0.1 sib.mine.zergpool.com +127.0.0.1 sib.na.mine.zpool.ca +127.0.0.1 sib.suprnova.cc +127.0.0.1 singapore01.hashvault.pro +127.0.0.1 sipc.simpool.vip +127.0.0.1 skein.asia.mine.zergpool.com +127.0.0.1 skein.eu.mine.zergpool.com +127.0.0.1 skein.eu.mine.zpool.ca +127.0.0.1 skein.jp.mine.zpool.ca +127.0.0.1 skein.mine.zergpool.com +127.0.0.1 skein.mine.zpool.ca +127.0.0.1 skein.mining-dutch.nl +127.0.0.1 skein.na.mine.zpool.ca +127.0.0.1 skein.sea.mine.zpool.ca +127.0.0.1 skein2.eu.mine.zpool.ca +127.0.0.1 skein2.mine.zergpool.com +127.0.0.1 skein2.na.mine.zpool.ca +127.0.0.1 skunk.eu.mine.zpool.ca +127.0.0.1 skunk.jp.mine.zpool.ca +127.0.0.1 skunk.na.mine.zpool.ca +127.0.0.1 soil.pool.sexy +127.0.0.1 solo-ae.2miners.com +127.0.0.1 solo-beam.2miners.com +127.0.0.1 solo-btcz.2miners.com +127.0.0.1 solo-btg.2miners.com +127.0.0.1 solo-clo.2miners.com +127.0.0.1 solo-ctxc.2miners.com +127.0.0.1 solo-dbix.2miners.com +127.0.0.1 solo-etc.2miners.com +127.0.0.1 solo-etc.altpool.pro +127.0.0.1 solo-eth.2miners.com +127.0.0.1 solo-etho.coolpool.top +127.0.0.1 solo-etp.2miners.com +127.0.0.1 solo-exp.2miners.com +127.0.0.1 solo-grin.2miners.com +127.0.0.1 solo-mwc.2miners.com +127.0.0.1 solo-nilu.mole-pool.net +127.0.0.1 solo-pirl.2miners.com +127.0.0.1 solo-rvn.2miners.com +127.0.0.1 solo-sero.mole-pool.net +127.0.0.1 solo-ubq.altpool.pro +127.0.0.1 solo-xmr.2miners.com +127.0.0.1 solo-xzc.2miners.com +127.0.0.1 solo-zcl.2miners.com +127.0.0.1 solo-zec.2miners.com +127.0.0.1 solo-zel.2miners.com +127.0.0.1 solo.antpool.com +127.0.0.1 solo.ckpool.org +127.0.0.1 soloetc.pool-moscow.ru +127.0.0.1 sonoa.asia.mine.zergpool.com +127.0.0.1 sonoa.eu.mine.zpool.ca +127.0.0.1 sonoa.jp.mine.zpool.ca +127.0.0.1 sonoa.mine.zergpool.com +127.0.0.1 sonoa.na.mine.zpool.ca +127.0.0.1 ss.antpool.com +127.0.0.1 ss.bfcpool.com +127.0.0.1 stellite.ingest-asia.cryptoknight.cc +127.0.0.1 stellite.ingest.cryptoknight.cc +127.0.0.1 stratum-asia.rplant.xyz +127.0.0.1 stratum-btm.antpool.com +127.0.0.1 stratum-dash.antpool.com +127.0.0.1 stratum-etc.antpool.com +127.0.0.1 stratum-eu.rplant.xyz +127.0.0.1 stratum-ltc.antpool.com +127.0.0.1 stratum-na.rplant.xyz +127.0.0.1 stratum-ravencoin.flypool.org +127.0.0.1 stratum-ru.rplant.xyz +127.0.0.1 stratum-sc.antpool.com +127.0.0.1 stratum-xmc.antpool.com +127.0.0.1 stratum-zec.antpool.com +127.0.0.1 stratum.aikapool.com +127.0.0.1 stratum.aionmine.org +127.0.0.1 stratum.antpool.com +127.0.0.1 stratum.bcmonster.com +127.0.0.1 stratum.coinminerz.com +127.0.0.1 stratum.cudopool.com +127.0.0.1 stratum.f2pool.com +127.0.0.1 stratum.gpuhot.com +127.0.0.1 stratum.happypool.co +127.0.0.1 stratum.icemining.ca +127.0.0.1 stratum.kano.is +127.0.0.1 stratum.mecrypto.club +127.0.0.1 stratum.miner-world.com +127.0.0.1 stratum.my2coins.com +127.0.0.1 stratum.naw.warihash.com +127.0.0.1 stratum.okpool.me +127.0.0.1 stratum.piratepool.io +127.0.0.1 stratum.ravenminer.com +127.0.0.1 stratum.sephira.co +127.0.0.1 stratum.slushpool.com +127.0.0.1 stratum.solomining.io +127.0.0.1 stratum.spaceworks.co +127.0.0.1 stratum.us2mine.com +127.0.0.1 stratum.yiimp.bobbpool.be +127.0.0.1 stratum.youcrazy.me +127.0.0.1 stratum.yourpool.org +127.0.0.1 stratum.zel.cash +127.0.0.1 stratum1.hashcryptos.com +127.0.0.1 stratum3.hashcryptos.com +127.0.0.1 stratum4.hashcryptos.com +127.0.0.1 sumo.ss.dxpool.com +127.0.0.1 sumokoin.miner.rocks +127.0.0.1 supportxmr.com +127.0.0.1 swap.gxn-miningpool.com +127.0.0.1 swap.ingest.cryptoknight.cc +127.0.0.1 swap.miningocean.org +127.0.0.1 swap2.luckypool.io +127.0.0.1 tbdice.org +127.0.0.1 texas.us.rvn.minermore.com +127.0.0.1 timetravel.eu.mine.zpool.ca +127.0.0.1 timetravel.jp.mine.zpool.ca +127.0.0.1 timetravel.na.mine.zpool.ca +127.0.0.1 tlo.cryptonote.club +127.0.0.1 tneoscrypt.na.mine.zpool.ca +127.0.0.1 tokenminingpool.com +127.0.0.1 trexmining.com +127.0.0.1 tribus.asia.mine.zergpool.com +127.0.0.1 tribus.eu.mine.zpool.ca +127.0.0.1 tribus.jp.mine.zpool.ca +127.0.0.1 tribus.mine.blazepool.com +127.0.0.1 tribus.mine.zergpool.com +127.0.0.1 tribus.mining-dutch.nl +127.0.0.1 tribus.na.mine.zpool.ca +127.0.0.1 trtl.pool.mine2gether.com +127.0.0.1 tsf.europool.me +127.0.0.1 tube.ingest.cryptoknight.cc +127.0.0.1 tube.steadyhash.org +127.0.0.1 tube4.luckypool.io +127.0.0.1 tw.sparkpool.com +127.0.0.1 ua-mining.com +127.0.0.1 ubiq-eu.maxhash.org +127.0.0.1 ubiq-us.maxhash.org +127.0.0.1 ubiq.suprnova.cc +127.0.0.1 ubiq.wattpool.net +127.0.0.1 ubq.altpool.pro +127.0.0.1 uk.stratu.ms +127.0.0.1 ukpool.electroneropulse.org +127.0.0.1 upx.miningocean.org +127.0.0.1 upxpool.com +127.0.0.1 us-ae.2miners.com +127.0.0.1 us-beam.2miners.com +127.0.0.1 us-btg.2miners.com +127.0.0.1 us-central.2acoin.org +127.0.0.1 us-central01.miningrigrentals.com +127.0.0.1 us-ckb.2miners.com +127.0.0.1 us-east-stratum.grinmint.com +127.0.0.1 us-east.crazypool.org +127.0.0.1 us-east.ethash-hub.miningpoolhub.com +127.0.0.1 us-east.ezil.me +127.0.0.1 us-east.kawpow-hub.miningpoolhub.com +127.0.0.1 us-east.randomx-hub.miningpoolhub.com +127.0.0.1 us-east.siamining.com +127.0.0.1 us-east.stratum.grin-pool.org +127.0.0.1 us-east.stratum.slushpool.com +127.0.0.1 us-east01.miningrigrentals.com +127.0.0.1 us-etc.2miners.com +127.0.0.1 us-grin.2miners.com +127.0.0.1 us-nw01.miningrigrentals.com +127.0.0.1 us-rvn.2miners.com +127.0.0.1 us-solo-ae.2miners.com +127.0.0.1 us-solo-beam.2miners.com +127.0.0.1 us-solo-btg.2miners.com +127.0.0.1 us-solo-etc.2miners.com +127.0.0.1 us-solo-grin.2miners.com +127.0.0.1 us-solo-rvn.2miners.com +127.0.0.1 us-solo-xzc.2miners.com +127.0.0.1 us-solo-zec.2miners.com +127.0.0.1 us-solo-zen.2miners.com +127.0.0.1 us-west.crazypool.org +127.0.0.1 us-west.ezil.me +127.0.0.1 us-west.siamining.com +127.0.0.1 us-west01.miningrigrentals.com +127.0.0.1 us-xzc.2miners.com +127.0.0.1 us-zec.2miners.com +127.0.0.1 us-zel.zellabs.net +127.0.0.1 us-zen.2miners.com +127.0.0.1 us-zil.rustpool.xyz +127.0.0.1 us.0769.it +127.0.0.1 us.bsod.pw +127.0.0.1 us.btgpool.pro +127.0.0.1 us.bytecoin-pool.org +127.0.0.1 us.emcd.io +127.0.0.1 us.equihub.pro +127.0.0.1 us.fastpool.xyz +127.0.0.1 us.frostypool.com +127.0.0.1 us.litecoinpool.org +127.0.0.1 us.miner.tokyo +127.0.0.1 us.mining.bit.tube +127.0.0.1 us.miningfield.com +127.0.0.1 us.nimpool.io +127.0.0.1 us.pool.ms +127.0.0.1 us.ravenminer.com +127.0.0.1 us.rig.tokyo +127.0.0.1 us.rito.minermore.com +127.0.0.1 us.rvn.minermore.com +127.0.0.1 us.ss.btc.com +127.0.0.1 us.stratu.ms +127.0.0.1 us.ubiqpool.io +127.0.0.1 us1-beam.flypool.org +127.0.0.1 us1-zil.shardpool.io +127.0.0.1 us1.fairpool.pro +127.0.0.1 us1.hellominer.com +127.0.0.1 us1.stratu.ms +127.0.0.1 us1.zhash.pro +127.0.0.1 us2-zil.rustpool.xyz +127.0.0.1 us2.litecoinpool.org +127.0.0.1 us2mine.com +127.0.0.1 usa-west.dgb256.online +127.0.0.1 uspool.electroneropulse.org +127.0.0.1 vdl.raptorpool.org +127.0.0.1 vegas-backup.xmrpool.net +127.0.0.1 veil-eu.coinblockers.com +127.0.0.1 veil-us.coinblockers.com +127.0.0.1 veil.suprnova.cc +127.0.0.1 vertcoin.hashalot.net +127.0.0.1 verushash.asia.mine.zergpool.com +127.0.0.1 verushash.eu.mine.zergpool.com +127.0.0.1 verushash.mine.zergpool.com +127.0.0.1 verushash.na.mine.zergpool.com +127.0.0.1 viabtc.com +127.0.0.1 viabtc.net +127.0.0.1 vidulum.dapool.io +127.0.0.1 vrsc.ginasismining.com +127.0.0.1 vtc.suprnova.cc +127.0.0.1 vvpool.com +127.0.0.1 webmng.semipool.com +127.0.0.1 webwxtc.semipool.com +127.0.0.1 west.us.rvn.minermore.com +127.0.0.1 wownero.ingest.cryptoknight.cc +127.0.0.1 www.eelpool.com +127.0.0.1 www.mineit.io +127.0.0.1 x11.asia.mine.zergpool.com +127.0.0.1 x11.br.nicehash.com +127.0.0.1 x11.eu.mine.zergpool.com +127.0.0.1 x11.eu.mine.zpool.ca +127.0.0.1 x11.eu.nicehash.com +127.0.0.1 x11.hk.nicehash.com +127.0.0.1 x11.in.nicehash.com +127.0.0.1 x11.jp.mine.zpool.ca +127.0.0.1 x11.jp.nicehash.com +127.0.0.1 x11.mine.blazepool.com +127.0.0.1 x11.mine.zergpool.com +127.0.0.1 x11.mine.zpool.ca +127.0.0.1 x11.mining-dutch.nl +127.0.0.1 x11.na.mine.zpool.ca +127.0.0.1 x11.sea.mine.zpool.ca +127.0.0.1 x11.usa.nicehash.com +127.0.0.1 x11evo.eu.mine.zpool.ca +127.0.0.1 x11evo.jp.mine.zpool.ca +127.0.0.1 x11evo.na.mine.zpool.ca +127.0.0.1 x11gost.mining-dutch.nl +127.0.0.1 x13.asia.mine.zergpool.com +127.0.0.1 x13.br.nicehash.com +127.0.0.1 x13.eu.mine.zpool.ca +127.0.0.1 x13.eu.nicehash.com +127.0.0.1 x13.hk.nicehash.com +127.0.0.1 x13.in.nicehash.com +127.0.0.1 x13.jp.mine.zpool.ca +127.0.0.1 x13.jp.nicehash.com +127.0.0.1 x13.mine.zergpool.com +127.0.0.1 x13.na.mine.zpool.ca +127.0.0.1 x13.usa.nicehash.com +127.0.0.1 x15.eu.mine.zpool.ca +127.0.0.1 x15.jp.mine.zpool.ca +127.0.0.1 x15.na.mine.zpool.ca +127.0.0.1 x16r.asia.mine.zergpool.com +127.0.0.1 x16r.br.nicehash.com +127.0.0.1 x16r.eu.mine.zergpool.com +127.0.0.1 x16r.eu.mine.zpool.ca +127.0.0.1 x16r.eu.nicehash.com +127.0.0.1 x16r.hk.nicehash.com +127.0.0.1 x16r.in.nicehash.com +127.0.0.1 x16r.jp.mine.zpool.ca +127.0.0.1 x16r.jp.nicehash.com +127.0.0.1 x16r.mine.blazepool.com +127.0.0.1 x16r.mine.zergpool.com +127.0.0.1 x16r.na.mine.zergpool.com +127.0.0.1 x16r.na.mine.zpool.ca +127.0.0.1 x16r.sea.mine.zpool.ca +127.0.0.1 x16r.usa.nicehash.com +127.0.0.1 x16rt.asia.mine.zergpool.com +127.0.0.1 x16rt.eu.mine.zpool.ca +127.0.0.1 x16rt.mine.zergpool.com +127.0.0.1 x16rt.na.mine.zpool.ca +127.0.0.1 x16rv2.asia.mine.zergpool.com +127.0.0.1 x16rv2.br.nicehash.com +127.0.0.1 x16rv2.eu.mine.zpool.ca +127.0.0.1 x16rv2.eu.nicehash.com +127.0.0.1 x16rv2.hk.nicehash.com +127.0.0.1 x16rv2.in.nicehash.com +127.0.0.1 x16rv2.jp.mine.zpool.ca +127.0.0.1 x16rv2.jp.nicehash.com +127.0.0.1 x16rv2.mine.zergpool.com +127.0.0.1 x16rv2.na.mine.zpool.ca +127.0.0.1 x16rv2.sea.mine.zpool.ca +127.0.0.1 x16rv2.tron-mining.com +127.0.0.1 x16rv2.usa.nicehash.com +127.0.0.1 x16s.asia.mine.zergpool.com +127.0.0.1 x16s.eu.mine.zpool.ca +127.0.0.1 x16s.jp.mine.zpool.ca +127.0.0.1 x16s.mine.zergpool.com +127.0.0.1 x16s.na.mine.zpool.ca +127.0.0.1 x16s.sea.mine.zpool.ca +127.0.0.1 x17.asia.mine.zergpool.com +127.0.0.1 x17.eu.mine.zpool.ca +127.0.0.1 x17.jp.mine.zpool.ca +127.0.0.1 x17.mine.zergpool.com +127.0.0.1 x17.na.mine.zpool.ca +127.0.0.1 x21s.asia.mine.zergpool.com +127.0.0.1 x21s.eu.mine.zpool.ca +127.0.0.1 x21s.mine.zergpool.com +127.0.0.1 x21s.na.mine.zpool.ca +127.0.0.1 x22i.eu.mine.zpool.ca +127.0.0.1 x22i.jp.mine.zpool.ca +127.0.0.1 x22i.mine.zergpool.com +127.0.0.1 x22i.na.mine.zpool.ca +127.0.0.1 x25x.asia.mine.zergpool.com +127.0.0.1 x25x.eu.mine.zergpool.com +127.0.0.1 x25x.eu.mine.zpool.ca +127.0.0.1 x25x.mine.zergpool.com +127.0.0.1 x25x.mine.zpool.ca +127.0.0.1 x25x.na.mine.zpool.ca +127.0.0.1 xao.pool.mine2gether.com +127.0.0.1 xcash.steadyhash.org +127.0.0.1 xd.frostypool.com +127.0.0.1 xdna.suprnova.cc +127.0.0.1 xeq.pool-pay.com +127.0.0.1 xerom.weeminepool.com +127.0.0.1 xevan.asia.mine.zergpool.com +127.0.0.1 xevan.eu.mine.zergpool.com +127.0.0.1 xevan.eu.mine.zpool.ca +127.0.0.1 xevan.jp.mine.zpool.ca +127.0.0.1 xevan.mine.blazepool.com +127.0.0.1 xevan.mine.zergpool.com +127.0.0.1 xevan.mine.zpool.ca +127.0.0.1 xevan.na.mine.zpool.ca +127.0.0.1 xhvpool.dlinodes.com +127.0.0.1 xmc.pool.minergate.com +127.0.0.1 xmr.2miners.com +127.0.0.1 xmr.antpool.com +127.0.0.1 xmr.pool.gntl.co.uk +127.0.0.1 xmr.pool.minergate.com +127.0.0.1 xmr.ss.dxpool.com +127.0.0.1 xmrpool.eu +127.0.0.1 xserty.com +127.0.0.1 xsg.altpool.pro +127.0.0.1 xta.pool-pay.com +127.0.0.1 xtri.minercountry.com +127.0.0.1 xzc.2miners.com +127.0.0.1 yescrypt.asia.mine.zergpool.com +127.0.0.1 yescrypt.eu.mine.zergpool.com +127.0.0.1 yescrypt.eu.mine.zpool.ca +127.0.0.1 yescrypt.jp.mine.zpool.ca +127.0.0.1 yescrypt.mine.zergpool.com +127.0.0.1 yescrypt.mine.zpool.ca +127.0.0.1 yescrypt.mining-dutch.nl +127.0.0.1 yescrypt.na.mine.zpool.ca +127.0.0.1 yescrypt.sea.mine.zpool.ca +127.0.0.1 yescryptr16.asia.mine.zergpool.com +127.0.0.1 yescryptr16.eu.mine.zergpool.com +127.0.0.1 yescryptr16.eu.mine.zpool.ca +127.0.0.1 yescryptr16.jp.mine.zpool.ca +127.0.0.1 yescryptr16.mine.zergpool.com +127.0.0.1 yescryptr16.mine.zpool.ca +127.0.0.1 yescryptr16.mining-dutch.nl +127.0.0.1 yescryptr16.na.mine.zpool.ca +127.0.0.1 yescryptr32.asia.mine.zergpool.com +127.0.0.1 yescryptr32.eu.mine.zergpool.com +127.0.0.1 yescryptr32.eu.mine.zpool.ca +127.0.0.1 yescryptr32.jp.mine.zpool.ca +127.0.0.1 yescryptr32.mine.zergpool.com +127.0.0.1 yescryptr32.mining-dutch.nl +127.0.0.1 yescryptr32.na.mine.zpool.ca +127.0.0.1 yescryptr8.eu.mine.zpool.ca +127.0.0.1 yescryptr8.jp.mine.zpool.ca +127.0.0.1 yescryptr8.na.mine.zpool.ca +127.0.0.1 yespower.asia.mine.zergpool.com +127.0.0.1 yespower.eu.mine.zergpool.com +127.0.0.1 yespower.eu.mine.zpool.ca +127.0.0.1 yespower.jp.mine.zpool.ca +127.0.0.1 yespower.mine.zergpool.com +127.0.0.1 yespower.mining-dutch.nl +127.0.0.1 yespower.na.mine.zpool.ca +127.0.0.1 yespowerr16.asia.mine.zergpool.com +127.0.0.1 yespowerr16.eu.mine.zergpool.com +127.0.0.1 yespowerr16.eu.mine.zpool.ca +127.0.0.1 yespowerr16.mine.zergpool.com +127.0.0.1 yespowerr16.mining-dutch.nl +127.0.0.1 yiimp.ccminer.org +127.0.0.1 yoc.2miners.com +127.0.0.1 zano.luckypool.io +127.0.0.1 zcl.2miners.com +127.0.0.1 zcl.altpool.pro +127.0.0.1 zcl.suprnova.cc +127.0.0.1 zcoin-eu.mintpond.com +127.0.0.1 zcoin-us.mintpond.com +127.0.0.1 zcoin.dapool.io +127.0.0.1 zcoin.mintpond.com +127.0.0.1 zdash.suprnova.cc +127.0.0.1 zec-eu.luxor.tech +127.0.0.1 zec-us.luxor.tech +127.0.0.1 zec.2miners.com +127.0.0.1 zec.altpool.pro +127.0.0.1 zec.f2pool.com +127.0.0.1 zec.luxor.tech +127.0.0.1 zec.ss.poolin.com +127.0.0.1 zel-eu.coinblockers.com +127.0.0.1 zel.2miners.com +127.0.0.1 zelsolo-us.coinblockers.com +127.0.0.1 zen-eu.luxor.tech +127.0.0.1 zen-us.luxor.tech +127.0.0.1 zen.2miners.com +127.0.0.1 zen.luxor.tech +127.0.0.1 zencash.f2pool.com +127.0.0.1 zer.equihub.pro +127.0.0.1 zergpool.com +127.0.0.1 zero.suprnova.cc +127.0.0.1 zeropool.io +127.0.0.1 zhash.br.nicehash.com +127.0.0.1 zhash.eu.nicehash.com +127.0.0.1 zhash.hk.nicehash.com +127.0.0.1 zhash.in.nicehash.com +127.0.0.1 zhash.jp.nicehash.com +127.0.0.1 zhash.usa.nicehash.com +127.0.0.1 zil.rustpool.xyz +127.0.0.1 zp-eu.leafpool.com +127.0.0.1 zpool.ca \ No newline at end of file diff --git a/ansible/roles/block-ips/tasks/main.yml b/ansible/roles/block-ips/tasks/main.yml index 55b9d71..080238b 100644 --- a/ansible/roles/block-ips/tasks/main.yml +++ b/ansible/roles/block-ips/tasks/main.yml @@ -1,22 +1,5 @@ --- -- name: Template bash script (block-outbound) - ansible.builtin.template: - src: block-outbound.sh.j2 - dest: /usr/local/bin/block-outbound.sh - owner: root - group: root - mode: '0755' - -- name: Add block-outbound service - ansible.builtin.template: - src: block-outbound.service.j2 - dest: /etc/systemd/system/block-outbound.service - owner: root - group: root - mode: '0644' - -- name: Enable block-outbound service - ansible.builtin.service: - name: block-outbound - state: reloaded - enabled: true +- name: Block hosts + ansible.builtin.blockinfile: + block: "{{ lookup('ansible.builtin.file', 'hosts-to-block') }}" + dest: /etc/hosts diff --git a/ansible/roles/block-ips/templates/block-outbound.service b/ansible/roles/block-ips/templates/block-outbound.service deleted file mode 100644 index 95231a1..0000000 --- a/ansible/roles/block-ips/templates/block-outbound.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Script that fetches IP lists from URLs and blocks outbound packets to them -After=network.target - -[Service] -UMask=022 -Type=simple -User=root -Group=root -ExecStart=/usr/local/bin/block-outbound.sh -MemoryLimit=1G -Restart=no - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/roles/block-ips/templates/block-outbound.sh.j2 b/ansible/roles/block-ips/templates/block-outbound.sh.j2 deleted file mode 100644 index b54b340..0000000 --- a/ansible/roles/block-ips/templates/block-outbound.sh.j2 +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/sh - -{% for url in block_ips_urls %} -curl -sL '{url}' \ -| awk '!/^\s*#/ {print $1}' \ -| xargs -n1 -P10 -i firewall-cmd --direct --add-rule ipv4 filter OUTPUT 0 -d "{}" -j DROP -{% endfor %}