diff --git a/.gitignore b/.gitignore index 02c5e8d..fd51a5d 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ orlo/_version.py .vagrant *.swp *.deb + +# OSX +.DS_Store diff --git a/Vagrantfile b/Vagrantfile index 10a00ad..73ec6fa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,7 +12,7 @@ Vagrant.configure(2) do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://atlas.hashicorp.com/search. - config.vm.box = "ubuntu/trusty64" + config.vm.box = "ubuntu/xenial64" # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs @@ -38,6 +38,7 @@ Vagrant.configure(2) do |config| # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder ".", "/vagrant", type: "virtualbox" # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. @@ -70,6 +71,9 @@ Vagrant.configure(2) do |config| # sudo apt-get install -y apache2 # SHELL config.vm.provision "shell", inline: <<-SHELL + # ubuntu 16.04 fix see https://github.com/mitchellh/vagrant/issues/7288 + echo 127.0.0.1 `hostname` |sudo tee -a /etc/hosts + # sudo sed -i 's/archive.ubuntu.com/nl.archive.ubuntu.com/g' /etc/apt/sources.list apt-get update apt-get -y install python-pip python-dev postgresql postgresql-server-dev-all @@ -80,32 +84,31 @@ Vagrant.configure(2) do |config| apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev # Build tools - apt-get -y install build-essential git-buildpackage debhelper python-dev dh-systemd python-virtualenv + apt-get -y install build-essential git-buildpackage debhelper python-dev \ + python3-dev dh-systemd python-virtualenv wget -P /tmp/ \ 'https://launchpad.net/ubuntu/+archive/primary/+files/dh-virtualenv_0.11-1_all.deb' dpkg -i /tmp/dh-virtualenv_0.11-1_all.deb apt-get -f install -y - pip install --upgrade pip - pip install virtualenv + pip install --upgrade pip setuptools + pip install --upgrade virtualenv # Virtualenv is to avoid conflict with Debian's python-six - virtualenv /home/vagrant/virtualenv/orlo - source /home/vagrant/virtualenv/orlo/bin/activate - echo "source ~/virtualenv/orlo/bin/activate" >> /home/vagrant/.profile + virtualenv /home/ubuntu/virtualenv/orlo + source /home/ubuntu/virtualenv/orlo/bin/activate + echo "source ~/virtualenv/orlo/bin/activate" >> /home/ubuntu/.profile pip install -r /vagrant/requirements.txt pip install -r /vagrant/requirements_testing.txt pip install -r /vagrant/docs/requirements.txt - # Flask-Testing hasn't been released to pip in ages :( - pip install --upgrade git+https://github.com/jarus/flask-testing.git - - sudo chown -R vagrant:vagrant /home/vagrant/virtualenv + sudo chown -R ubuntu:ubuntu /home/ubuntu/virtualenv # Create the database python /vagrant/create_db.py + python /vagrant/setup.py develop mkdir /etc/orlo - chown vagrant:root /etc/orlo + chown ubuntu:root /etc/orlo SHELL end diff --git a/bin/orlo b/debian/bin/orlo similarity index 100% rename from bin/orlo rename to debian/bin/orlo diff --git a/debian/changelog b/debian/changelog index 13cbd14..7e67d69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,101 @@ +orlo (0.2.0.pre5) UNRELEASED; urgency=medium + + [ Alex Forbes ] + * Add dependant python virtualenv package to Vagrantfile + * Add outline for Deploy process + * Use release object instead of re-declaring, add live server test + + [ Ivan Coppa ] + * Support for Release metadata attributes, Deploy endpoint + * disabling debug + + [ Alex Forbes ] + * Add integrated dummy deployer test + * Use relative path for deployer.rb + + [ Marcel Kuiper ] + * Added POC for Token based AUthentication + + [ Ivan Coppa ] + * enable deploy api call + + [ Marcel Kuiper ] + * processed flake8 recommendations on user_auth.py + + [ ivancoppa ] + * create base data + + [ Alex Forbes ] + * Add last minute changes to create script + * Add last minute hacks to make everything work + * Add orlo_release to deployer env + * Create configuration for auth + + * Make URL a parameter for auth tests + * Only apply login_required when security is enabled + * Rename test classes to start with "Test" + * Move auth config into setup/teardown + * Add root page, test fixes + * Skip deploy test for now + * Move route_api to route_releases, rename metadatas => metadata + * Make token lifetime configurable, update orlo.ini with new options + * Bump version of Flask-HTTPAuth + * Add status code to OrloAuthError and add OrloConfigError + * Remove login_handler in favour or separate TokenAuth module + * Seperate token auth from http basic auth + * Only require auth when configured + * Implement deployer.py test deployer script + * Bump orloclient version, use py.test for travis + * Add pytz to requirements.txt + * Add requirements_testing.txt + * Move pytest to requirements_testing + * Use /bin/true in deploy shell test + + [ Dustin Nguyen ] + * Added ldap auth + * Added ldap mock test + + [ Alex Forbes ] + * Misc changes to deploy + * Tidy up example deployer + * Bump to pre-release 0.2 + * Tidy comments in example deployer + * Use absolute path for version file + * Use orlo.cli as entry point instead of custom script + * Update Vagrantfile + * Fix silly bug, read config from file after set + * Move debian packaging files in to debian dir + * Explicitly set synced_folder to virtualbox provider + * [lintian] fix description + * Add /version url and test + * Add --version option to command line + * Handle getting an invalid release id gracefully + * Reduce test verbosity by disabling debug and removing prints + * [debian] Add prerm script to stop service + * Separate post_releases_start and post_releases_deploy + * Log output of deploy + * Use json.dumps instead of str() for reliable conversion + * Return output of deploy to client + * Add .DS_Store to gitignore + * Add payload of arguments to error + * Add configuration to documentation + * Break out ExecStart command into multiple lines for clarity + * Add note to test_auth file + * Make config file configurable by environment variable + * Include logger name in logs + * Update requirements for py3, use >= where possible + * Remove indirect dependencies + * Update Vagrantfile to Ubuntu Xenial + * Remove indirect imports + * [py3] fix import errors + * Fix tests to run under python3 + * Fix for python2 again + * Add own fork of flask-testing + * Fix /versions sometimes returning wrong version + + [ Ubuntu ] + + -- Alex Forbes Thu, 14 Apr 2016 16:17:34 +0000 + orlo (0.1.1) stable; urgency=medium * Cast package_rollback as a boolean diff --git a/debian/control b/debian/control index b1dac0a..61e782a 100644 --- a/debian/control +++ b/debian/control @@ -6,8 +6,10 @@ Build-Depends: debhelper (>= 9), python, dh-virtualenv, gcc, python-dev, postgre Standards-Version: 3.9.5 Package: orlo -Architecture: all -Depends: ${python:Depends}, ${misc:Depends} -Description: An API for tracking deployments, written with Python, Flask and SqlAlchemy. +Architecture: amd64 +Depends: ${misc:Depends} +Description: API for tracking deployments + Written with Python, Flask and SQLAlchemy + See http://orlo.readthedocs.org/en/latest/ diff --git a/debian/install b/debian/install index f52720d..9852a15 100644 --- a/debian/install +++ b/debian/install @@ -1,3 +1,3 @@ -bin/orlo usr/bin +debian/bin/orlo usr/bin etc/orlo.ini /etc/orlo -systemd/orlo.service /lib/systemd/system +debian/systemd/orlo.service /lib/systemd/system diff --git a/debian/orlo.lintian-overrides b/debian/orlo.lintian-overrides new file mode 100644 index 0000000..9855ebc --- /dev/null +++ b/debian/orlo.lintian-overrides @@ -0,0 +1,19 @@ +# Obviously, many of these are legitimate packaging problems and should be resolved. +# Most though, are by virtue of dh-virtualenv +# +orlo binary: wrong-path-for-interpreter +# Python is packaged in: +orlo binary: python-script-but-no-python-dep +# Several pip packages cause this: +orlo binary: executable-not-elf-or-script +orlo binary: souce-is-missing docs/ +# Werkzeug, sphinx: +orlo binary: embedded-javascript-library +orlo binary: package-installs-python-egg usr/share/python/orlo/lib/python2.7/site-packages/flask/testsuite/test_apps/lib/python2.5/site-packages/SiteEgg.egg + +# Binary files within the virtualenv: +orlo binary: arch-dependent-file-in-usr-share usr/share/python/orlo/bin/python +orlo binary: arch-dependent-file-in-usr-share usr/share/python/orlo/lib/python2.7/site-packages/_ldap.so + +# Can't be overridden, would be nice if dh-virtualenv handled this +orlo binary: package-installs-python-bytecode diff --git a/debian/preinst b/debian/preinst index 52c6e9f..fc56249 100644 --- a/debian/preinst +++ b/debian/preinst @@ -14,3 +14,5 @@ touch /var/log/orlo/app.log chown -R orlo:orlo /var/{lib,log}/orlo chmod 755 /var/{lib,log}/orlo chmod 664 /var/log/orlo/app.log + +#DEBHELPER# diff --git a/debian/prerm b/debian/prerm new file mode 100644 index 0000000..a6f26fc --- /dev/null +++ b/debian/prerm @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +systemctl stop orlo.service \ No newline at end of file diff --git a/debian/rules b/debian/rules index 1d28e71..d22413c 100755 --- a/debian/rules +++ b/debian/rules @@ -4,4 +4,4 @@ dh $@ --with systemd --with python-virtualenv override_dh_virtualenv: - dh_virtualenv --no-test + dh_virtualenv --no-test --python /usr/bin/python diff --git a/debian/source.lintian-overrides b/debian/source.lintian-overrides new file mode 100644 index 0000000..7ef2ea4 --- /dev/null +++ b/debian/source.lintian-overrides @@ -0,0 +1 @@ +orlo source: source-is-missing docs/* diff --git a/debian/systemd/orlo.service b/debian/systemd/orlo.service new file mode 100644 index 0000000..46860b2 --- /dev/null +++ b/debian/systemd/orlo.service @@ -0,0 +1,21 @@ +# Systemd unit file for orlo + +[Unit] +Description=orlo +After=network.target +ConditionPathExists=/usr/share/python/orlo/bin/gunicorn + +[Service] +Type=simple +User=orlo +Group=orlo +ExecStart=/usr/share/python/orlo/bin/gunicorn \ + -w 4 -b 127.0.0.1:8080 \ + --access-logfile /var/log/orlo/gunicorn-access.log \ + --log-level debug \ + --error-logfile /var/log/orlo/gunicorn-error.log \ + --log-file /var/log/orlo/gunicorn.log \ + orlo:app + +[Install] +WantedBy=multi-user.target diff --git a/docs/config.rst b/docs/config.rst new file mode 100644 index 0000000..5a12d41 --- /dev/null +++ b/docs/config.rst @@ -0,0 +1,70 @@ +Configuration +============= + +[main] +`````` + +:debug_mode: `true` or `false`. Default `false`. Enables Flask's debug mode. +:propagate_exceptions: `true` or `false`. Default `true`. Sets + 'PROPAGATE_EXCEPTIONS` in Flask. See + `Flask documentation `_ + for details. +:time_format: A `strftime `_ + string. Default `%Y-%m-%dT%H:%M:%SZ`. +:time_zone: Local time zone, as understood by pytz. Internally, + all timestamps are stored in UTC. The timestamp is interpreted by Arrow when + timestamps are given by the user (e.g. on recording a release), this + setting merely reflects what time zone is given back to GET requests. + Default `UTC`. +:strict_slashes: `true` or `false`. Default `false`. By default, Werkzeug + (what Flask uses underneath), will automatically "handle" trailing slashes, + with the result that /foo/ and /foo are the same url. This disables that + behaviour. It is recommended that you leave this set to false. See the + `Werkzeug documentation `_ + for more information. +:base_url: The external url which points to your web app. Required for + callbacks. Default `http://localhost:8080`. + +[security] +`````````` + +:enabled: `true` or `false`. Enables security. +:passwd_file: Path to a `htpasswd `_ + file to use for authentication. +:secret_key: A secret key to use for token encryption. Default `change_me`. + If security is enabled and this is still set to `change_me`, Orlo will + refuse to start. +:token_ttl: The length of times that tokens should live for in seconds. + Tokens automatically expire after this. Default `3600`. +:ldap_server: Ldap server to use for ldap requests. Default `localhost + .localdomain` +:ldap_port: Ldap port to use for ldap requests. Default `389`. +:user_base_dn: Ldap dn in which to search for users. Default `ou=people, + ou=example,ou=test` + + +[db] +```` + +:uri: Database uri for SQLAlchemy. See `Flask-SQLAlchemy `_ + docs for details. Default `postgres://orlo:password@localhost:5432/orlo` +:echo_queries: Whether or not to echo sql queries to log. Default `false`. + +[logging] +````````` + +:level: Logging level, valid values `debug`, `info`, `warning`, `error`. + Default `info`. +:file: Log file to use. If not set, logging only goes to stdout. + +[deploy] +```````` + +:timeout: How long to wait for the deploy script to complete in seconds. If it + does not complete within this time, the deploy is considered failed and + an exception is raised. Default `3600`. + +[deploy_shell] +`````````````` + +:command_path: Path to the deployment script. Defaults to Orlo's test deployer. diff --git a/docs/index.rst b/docs/index.rst index 3c3412b..3beb449 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. Orlo -====== +==== Orlo is an API for tracking deployments, written with Python, Flask and SqlAlchemy. @@ -18,11 +18,12 @@ Contents :maxdepth: 2 install + config rest About Orlo ------------- +---------- Orlo aims to cover the needs of all eCG platforms with respect to gathering information about deployments, while being simple to integrate with existing deployment software and scripts. This currently includes: @@ -39,7 +40,7 @@ With this information, it will be possible to build dashboards and more intellig The API should also be agnostic to release process, server container or packaging format - all platforms do things differently. It should be forgiving and "do the right thing" in the case of missing data, as not all platforms will use every field. Why "orlo"? -------------- +----------- Originally this project was called "Sponge", because sponges are absorbent. But it turns out that name was already in use on readthedocs.org, so it was renamed. In English, orlo means "a plinth supporting the base of a column". diff --git a/docs/install.rst b/docs/install.rst index eeade3f..6019b53 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -4,7 +4,15 @@ Installation **This is a rough draft** This documentation is in an early state and your mileage may vary, please raise issues on github for any problems you encounter. -Currently, we do not publish packages to pip, thus the only way is currently from source, but this may change when the project reaches maturity. +From pip +-------- +:: + + pip install orlo + + +The pip build is not always up to date as things are still in flux. At this early stage, it is recommended to install from source. + From source ----------- @@ -18,6 +26,14 @@ Clone the github repo: python setup.py install +Or: + +:: + + pip install https://github.com/eBayClassifiedsGroup/orlo.git + + + Developing in Vagrant --------------------- @@ -111,7 +127,7 @@ If installed from the debian package run orlo setup_database -If you are running from source the orlo script can be found in ./bin, but you may need to invoke it explicity from your python interpreter, as it is hard-coded to the dh-virtualenv path. +If you are running from source the orlo script can be found in ./bin, but you may need to invoke it explicitly from your python interpreter, as it is hard-coded to the dh-virtualenv path. Running under Gunicorn @@ -133,4 +149,4 @@ And it should return 'pong' Nginx Setup ----------- -We strongly recommend running orlo behind a proxy such as nginx. An example configuration is provided under ./etc/ +We strongly recommend running orlo behind a proxy such as nginx, with TLS if you plan to use authentication. An example configuration is provided under ./etc/ diff --git a/orlo/__init__.py b/orlo/__init__.py index 0be2fc9..49bd07a 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -1,10 +1,21 @@ +from __future__ import print_function, division, absolute_import +from __future__ import unicode_literals from flask import Flask import logging from logging.handlers import RotatingFileHandler +from logging import Formatter +import sys + +from orlo.config import config, CONFIG_FILE +from orlo.exceptions import OrloStartupError, OrloError, OrloAuthError, \ + OrloConfigError + +try: + # _version is created by setup.py + from orlo._version import __version__ +except ImportError: + __version__ = "TEST_BUILD" -from orlo.config import config -from orlo.exceptions import OrloStartupError -from orlo._version import __version__ app = Flask(__name__) @@ -17,33 +28,44 @@ if config.getboolean('db', 'echo_queries'): app.config['SQLALCHEMY_ECHO'] = True +if not config.getboolean('main', 'strict_slashes'): + app.url_map.strict_slashes = False + # Debug mode ignores all custom logging and should only be used in # local testing... if config.getboolean('main', 'debug_mode'): app.debug = True -# ...as opposed to loglevel debug, which can be used anywhere -if config.getboolean('logging', 'debug'): - app.logger.setLevel(logging.DEBUG) - -app.logger.debug('Debug enabled') +if not app.debug: + log_level = config.get('logging', 'level') + if log_level == 'debug': + app.logger.setLevel(logging.DEBUG) + elif log_level == 'info': + app.logger.setLevel(logging.INFO) + elif log_level == 'warning': + app.logger.setLevel(logging.WARNING) + elif log_level == 'error': + app.logger.setLevel(logging.ERROR) -if not config.getboolean('main', 'strict_slashes'): - app.url_map.strict_slashes = False + logfile = config.get('logging', 'file') + if logfile != 'disabled': + file_handler = RotatingFileHandler( + logfile, + maxBytes=1048576, + backupCount=1, + ) + log_format = config.get('logging', 'format') + formatter = Formatter(log_format) -logfile = config.get('logging', 'file') -if logfile != 'disabled': - handler = RotatingFileHandler( - logfile, - maxBytes=1048576, - backupCount=1, - ) - app.logger.addHandler(handler) + file_handler.setFormatter(formatter) + app.logger.addHandler(file_handler) if config.getboolean('security', 'enabled') and \ config.get('security', 'secret_key') == 'change_me': - raise OrloStartupError("Security is enabled, please configure the secret key") + raise OrloStartupError( + "Security is enabled, please configure the secret key") +app.logger.debug("Log level: {}".format(config.get('logging', 'level'))) # Must be imported last import orlo.error_handlers @@ -54,3 +76,5 @@ import orlo.route_stats import orlo.user_auth +app.logger.info("Startup completed with configuration from {}".format( + CONFIG_FILE)) diff --git a/orlo/cli.py b/orlo/cli.py index 7af90ae..723be70 100644 --- a/orlo/cli.py +++ b/orlo/cli.py @@ -1,6 +1,13 @@ from __future__ import print_function import argparse +try: + from orlo import __version__ +except ImportError: + # _version.py doesn't exist + __version__ = "TEST_BUILD" + + __author__ = 'alforbes' """ @@ -11,10 +18,14 @@ def parse_args(): - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(prog='orlo') + + parser.add_argument('--version', '-v', action='version', + version='%(prog)s {}'.format(__version__)) p_config = argparse.ArgumentParser(add_help=False) - p_config.add_argument('--file', '-f', dest='filepath', help="File to write to", + p_config.add_argument('--file', '-f', dest='file_path', + help="Config file to read/write", default='/etc/orlo/orlo.ini') p_database = argparse.ArgumentParser(add_help=False) @@ -32,12 +43,12 @@ def parse_args(): sp_database = subparsers.add_parser( 'setup_database', help="Initialise the configured DB", - parents=[p_database]) + parents=[p_database, p_config]) sp_database.set_defaults(func=setup_database) sp_run_server = subparsers.add_parser( 'run_server', help="Run a test server", - parents=[p_server]) + parents=[p_server, p_config]) sp_run_server.set_defaults(func=run_server) return parser.parse_args() @@ -45,7 +56,7 @@ def parse_args(): def write_config(args): from orlo import config - config_file = open(args.filepath, 'w') + config_file = open(args.file_path, 'w') config.write(config_file) diff --git a/orlo/config.py b/orlo/config.py index f90d87a..148faf1 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -1,12 +1,18 @@ from __future__ import print_function -import ConfigParser import os +from six.moves.configparser import RawConfigParser + __author__ = 'alforbes' -config = ConfigParser.ConfigParser() +try: + CONFIG_FILE = os.environ['ORLO_CONFIG'] +except KeyError: + CONFIG_FILE = '/etc/orlo/orlo.ini' + +config = RawConfigParser() config.add_section('main') -config.set('main', 'debug_mode', 'true') +config.set('main', 'debug_mode', 'false') config.set('main', 'propagate_exceptions', 'true') config.set('main', 'time_format', '%Y-%m-%dT%H:%M:%SZ') config.set('main', 'time_zone', 'UTC') @@ -15,9 +21,11 @@ config.add_section('security') config.set('security', 'enabled', 'false') -config.set('security', 'passwd_file', os.path.dirname(__file__) + '/../etc/passwd') +config.set('security', 'passwd_file', + os.path.dirname(__file__) + '/../etc/passwd') config.set('security', 'secret_key', 'change_me') -# NOTE: orlo.__init__ checks that secret_key is not "change_me" when security is enabled +# NOTE: orlo.__init__ checks that secret_key is not "change_me" when security +# is enabled # Do not change the default here without updating __init__ as well. config.set('security', 'token_ttl', '3600') config.set('security', 'ldap_server', 'localhost.localdomain') @@ -29,14 +37,19 @@ config.set('db', 'echo_queries', 'false') config.add_section('logging') -config.set('logging', 'debug', 'false') +config.set('logging', 'level', 'info') config.set('logging', 'file', 'disabled') +config.set('logging', 'format', '%(asctime)s [%(name)s] %(levelname)s %(' + 'module)s:%(funcName)s:%(lineno)d - %(' + 'message)s') config.add_section('deploy') -config.set('deploy', 'timeout', '3600') # How long to timeout external deployer calls +config.set('deploy', 'timeout', + '3600') # How long to timeout external deployer calls config.add_section('deploy_shell') -config.set('deploy_shell', 'command_path', os.path.dirname(os.path.abspath(__file__)) + +config.set('deploy_shell', 'command_path', + os.path.dirname(os.path.abspath(__file__)) + '/../deployer.py') -config.read('/etc/orlo/orlo.ini') +config.read(CONFIG_FILE) diff --git a/orlo/deploy.py b/orlo/deploy.py index 4e617f5..166f9a0 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -2,10 +2,9 @@ import json import subprocess from threading import Timer -from orlo import app from orlo.config import config -from orlo.exceptions import OrloError - +from orlo.exceptions import OrloDeployError +from orlo import app __author__ = 'alforbes' @@ -82,7 +81,7 @@ def start(self): args = [config.get('deploy_shell', 'command_path')] for p in self.release.packages: args.append("{}={}".format(p.name, p.version)) - print("Args: {}".format(str(args))) + app.logger.debug("Args: {}".format(str(args))) env = { 'ORLO_URL': self.server_url, @@ -90,17 +89,17 @@ def start(self): } for key, value in self.release.to_dict().items(): my_key = "ORLO_" + key.upper() - env[my_key] = str(value) + env[my_key] = json.dumps(value) - print("Env: {}".format(json.dumps(env))) + app.logger.debug("Env: {}".format(json.dumps(env))) metadata = {} for m in self.release.metadata: metadata.update(m.to_dict()) in_data = json.dumps(metadata) - self.run_command(args, env, in_data, - timeout_sec=config.getint('deploy', 'timeout')) + return self.run_command( + args, env, in_data, timeout_sec=config.getint('deploy', 'timeout')) def kill(self): """ @@ -117,30 +116,51 @@ def run_command(args, env, in_data, timeout_sec=3600): :param env: Dict of environment variables :param in_data: String to pass to stdin - :param args: List of arguments + :param list args: Arguments (i.e. full list including command, + as you would pass to subprocess) :param timeout_sec: Timeout in seconds, 1 hour by default :return: """ - proc = subprocess.Popen( - args, - env=env, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + try: + proc = subprocess.Popen( + args, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except OSError as e: + raise OrloDeployError( + message="OSError starting process: {}".format( + e.strerror), + payload={'arguments': args} + ) timer = Timer(timeout_sec, proc.kill) out = err = " " try: timer.start() - out, err = proc.communicate(in_data) + out, err = proc.communicate(in_data.encode('utf-8')) finally: timer.cancel() - print("Out:\n{}".format(out)) - print("Err:\n{}".format(err)) + app.logger.debug("Out:\n{}".format(out)) + app.logger.debug("Err:\n{}".format(err)) if proc.returncode is not 0: - raise OrloError("Subprocess exited with code {}".format( - proc.returncode), status_code=500) - print("end run") - + raise OrloDeployError( + message="Subprocess exited with code {}".format( + proc.returncode), + payload={ + 'stdout': out, + 'stderr': err, + }, + status_code=500) + else: + app.logger.info( + "Deploy completed successfully. Output:\n{}".format(out)) + app.logger.debug("end run") + + return { + 'stdout': out, + 'stderr': err, + } diff --git a/orlo/exceptions.py b/orlo/exceptions.py index 85c4fc2..c3fb4c2 100644 --- a/orlo/exceptions.py +++ b/orlo/exceptions.py @@ -1,4 +1,6 @@ from __future__ import print_function +import logging +import sys __author__ = 'alforbes' @@ -45,3 +47,7 @@ class OrloConfigError(Exception): def __init__(self, message): Exception.__init__(self) print("Configuration Error: " + message) + + +class OrloDeployError(OrloError): + status_code = 500 diff --git a/orlo/orm.py b/orlo/orm.py index 2e968e3..30d0702 100644 --- a/orlo/orm.py +++ b/orlo/orm.py @@ -1,5 +1,3 @@ -from __future__ import print_function, unicode_literals - from flask.ext.sqlalchemy import SQLAlchemy from sqlalchemy_utils.types.uuid import UUIDType from sqlalchemy_utils.types.arrow import ArrowType @@ -82,7 +80,7 @@ def __init__(self, platforms, user, team=None, references=None): self.start() def __str__(self): - return unicode(self.to_dict()) + return self.to_dict() def to_dict(self): time_format = config.get('main', 'time_format') @@ -92,7 +90,7 @@ def to_dict(self): metadata.update(m.to_dict()) return { - 'id': unicode(self.id), + 'id': str(self.id), 'packages': [p.to_dict() for p in self.packages], 'platforms': [platform.name for platform in self.platforms], 'references': string_to_list(self.references), @@ -161,7 +159,8 @@ def stop(self, success): :param success: Whether or not the package deploy succeeded """ if self.stime is None: - raise OrloWorkflowError("Can not stop a package which has not been started") + raise OrloWorkflowError( + "Can not stop a package which has not been started") self.ftime = arrow.now(config.get('main', 'time_zone')) td = self.ftime - self.stime @@ -175,7 +174,7 @@ def stop(self, success): def to_dict(self): time_format = config.get('main', 'time_format') return { - 'id': unicode(self.id), + 'id': str(self.id), 'name': self.name, 'version': self.version, 'stime': self.stime.strftime(config.get('main', 'time_format')) if self.stime else None, diff --git a/orlo/queries.py b/orlo/queries.py index 3b87e26..30d4261 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -1,11 +1,10 @@ from __future__ import print_function -import calendar import datetime import arrow from orlo import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage -from collections import OrderedDict +from sqlalchemy import and_, exc __author__ = 'alforbes' @@ -14,11 +13,22 @@ """ +def get_release(release_id): + """ + Fetch a single release + + :param release_id: + :return: + """ + return db.session.query(Release).filter(Release.id == release_id) + + def filter_release_status(query, status): """ Filter the given query by the given release status - Release status is special, because it's actually determined by the package status + Release status is special, because it's actually determined by the + package status :param query: Query object :param status: The status to filter on @@ -27,15 +37,16 @@ def filter_release_status(query, status): enums = Package.status.property.columns[0].type.enums if status not in enums: raise InvalidUsage("Invalid package status, {} is not in {}".format( - status, str(enums))) + status, str(enums))) if status in ["SUCCESSFUL", "NOT_STARTED"]: # ALL packages must match this status for it to apply to the release - # Query logic translates to "Releases which do not have any packages which satisfy + # Query logic translates to "Releases which do not have any packages + # which satisfy # the condition 'Package.status != status'". I.E, all match. query = query.filter( - ~Release.packages.any( - Package.status != status - )) + ~Release.packages.any( + Package.status != status + )) elif status in ["FAILED", "IN_PROGRESS"]: # ANY package can match for this status to apply to the release query = query.filter(Release.packages.any(Package.status == status)) @@ -53,15 +64,16 @@ def filter_release_rollback(query, rollback): if rollback is True: # Only count releases which have a rollback package query = query.filter( - Release.packages.any(Package.rollback == True) + Release.packages.any(Package.rollback == True) ) elif rollback is False: # Only count releases which do not have any rollback packages query = query.filter( - ~Release.packages.any(Package.rollback == True) + ~Release.packages.any(Package.rollback == True) ) else: # What the hell did you pass? - raise TypeError("Bad rollback parameter: '{}', type {}. Boolean expected.".format( + raise TypeError( + "Bad rollback parameter: '{}', type {}. Boolean expected.".format( rollback, type(rollback))) return query @@ -76,7 +88,7 @@ def apply_filters(query, args): :return: filtered query object """ - for field, value in args.iteritems(): + for field, value in args.items(): if field == 'latest': # this is not a comparison continue @@ -142,7 +154,8 @@ def apply_filters(query, args): value = arrow.get(value) # Do comparisons - app.logger.debug("Filtering: {} {} {}".format(filter_field, comparison, value)) + app.logger.debug( + "Filtering: {} {} {}".format(filter_field, comparison, value)) if comparison == '==': query = query.filter(filter_field == value) if comparison == '<': @@ -169,7 +182,8 @@ def releases(**kwargs): if any(field.startswith('package_') for field in kwargs.keys()) \ or "status" in kwargs.keys(): - # Package attributes need the join, as does status as it's really a package + # Package attributes need the join, as does status as it's really a + # package # attribute query = db.session.query(Release).join(Package) else: @@ -179,7 +193,8 @@ def releases(**kwargs): try: query = apply_filters(query, kwargs) except AttributeError as e: - raise InvalidUsage("An invalid field was specified: {}".format(e.message)) + raise InvalidUsage( + "An invalid field was specified: {}".format(e.args[0])) if desc: stime_field = Release.stime.desc @@ -227,7 +242,7 @@ def user_info(username): :return: """ query = db.session.query( - Release.user, db.func.count(Release.id)) \ + Release.user, db.func.count(Release.id)) \ .filter(Release.user == username) \ .group_by(Release.user) @@ -270,7 +285,7 @@ def team_info(team_name): :return: """ query = db.session.query( - Release.user, db.func.count(Release.id)) \ + Release.user, db.func.count(Release.id)) \ .filter(Release.team == team_name) \ .group_by(Release.team) @@ -322,7 +337,7 @@ def package_info(package_name): :return: """ query = db.session.query( - Package.name, db.func.count(Package.id)) \ + Package.name, db.func.count(Package.id)) \ .filter(Package.name == package_name) \ .group_by(Package.name) @@ -337,7 +352,8 @@ def package_list(platform=None): """ query = db.session.query(Package.name).distinct() if platform: - query = query.join(Release).filter(Release.platforms.any(Platform.name == platform)) + query = query.join(Release).filter( + Release.platforms.any(Platform.name == platform)) return query @@ -346,17 +362,20 @@ def package_versions(platform=None): """ List the current version of all packages - It is not sufficient to just return the highest version of each successful package, - as they can be rolled back, so we determine the version by last release time + It is not sufficient to just return the highest version of each successful + package, as they can be rolled back, so we determine the version by last + release time :param platform: Platform to filter on """ - # Sub query gets a list of successful packages by last successful release time + # Sub query gets a list of successful packages by last successful release + # time sub_q = db.session.query( - Package.name, db.func.max(Package.stime).label('max_stime')) \ + Package.name.label('name'), + db.func.max(Package.stime).label('max_stime')) \ .filter(Package.status == 'SUCCESSFUL') - if platform: # filter releases not on this platform + if platform: # filter by platform sub_q = sub_q \ .join(Release) \ .filter(Release.platforms.any(Platform.name == platform)) @@ -368,14 +387,15 @@ def package_versions(platform=None): q = db.session.query( Package.name, Package.version) \ - .join(sub_q, sub_q.c.max_stime == Package.stime) \ + .join(sub_q, and_(sub_q.c.max_stime == Package.stime, + sub_q.c.name == Package.name)) \ .group_by(Package.name, Package.version) return q -def count_releases(user=None, package=None, team=None, platform=None, status=None, - rollback=None, stime=None, ftime=None): +def count_releases(user=None, package=None, team=None, platform=None, + status=None, rollback=None, stime=None, ftime=None): """ Return the number of releases with the attributes specified @@ -386,25 +406,28 @@ def count_releases(user=None, package=None, team=None, platform=None, status=Non :param string platform: Filter by platform :param string stime: Filter by releases that started after :param string ftime: Filter by releases that started before - :param boolean rollback: Filter on whether or not the release contains a rollback + :param boolean rollback: Filter on whether or not the release contains a \ + rollback :return: Query - Note that rollback and status are special fields when applied to a release, as they are - Package attributes. + Note that rollback and status are special fields when applied to a + release, as they are Package attributes. - A "successful" or "in progress" release is defined as a release where all packages match the - status. Conversely, a "failed" or "not started" release is defined as a release where any - package matches. + A "successful" or "in progress" release is defined as a release where all + packages match the status. Conversely, a "failed" or "not started" release + is defined as a release where any package matches. - For rollbacks, if any package is a rollback the release is included, otherwise if all - packages are not rollbacks the release obviously isn't either. + For rollbacks, if any package is a rollback the release is included, + otherwise if all packages are not rollbacks the release obviously isn't + either. - Implication of this is that a release can be both "failed" and "in progress". + Implication of this is that a release can be both "failed" and "in + progress". """ args = { - 'user': user, 'package': package, 'team': team, 'platform': platform, 'status': status, - 'rollback': rollback, 'stime': stime, 'ftime': ftime, + 'user': user, 'package': package, 'team': team, 'platform': platform, + 'status': status, 'rollback': rollback, 'stime': stime, 'ftime': ftime, } app.logger.debug("Entered count_releases with args: {}".format(str(args))) @@ -432,7 +455,8 @@ def count_releases(user=None, package=None, team=None, platform=None, status=Non return query -def count_packages(user=None, team=None, platform=None, status=None, rollback=None): +def count_packages(user=None, team=None, platform=None, status=None, + rollback=None): """ Return the number of packages with the attributes specified @@ -468,7 +492,7 @@ def platform_summary(): """ query = db.session.query( - Platform.name, db.func.count(Platform.id)) \ + Platform.name, db.func.count(Platform.id)) \ .join(release_platform) \ .group_by(Platform.name) @@ -484,7 +508,7 @@ def platform_info(platform_name): """ query = db.session.query( - Platform.name, db.func.count(Platform.id)) \ + Platform.name, db.func.count(Platform.id)) \ .filter(Platform.name == platform_name) \ .group_by(Platform.name) @@ -502,5 +526,3 @@ def platform_list(): .join(release_platform) return query - - diff --git a/orlo/route_base.py b/orlo/route_base.py index 314a5ef..22fa32a 100644 --- a/orlo/route_base.py +++ b/orlo/route_base.py @@ -1,6 +1,7 @@ from __future__ import print_function from flask import jsonify, request from orlo import app +from orlo import __version__ __author__ = 'alforbes' @@ -31,6 +32,17 @@ def root(): }) +@app.route('/version', methods=['GET']) +def version(): + """ + Display version information + :return: + """ + return jsonify({ + 'version': str(__version__) + }) + + @app.route('/ping', methods=['GET']) def ping(): """ diff --git a/orlo/route_import.py b/orlo/route_import.py index fd8838a..360fee4 100644 --- a/orlo/route_import.py +++ b/orlo/route_import.py @@ -1,4 +1,3 @@ -from __future__ import print_function import arrow import json from flask import jsonify, request @@ -146,4 +145,4 @@ def post_import(): releases.append(release.id) - return jsonify({'releases': [unicode(x) for x in releases]}), 200 + return jsonify({'releases': [str(x) for x in releases]}), 200 diff --git a/orlo/route_info.py b/orlo/route_info.py index 97647a4..8086e39 100644 --- a/orlo/route_info.py +++ b/orlo/route_info.py @@ -84,15 +84,16 @@ def info_packages(package=None): platform = request.args.get('platform') if package: - packages = queries.package_info(package) + packages = queries.package_info(package).all() else: - packages = queries.package_summary(platform=platform) + packages = queries.package_summary(platform=platform).all() d = {} for package, count in packages: d[package] = {'releases': count} - return jsonify(packages), 200 + + return jsonify(d), 200 @app.route('/info/packages/list', methods=['GET']) diff --git a/orlo/route_releases.py b/orlo/route_releases.py index 9062c6b..4d79c29 100644 --- a/orlo/route_releases.py +++ b/orlo/route_releases.py @@ -2,10 +2,11 @@ from orlo import app, queries, config from orlo.exceptions import InvalidUsage from orlo.user_auth import token_auth -from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, ReleaseMetadata, Platform -from orlo.util import validate_request_json, create_release, validate_release_input, \ - validate_package_input, fetch_release, create_package, fetch_package, stream_json_list, \ - str_to_bool +from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, \ + ReleaseMetadata, Platform +from orlo.util import validate_request_json, create_release, \ + validate_release_input, validate_package_input, fetch_release, \ + create_package, fetch_package, stream_json_list, str_to_bool, is_uuid from orlo.deploy import ShellDeploy from orlo.user_auth import conditional_auth @@ -34,8 +35,8 @@ def post_releases(): curl -H "Content-Type: application/json" \\ -X POST \\ http://127.0.0.1/releases \\ - -d '{"note": "blah", "platforms": ["site1"], "references": ["ticket"], "team": "A-Team", - "user": "aforbes"}' + -d '{"note": "blah", "platforms": ["site1"], "references": ["ticket"], + "team": "A-Team", "user": "aforbes"}' """ validate_release_input(request) release = create_release(request) @@ -51,7 +52,8 @@ def post_releases(): app.logger.info( 'Create release {}, references: {}, platforms: {}'.format( - release.id, release.notes, release.references, release.platforms, release.metadata) + release.id, release.notes, release.references, release.platforms, + release.metadata) ) release.start() @@ -81,8 +83,7 @@ def post_packages(release_id): .. sourcecode:: shell curl -H "Content-Type: application/json" \\ - -X POST \\ - http://127.0.0.1/releases/${RELEASE_ID}/packages \\ + -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages \\ -d '{"name": "test-package", "version": "1.0.1"}' """ validate_package_input(request, release_id) @@ -91,9 +92,9 @@ def post_packages(release_id): package = create_package(release.id, request) app.logger.info( - 'Create package {}, release {}, name {}, version {}'.format( - package.id, release.id, request.json['name'], - request.json['version'])) + 'Create package {}, release {}, name {}, version {}'.format( + package.id, release.id, request.json['name'], + request.json['version'])) db.session.add(package) db.session.commit() @@ -115,7 +116,7 @@ def post_results(release_id, package_id): """ results = PackageResult(package_id, str(request.json)) app.logger.info("Post results, release {}, package {}".format( - release_id, package_id)) + release_id, package_id)) db.session.add(results) db.session.commit() return '', 204 @@ -123,11 +124,9 @@ def post_results(release_id, package_id): @app.route('/releases//deploy', methods=['POST']) @conditional_auth(token_auth.token_required) -def post_releases_start(release_id): +def post_releases_deploy(release_id): """ - Indicate that a release is starting - - This trigger the start of the deploy + Deploy a Release :param string release_id: Release UUID @@ -139,14 +138,33 @@ def post_releases_start(release_id): -X POST http://127.0.0.1/releases/${RELEASE_ID}/deploy """ release = fetch_release(release_id) - # TODO call deploy Class start Method app.logger.info("Release start, release {}".format(release_id)) + release.start() db.session.add(release) db.session.commit() + # TODO call deploy Class start Method, i.e. pure python rather than shell deploy = ShellDeploy(release) - deploy.start() + output = deploy.start() + return jsonify(output), 200 + + +@app.route('/releases//start', methods=['POST']) +@conditional_auth(token_auth.token_required) +def post_releases_start(release_id): + """ + Indicate that a release is starting + + :param release_id: + :return: + """ + release = fetch_release(release_id) + app.logger.info("Release start, release {}".format(release_id)) + release.start() + + db.session.add(release) + db.session.commit() return '', 204 @@ -193,11 +211,12 @@ def post_packages_start(release_id, package_id): .. sourcecode:: shell - curl -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages/${PACKAGE_ID}/start + curl -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages/${ + PACKAGE_ID}/start """ package = fetch_package(release_id, package_id) app.logger.info("Package start, release {}, package {}".format( - release_id, package_id)) + release_id, package_id)) package.start() db.session.add(package) @@ -217,7 +236,8 @@ def post_packages_stop(release_id, package_id): .. sourcecode:: shell curl -H "Content-Type: application/json" \\ - -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages/${PACKAGE_ID}/stop \\ + -X POST \\ + http://127.0.0.1/releases/${RELEASE_ID}/packages/${PACKAGE_ID}/stop \\ -d '{"success": "true"}' :param string package_id: Package UUID @@ -228,7 +248,7 @@ def post_packages_stop(release_id, package_id): package = fetch_package(release_id, package_id) app.logger.info("Package stop, release {}, package {}, success {}".format( - release_id, package_id, success)) + release_id, package_id, success)) package.stop(success=success) db.session.add(package) @@ -271,7 +291,9 @@ def post_releases_metadata(release_id): validate_request_json(request) meta = request if not meta: - raise InvalidUsage("Must include metadata in posted document: es {\"key\" : \"value\"}") + raise InvalidUsage( + "Must include metadata in posted document: es {\"key\" : " + "\"value\"}") for key, value in request.json.items(): app.logger.info("Adding Metadata to release {}".format(release_id)) @@ -288,25 +310,34 @@ def get_releases(release_id=None): """ Return a list of releases to the client, filters optional - :param string release_id: Optionally specify a single release UUID to fetch. \ + :param string release_id: Optionally specify a single release UUID to + fetch. \ This does not disable filters. - :query int desc: Normally results are returned ordered by stime ascending, setting + :query int desc: Normally results are returned ordered by stime + ascending, setting desc to true will reverse this and sort by stime descending :query int limit: Limit the results by int :query int offset: Offset the results by int :query string package_name: Filter releases by package name :query string user: Filter releases by user the that performed the release :query string platform: Filter releases by platform - :query string stime_before: Only include releases that started before timestamp given - :query string stime_after: Only include releases that started after timestamp given - :query string ftime_before: Only include releases that finished before timestamp given - :query string ftime_after: Only include releases that finished after timestamp given + :query string stime_before: Only include releases that started before \ + timestamp given + :query string stime_after: Only include releases that started after \ + timestamp given + :query string ftime_before: Only include releases that finished before \ + timestamp given + :query string ftime_after: Only include releases that finished after \ + timestamp given :query string team: Filter releases by team - :query string status: Filter by release status. This field is calculated from the package. \ - status, see special note below. - :query int duration_lt: Only include releases that took less than (int) seconds - :query int duration_gt: Only include releases that took more than (int) seconds - :query boolean package_rollback: Filter on whether or not the releases contain a rollback + :query string status: Filter by release status. This field is calculated \ + from the package status, see special note below. + :query int duration_lt: Only include releases that took less than (int) \ + seconds + :query int duration_gt: Only include releases that took more than (int) \ + seconds + :query boolean package_rollback: Filter on whether or not the releases \ + contain a rollback :query string package_name: Filter by package name :query string package_version: Filter by package version :query int package_duration_gt: Filter by packages of duration greater than @@ -315,26 +346,32 @@ def get_releases(release_id=None): "NOT_STARTED", "IN_PROGRESS", "SUCCESSFUL", "FAILED" **Note for time arguments**: - The timestamp format you must use is specified in /etc/orlo/orlo.ini. All times are UTC. + The timestamp format you must use is specified in /etc/orlo/orlo.ini. + All times are UTC. **Note on status**: - The release status is calculated from the packages it contains. The possible values are - the same as a package. For a release to be considered "SUCCESSFUL" or "NOT_STARTED", - all packages must have this value. If any one package has the value "IN_PROGRESS" or - "FAILED", that status applies to the whole release, with "FAILED" overriding "IN_PROGRESS". + The release status is calculated from the packages it contains. The + possible values are the same as a package. For a release to be + considered "SUCCESSFUL" or "NOT_STARTED", all packages must have this + value. If any one package has the value "IN_PROGRESS" or "FAILED", + that status applies to the whole release, with "FAILED" overriding + "IN_PROGRESS". """ - booleans = ('rollback', 'package_rollback', ) + booleans = ('rollback', 'package_rollback',) - if release_id: # Simple - query = db.session.query(Release).filter(Release.id == release_id) - elif len(request.args.keys()) == 0: + if release_id: # Simple, just fetch one release + if not is_uuid(release_id): + raise InvalidUsage("Release ID given is not a valid UUID") + query = queries.get_release(release_id) + elif len([x for x in request.args.keys()]) == 0: raise InvalidUsage("Please specify a filter. See " - "http://orlo.readthedocs.org/en/latest/rest.html#get--releases for " - "more info") + "http://orlo.readthedocs.org/en/latest/rest.html" + "#get--releases for more info") else: # Bit more complex - # Flatten args, as the ImmutableDict puts some values in a list when expanded + # Flatten args, as the ImmutableDict puts some values in a list when + # expanded args = {} for k in request.args.keys(): if k in booleans: @@ -343,4 +380,9 @@ def get_releases(release_id=None): args[k] = request.args.get(k) query = queries.releases(**args) - return Response(stream_json_list('releases', query), content_type='application/json') + # Execute eagerly to avoid confusing stack traces within the Response on + # error + db.session.execute(query) + + return Response(stream_json_list('releases', query), + content_type='application/json') diff --git a/orlo/route_stats.py b/orlo/route_stats.py index 8d40230..41f4019 100644 --- a/orlo/route_stats.py +++ b/orlo/route_stats.py @@ -1,8 +1,7 @@ -from __future__ import print_function import arrow from flask import request, jsonify -import stats +from orlo import stats from orlo import app from orlo.exceptions import InvalidUsage import orlo.queries as queries diff --git a/orlo/util.py b/orlo/util.py index 7d478c9..e3421dd 100644 --- a/orlo/util.py +++ b/orlo/util.py @@ -1,19 +1,19 @@ from __future__ import print_function, unicode_literals -import arrow -import datetime from flask import json from orlo import app from orlo.orm import db, Release, Package, Platform from orlo.exceptions import InvalidUsage from sqlalchemy.orm import exc from six import string_types +import uuid __author__ = 'alforbes' def append_or_create_platforms(request_platforms): """ - Create the platforms if they don't exist, and return a list of Platform objects + Create the platforms if they don't exist, and return a list of Platform + objects :param list request_platforms: List of strings denoting platform names """ @@ -106,7 +106,8 @@ def validate_request_json(request): try: request.json except Exception: - # This is pretty ugly, but we want something more user friendly than "Bad Request" + # This is pretty ugly, but we want something more user friendly than + # "Bad Request" raise InvalidUsage("Could not parse JSON document") if not request.json: @@ -125,7 +126,8 @@ def validate_package_input(request, release_id): if not 'name' in request.json or not 'version' in request.json: raise InvalidUsage("Missing name / version in request body.") - app.logger.debug("Package request validated, release_id {}".format(release_id)) + app.logger.debug( + "Package request validated, release_id {}".format(release_id)) return True @@ -150,10 +152,12 @@ def list_to_string(array): def stream_json_list(heading, iterator): """ - A lagging generator to stream JSON so we don't have to hold everything in memory + A lagging generator to stream JSON so we don't have to hold everything in + memory - This is a little tricky, as we need to omit the last comma to make valid JSON, - thus we use a lagging generator, similar to http://stackoverflow.com/questions/1630320/ + This is a little tricky, as we need to omit the last comma to make valid + JSON, thus we use a lagging generator, similar to + http://stackoverflow.com/questions/1630320/ :param heading: The title of the set, e.g. "releases" :param iterator: Any object with __iter__(), e.g. SQLAlchemy Query @@ -162,7 +166,8 @@ def stream_json_list(heading, iterator): try: prev_release = next(iterator) # get first result except StopIteration: - # StopIteration here means the length was zero, so yield a valid releases doc and stop + # StopIteration here means the length was zero, so yield a valid + # releases doc and stop yield '{{"{}": []}}'.format(heading) raise StopIteration @@ -190,3 +195,17 @@ def str_to_bool(value): if isinstance(value, int): return True if value > 0 else False raise ValueError("Value {} can not be cast as boolean".format(value)) + + +def is_uuid(string): + """ + Test whether a string is a valid UUID + + :param string: + :return: + """ + try: + uuid.UUID(string) + except ValueError: + return False + return True diff --git a/requirements.txt b/requirements.txt index d774bb7..591387a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,14 @@ gunicorn -arrow==0.7.0 +arrow>=0.7.0 Flask>=0.10.1 Flask-SQLAlchemy>=2.0 Flask-Migrate>=1.5.1 Flask-HTTPAuth>=2.7.1 Flask-TokenAuth>=0.0.2 requests>=2.4.2 -SQLAlchemy==1.0.8 -sqlalchemy-utils==0.31.0 -Jinja2==2.7.1 -MarkupSafe==0.18 -Werkzeug==0.9.4 -itsdangerous==0.23 -passlib==1.6.1 -psycopg2==2.6.1 -python-ldap>=2.4.25 +SQLAlchemy>=1.0.8 +sqlalchemy-utils>=0.31.0 +psycopg2>=2.6.1 +pyldap>=2.4.25 pytz +six>=1.10.0 diff --git a/requirements_testing.txt b/requirements_testing.txt index 70066f0..0382455 100644 --- a/requirements_testing.txt +++ b/requirements_testing.txt @@ -1,24 +1,20 @@ Flask-HTTPAuth>=2.7.1 Flask-Migrate>=1.5.1 Flask-SQLAlchemy>=2.0 -Flask-Testing>=0.4.2 Flask-TokenAuth>=0.0.2 Flask>=0.10.1 -Jinja2==2.7.1 -MarkupSafe==0.18 -SQLAlchemy==1.0.8 -Werkzeug==0.9.4 -arrow==0.7.0 +SQLAlchemy>=1.0.8 +arrow>=0.7.0 gunicorn -itsdangerous==0.23 orloclient>=0.1.1 -passlib==1.6.1 -psycopg2==2.6.1 +passlib>=1.6.1 +psycopg2>=2.6.1 pytest pytz requests>=2.4.2 -sqlalchemy-utils==0.31.0 +sqlalchemy-utils>=0.31.0 mock==1.3.0 -mockldap==0.2.6 -python-ldap==2.4.25 -six==1.10.0 +mockldap>=0.2.6 +pyldap>=2.4.25 +six>=1.10.0 +git+https://github.com/al4/flask-testing.git diff --git a/setup.py b/setup.py index 19e1ec5..7994e6c 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre1' +VERSION = '0.2.0' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) @@ -35,6 +35,7 @@ 'arrow', 'gunicorn', 'psycopg2', + 'pyldap', 'pytz', 'sphinxcontrib-httpdomain', 'sqlalchemy-utils', diff --git a/systemd/orlo.service b/systemd/orlo.service deleted file mode 100644 index 73ee3bd..0000000 --- a/systemd/orlo.service +++ /dev/null @@ -1,15 +0,0 @@ -# Systemd unit file for orlo - -[Unit] -Description=orlo -After=network.target -ConditionPathExists=/usr/share/python/orlo/bin/gunicorn - -[Service] -Type=simple -User=orlo -Group=orlo -ExecStart=/usr/share/python/orlo/bin/gunicorn -w 4 -b 127.0.0.1:8080 orlo:app --access-logfile /var/log/orlo/gunicorn-access.log --log-level debug --error-logfile /var/log/orlo/gunicorn-error.log --log-file /var/log/orlo/gunicorn.log - -[Install] -WantedBy=multi-user.target diff --git a/tests/__init__.py b/tests/__init__.py index 5ad64c1..05b4ea8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,7 +13,7 @@ def create_app(self): app.config['SQLALCHEMY_DATABASE_URI'] = 'postgres://orlo:password@localhost:5432/orlo' # app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://' app.config['TESTING'] = True - app.config['DEBUG'] = True + app.config['DEBUG'] = False app.config['TRAP_HTTP_EXCEPTIONS'] = True app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False diff --git a/tests/test_auth.py b/tests/test_auth.py index 5570e55..ccbc606 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -33,11 +33,13 @@ def auth_required(): response.status_code = 200 return response + @orlo.app.route('/test/user') @conditional_auth(user_auth.login_required) def get_resource(): return jsonify({'data': 'Hello, %s!' % g.current_user}) + class OrloAuthTest(TestCase): """ Base test class to setup the app @@ -45,7 +47,8 @@ class OrloAuthTest(TestCase): top = ('o=test', {'o': ['test']}) example = ('ou=example,o=test', {'ou': ['example']}) people = ('ou=people,ou=example,o=test', {'ou': ['other']}) - ldapuser = ('uid=ldapuser,ou=people,ou=example,o=test', {'uid': ['ldapuser'], 'userPassword': ['ldapuserpw']}) + ldapuser = ('uid=ldapuser,ou=people,ou=example,o=test', + {'uid': ['ldapuser'], 'userPassword': ['ldapuserpw']}) # This is the content of our mock LDAP directory. It takes the form # {dn: {attr: [value, ...], ...}, ...}. directory = dict([top, example, people, ldapuser]) @@ -54,7 +57,7 @@ def create_app(self): self.app = orlo.app self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://' self.app.config['TESTING'] = True - self.app.config['DEBUG'] = True + self.app.config['DEBUG'] = False self.app.config['TRAP_HTTP_EXCEPTIONS'] = True self.app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False @@ -75,15 +78,21 @@ def setUp(self): self.mockldap.start() self.ldapobj = self.mockldap['ldap://localhost/'] self.orig_security_enabled = orlo.config.get('security', 'enabled') - self.orig_security_secret_key = orlo.config.set('security', 'secret_key') - self.orig_security_ldap_server = orlo.config.set('security', 'ldap_server') + self.orig_security_secret_key = orlo.config.set('security', + 'secret_key') + self.orig_security_ldap_server = orlo.config.set('security', + 'ldap_server') self.orig_security_ldap_port = orlo.config.set('security', 'ldap_port') - self.orig_security_user_base_dn = orlo.config.set('security', 'user_base_dn') + self.orig_security_user_base_dn = orlo.config.set('security', + 'user_base_dn') orlo.config.set('security', 'enabled', 'true') - orlo.config.set('security', 'secret_key', 'It does not matter how slowly you go so long as you do not stop') + orlo.config.set('security', 'secret_key', 'It does not matter how ' + 'slowly you go so long as ' + 'you do not stop') orlo.config.set('security', 'ldap_server', 'localhost') orlo.config.set('security', 'ldap_port', '389') - orlo.config.set('security', 'user_base_dn', 'ou=people,ou=example,o=test') + orlo.config.set('security', 'user_base_dn', + 'ou=people,ou=example,o=test') def tearDown(self): db.session.remove() @@ -94,21 +103,6 @@ def tearDown(self): orlo.config.set('security', 'secret_key', self.orig_security_secret_key) def get_with_basic_auth(self, path, username='testuser', password='blah'): - """ - Do a request with basic auth - - :param path: - :param username: - :param password: - """ - h = Headers() - h.add('Authorization', 'Basic ' + base64.b64encode( - '{u}:{p}'.format(u=username, p=password) - )) - response = Client.open(self.client, path=path, headers=h) - return response - - def get_with_ldap_auth(self, path, username='ldapuser', password='ldapuserpw'): """ Do a request with ldap auth @@ -117,9 +111,9 @@ def get_with_ldap_auth(self, path, username='ldapuser', password='ldapuserpw'): :param password: """ h = Headers() - h.add('Authorization', 'Basic ' + base64.b64encode( - '{u}:{p}'.format(u=username, p=password) - )) + s_auth = base64.b64encode('{u}:{p}'.format( + u=username, p=password).encode('utf-8')) + h.add('Authorization', 'Basic ' + s_auth.decode('utf-8')) response = Client.open(self.client, path=path, headers=h) return response @@ -145,7 +139,8 @@ def post_with_token_auth(self, path, token, data): h = Headers() h.add('X-Auth-Token', token) h.add('Content-Type', 'application/json') - response = Client.open(self.client, method='POST', data=data, path=path, headers=h) + response = Client.open(self.client, method='POST', data=data, path=path, + headers=h) return response def get_token(self): @@ -214,7 +209,7 @@ def test_with_login(self): self.assert200(response) def test_with_ldap_login(self): - response = self.get_with_ldap_auth( + response = self.get_with_basic_auth( '/test/auth_required', username='ldapuser', password='ldapuserpw' ) self.assert200(response) @@ -258,4 +253,4 @@ def test_post_releases_with_token_returns_400(self): self.URL_PATH, token=token, data={'foo': 'bar'}, ) self.assert400(response) - self.assertIn('message', response.data) + self.assertIn(b'message', response.data) diff --git a/tests/test_contract.py b/tests/test_contract.py index d5caa09..1950b94 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -181,7 +181,7 @@ def test_create_release(self): Create a release """ release_id = self._create_release() - self.assertEqual(uuid.UUID(release_id).get_version(), 4) + self.assertEqual(uuid.UUID(release_id).version, 4) def test_create_package(self): """ @@ -189,7 +189,7 @@ def test_create_package(self): """ release_id = self._create_release() package_id = self._create_package(release_id) - self.assertEqual(uuid.UUID(package_id).get_version(), 4) + self.assertEqual(uuid.UUID(package_id).version, 4) def test_add_results(self): """ @@ -357,7 +357,7 @@ def _get_releases(self, release_id=None, filters=None, expected_status=200): except AssertionError as err: print(results_response.data) raise - r_json = json.loads(results_response.data) + r_json = json.loads(results_response.data.decode('utf-8')) return r_json def test_ping(self): @@ -368,6 +368,15 @@ def test_ping(self): self.assert200(response) self.assertEqual('pong', response.json['message']) + def test_version(self): + """ + Test the version url + """ + from orlo import __version__ + response = self.client.get('/version') + self.assert200(response) + self.assertEqual(__version__, response.json['version']) + def test_get_single_release(self): """ Fetch a single release @@ -378,6 +387,13 @@ def test_get_single_release(self): self.assertEqual(1, len(r['releases'])) self.assertEqual(release_id, r['releases'][0]['id']) + def test_get_single_release_invalid(self): + """ + Test that we return 400 with an invalid release ID + """ + r = self._get_releases(release_id="not_a_valid_uuid", + expected_status=400) + def test_get_releases(self): """ Test the list of releases diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 8539147..cc23d59 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -160,3 +160,21 @@ def test_start_example_deployer(self): deploy.start() + def test_output(self): + """ + Test that we return the output of the deploy + + Not a good test, as it relies on the test-package being an argument, + and simply echoing it back. This is the "spec", but this test could + break if the arguments change. + """ + with ConfigChange('deploy', 'timeout', '3'), \ + ConfigChange('deploy_shell', 'command_path', '/bin/echo'): + deploy = ShellDeploy(self.release) + + # Override server_url, normally it is set by config: + deploy.server_url = self.get_server_url() + + output = deploy.start() + self.assertEqual(output['stdout'], b'test-package=1.2.3\n') + diff --git a/tests/test_orm.py b/tests/test_orm.py index 130ff50..01ce3b0 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -9,6 +9,7 @@ import arrow import datetime import uuid +from six import string_types __author__ = 'alforbes' @@ -148,15 +149,15 @@ def test_release_types(self): Test the types returned by Release objects are OK """ r = db.session.query(Release).first() - self.assertIs(type(r.id), uuid.UUID) + self.assertIsInstance(r.id, uuid.UUID) self.assertIs(hasattr(r.notes, '__iter__'), True) self.assertIs(hasattr(r.platforms, '__iter__'), True) - self.assertIs(type(r.references), unicode) + self.assertIsInstance(r.references, string_types) self.assertIs(type(list(r.references)), list) - self.assertIs(type(r.stime), arrow.arrow.Arrow) - self.assertIs(type(r.ftime), arrow.arrow.Arrow) - self.assertIs(type(r.duration), datetime.timedelta) - self.assertIs(type(r.team), unicode) + self.assertIsInstance(r.stime, arrow.arrow.Arrow) + self.assertIsInstance(r.ftime, arrow.arrow.Arrow) + self.assertIsInstance(r.duration, datetime.timedelta) + self.assertIsInstance(r.team, string_types) def test_package_types(self): """ @@ -164,12 +165,10 @@ def test_package_types(self): """ p = db.session.query(Package).first() x = db.session.query(Package).all() - print(x) - print(len(x)) - self.assertIs(type(p.id), uuid.UUID) - self.assertIs(type(p.name), unicode) - self.assertIs(type(p.stime), arrow.arrow.Arrow) - self.assertIs(type(p.ftime), arrow.arrow.Arrow) - self.assertIs(type(p.duration), datetime.timedelta) - self.assertIs(type(p.status), unicode) - self.assertIs(type(p.version), unicode) + self.assertIsInstance(p.id, uuid.UUID) + self.assertIsInstance(p.name, string_types) + self.assertIsInstance(p.stime, arrow.arrow.Arrow) + self.assertIsInstance(p.ftime, arrow.arrow.Arrow) + self.assertIsInstance(p.duration, datetime.timedelta) + self.assertIsInstance(p.status, string_types) + self.assertIsInstance(p.version, string_types) diff --git a/tests/test_queries.py b/tests/test_queries.py index dab958d..68b2cbc 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -6,6 +6,7 @@ import orlo.exceptions import orlo.stats from time import sleep +import sqlalchemy.orm __author__ = 'alforbes' @@ -183,6 +184,13 @@ def test_package_summary(self): self.assertIn('packageOne', packages) self.assertIn('packageTwo', packages) + def test_package_summary_returns_query(self): + rid1 = self._create_release() + self._create_package(rid1, name='packageOne') + + result = orlo.queries.package_summary() + self.assertIsInstance(result, sqlalchemy.orm.query.Query) + def test_package_summary_with_platform(self): """ Test package_summary @@ -320,20 +328,40 @@ def test_package_versions_with_rollback(self): self.assertIn(('packageOne', '1.0.1'), versions) +class TestInfo(OrloQueryTest): + """ + Test the _info functions + """ + + def _create_test_package(self): + rid = self._create_release(platforms=['platformOne']) + pid = self._create_package(rid, name='packageOne', version='1.0.1') + return pid + + def test_returns_query(self): + """ + Assert that package_info should return a query + """ + self._create_test_package() + result = orlo.queries.package_info('packageOne') + self.assertIsInstance(result, sqlalchemy.orm.query.Query) + + class TestCountReleases(OrloQueryTest): """ Parent class for testing the CountReleases function - By subclassing it and overriding ARGS, we can test different combinations of arguments - with the same test code. + By subclassing it and overriding ARGS, we can test different combinations of + arguments with the same test code. - INCLUSIVE_ARGS represents a set of arguments that will match the releases created (see - the functions in OrloQueryTest for what those are) - EXCLUSIVE_ARGS represents a set of arguments that will not match any releases created, - i.e. should return a count of zero + INCLUSIVE_ARGS represents a set of arguments that will match the releases + created (see the functions in OrloQueryTest for what those are) + EXCLUSIVE_ARGS represents a set of arguments that will not match any + releases created, i.e. should return a count of zero - This parent class has tests that should be the same result no matter what the arguments - (except the exclusive case which must always a count of zero so we define it here) + This parent class has tests that should be the same result no matter what + the arguments (except the exclusive case which must always a count of zero + so we define it here) """ INCLUSIVE_ARGS = {} # Args we are testing, result should include these diff --git a/tests/test_route_import.py b/tests/test_route_import.py index 22e1340..fc8f259 100644 --- a/tests/test_route_import.py +++ b/tests/test_route_import.py @@ -84,7 +84,7 @@ def test_import_param_user(self): """ Test import user matches """ - self.assertEqual(self.release.user, unicode(self.doc_dict[0]['user'])) + self.assertEqual(self.release.user, self.doc_dict[0]['user']) def test_import_param_package_name(self): """ diff --git a/tests/test_stats.py b/tests/test_stats.py index a774043..ac8ff8f 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -135,7 +135,6 @@ def test_package_time_month(self): year = str(arrow.utcnow().year) month = str(arrow.utcnow().month) self.assertEqual(7, result[year][month]['test-package']['normal']['successful']) - print(result) def test_package_time_week(self): """ diff --git a/tests/test_stats_empirical.py b/tests/test_stats_empirical.py index 1a03302..4225f7e 100644 --- a/tests/test_stats_empirical.py +++ b/tests/test_stats_empirical.py @@ -64,7 +64,7 @@ def setUp(self): self.create_release(False, False) db.session.commit() - print("Total releases created: {}".format(self.releases)) + # print("Total releases created: {}".format(self.releases)) def create_release(self, normal, successful, user='test_user', team='test_team', platform='test_platform'):