From f4f939fd4c28e6a31de1a551300a161ff47b354f Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 14 Apr 2016 14:50:03 +0100 Subject: [PATCH 01/36] Move debian packaging files in to debian dir --- {bin => debian/bin}/orlo | 0 debian/install | 4 ++-- {systemd => debian/systemd}/orlo.service | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {bin => debian/bin}/orlo (100%) rename {systemd => debian/systemd}/orlo.service (100%) diff --git a/bin/orlo b/debian/bin/orlo similarity index 100% rename from bin/orlo rename to debian/bin/orlo diff --git a/debian/install b/debian/install index f52720d..9852a15 100644 --- a/debian/install +++ b/debian/install @@ -1,3 +1,3 @@ -bin/orlo usr/bin +debian/bin/orlo usr/bin etc/orlo.ini /etc/orlo -systemd/orlo.service /lib/systemd/system +debian/systemd/orlo.service /lib/systemd/system diff --git a/systemd/orlo.service b/debian/systemd/orlo.service similarity index 100% rename from systemd/orlo.service rename to debian/systemd/orlo.service From 491223c13be4664abca9366d6f43567f2142cb5a Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 14 Apr 2016 15:04:11 +0100 Subject: [PATCH 02/36] Explicitly set synced_folder to virtualbox provider --- Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Vagrantfile b/Vagrantfile index 10a00ad..8dbae63 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -38,6 +38,7 @@ Vagrant.configure(2) do |config| # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder ".", "/vagrant", type: "virtualbox" # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. From 2621a23308d143e7237817a220f05c7be2101d59 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 14 Apr 2016 15:04:35 +0100 Subject: [PATCH 03/36] [lintian] fix description --- debian/control | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index b1dac0a..e46dcbb 100644 --- a/debian/control +++ b/debian/control @@ -8,6 +8,8 @@ Standards-Version: 3.9.5 Package: orlo Architecture: all Depends: ${python:Depends}, ${misc:Depends} -Description: An API for tracking deployments, written with Python, Flask and SqlAlchemy. +Description: API for tracking deployments + Written with Python, Flask and SQLAlchemy + See http://orlo.readthedocs.org/en/latest/ From 19ade955f3f994da5fcdc6f7e47319d912c1b4ac Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 14 Apr 2016 16:11:53 +0100 Subject: [PATCH 04/36] Packaging tweaks, reduce lintian output a little Build amd64 package, as the result is not _all by any stretch --- debian/control | 4 ++-- debian/orlo.lintian-overrides | 19 +++++++++++++++++++ debian/preinst | 2 ++ debian/source.lintian-overrides | 1 + 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 debian/orlo.lintian-overrides create mode 100644 debian/source.lintian-overrides diff --git a/debian/control b/debian/control index e46dcbb..61e782a 100644 --- a/debian/control +++ b/debian/control @@ -6,8 +6,8 @@ Build-Depends: debhelper (>= 9), python, dh-virtualenv, gcc, python-dev, postgre Standards-Version: 3.9.5 Package: orlo -Architecture: all -Depends: ${python:Depends}, ${misc:Depends} +Architecture: amd64 +Depends: ${misc:Depends} Description: API for tracking deployments Written with Python, Flask and SQLAlchemy See http://orlo.readthedocs.org/en/latest/ diff --git a/debian/orlo.lintian-overrides b/debian/orlo.lintian-overrides new file mode 100644 index 0000000..9855ebc --- /dev/null +++ b/debian/orlo.lintian-overrides @@ -0,0 +1,19 @@ +# Obviously, many of these are legitimate packaging problems and should be resolved. +# Most though, are by virtue of dh-virtualenv +# +orlo binary: wrong-path-for-interpreter +# Python is packaged in: +orlo binary: python-script-but-no-python-dep +# Several pip packages cause this: +orlo binary: executable-not-elf-or-script +orlo binary: souce-is-missing docs/ +# Werkzeug, sphinx: +orlo binary: embedded-javascript-library +orlo binary: package-installs-python-egg usr/share/python/orlo/lib/python2.7/site-packages/flask/testsuite/test_apps/lib/python2.5/site-packages/SiteEgg.egg + +# Binary files within the virtualenv: +orlo binary: arch-dependent-file-in-usr-share usr/share/python/orlo/bin/python +orlo binary: arch-dependent-file-in-usr-share usr/share/python/orlo/lib/python2.7/site-packages/_ldap.so + +# Can't be overridden, would be nice if dh-virtualenv handled this +orlo binary: package-installs-python-bytecode diff --git a/debian/preinst b/debian/preinst index 52c6e9f..fc56249 100644 --- a/debian/preinst +++ b/debian/preinst @@ -14,3 +14,5 @@ touch /var/log/orlo/app.log chown -R orlo:orlo /var/{lib,log}/orlo chmod 755 /var/{lib,log}/orlo chmod 664 /var/log/orlo/app.log + +#DEBHELPER# diff --git a/debian/source.lintian-overrides b/debian/source.lintian-overrides new file mode 100644 index 0000000..7ef2ea4 --- /dev/null +++ b/debian/source.lintian-overrides @@ -0,0 +1 @@ +orlo source: source-is-missing docs/* From 59646f85ed3733547737fd859667b3ef30b5b584 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 14 Apr 2016 17:26:38 +0100 Subject: [PATCH 05/36] Update debian changelog, orlo 0.2.0 pre-release --- debian/changelog | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/debian/changelog b/debian/changelog index 13cbd14..eb3ceef 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,74 @@ +orlo (0.2.0-pre2) UNRELEASED; urgency=medium + + [ Alex Forbes ] + * Add dependant python virtualenv package to Vagrantfile + * Add outline for Deploy process + * Use release object instead of re-declaring, add live server test + + [ Ivan Coppa ] + * Support for Release metadata attributes, Deploy endpoint + * disabling debug + + [ Alex Forbes ] + * Add integrated dummy deployer test + * Use relative path for deployer.rb + + [ Marcel Kuiper ] + * Added POC for Token based AUthentication + + [ Ivan Coppa ] + * enable deploy api call + + [ Marcel Kuiper ] + * processed flake8 recommendations on user_auth.py + + [ ivancoppa ] + * create base data + + [ Alex Forbes ] + * Add last minute changes to create script + * Add last minute hacks to make everything work + * Add orlo_release to deployer env + * Create configuration for auth + + * Make URL a parameter for auth tests + * Only apply login_required when security is enabled + * Rename test classes to start with "Test" + * Move auth config into setup/teardown + * Add root page, test fixes + * Skip deploy test for now + * Move route_api to route_releases, rename metadatas => metadata + * Make token lifetime configurable, update orlo.ini with new options + * Bump version of Flask-HTTPAuth + * Add status code to OrloAuthError and add OrloConfigError + * Remove login_handler in favour or separate TokenAuth module + * Seperate token auth from http basic auth + * Only require auth when configured + * Implement deployer.py test deployer script + * Bump orloclient version, use py.test for travis + * Add pytz to requirements.txt + * Add requirements_testing.txt + * Move pytest to requirements_testing + * Use /bin/true in deploy shell test + + [ Dustin Nguyen ] + * Added ldap auth + * Added ldap mock test + + [ Alex Forbes ] + * Misc changes to deploy + * Tidy up example deployer + * Bump to pre-release 0.2 + * Tidy comments in example deployer + * Use absolute path for version file + * Use orlo.cli as entry point instead of custom script + * Update Vagrantfile + * Fix silly bug, read config from file after set + * Move debian packaging files in to debian dir + * Explicitly set synced_folder to virtualbox provider + * [lintian] fix description + + -- Alex Forbes Thu, 14 Apr 2016 16:17:34 +0000 + orlo (0.1.1) stable; urgency=medium * Cast package_rollback as a boolean From 2fab10e9a34e68cef80c05812a3051b35bc348b9 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 19 Apr 2016 14:31:10 +0100 Subject: [PATCH 06/36] Add /version url and test If _version.py doesn't exist, use "TEST_BUILD" + misc pep8 --- orlo/__init__.py | 8 +++++++- orlo/config.py | 13 +++++++++---- orlo/route_base.py | 12 ++++++++++++ setup.py | 2 +- tests/test_auth.py | 3 +++ tests/test_contract.py | 10 ++++++++++ 6 files changed, 42 insertions(+), 6 deletions(-) diff --git a/orlo/__init__.py b/orlo/__init__.py index 0be2fc9..8769304 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -4,7 +4,13 @@ from orlo.config import config from orlo.exceptions import OrloStartupError -from orlo._version import __version__ + +try: + # _version is created by setup.py + from orlo._version import __version__ +except ImportError: + __version__ = "TEST_BUILD" + app = Flask(__name__) diff --git a/orlo/config.py b/orlo/config.py index f90d87a..03829d0 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -1,6 +1,7 @@ from __future__ import print_function import ConfigParser import os + __author__ = 'alforbes' config = ConfigParser.ConfigParser() @@ -15,9 +16,11 @@ config.add_section('security') config.set('security', 'enabled', 'false') -config.set('security', 'passwd_file', os.path.dirname(__file__) + '/../etc/passwd') +config.set('security', 'passwd_file', + os.path.dirname(__file__) + '/../etc/passwd') config.set('security', 'secret_key', 'change_me') -# NOTE: orlo.__init__ checks that secret_key is not "change_me" when security is enabled +# NOTE: orlo.__init__ checks that secret_key is not "change_me" when security +# is enabled # Do not change the default here without updating __init__ as well. config.set('security', 'token_ttl', '3600') config.set('security', 'ldap_server', 'localhost.localdomain') @@ -33,10 +36,12 @@ config.set('logging', 'file', 'disabled') config.add_section('deploy') -config.set('deploy', 'timeout', '3600') # How long to timeout external deployer calls +config.set('deploy', 'timeout', + '3600') # How long to timeout external deployer calls config.add_section('deploy_shell') -config.set('deploy_shell', 'command_path', os.path.dirname(os.path.abspath(__file__)) + +config.set('deploy_shell', 'command_path', + os.path.dirname(os.path.abspath(__file__)) + '/../deployer.py') config.read('/etc/orlo/orlo.ini') diff --git a/orlo/route_base.py b/orlo/route_base.py index 314a5ef..22fa32a 100644 --- a/orlo/route_base.py +++ b/orlo/route_base.py @@ -1,6 +1,7 @@ from __future__ import print_function from flask import jsonify, request from orlo import app +from orlo import __version__ __author__ = 'alforbes' @@ -31,6 +32,17 @@ def root(): }) +@app.route('/version', methods=['GET']) +def version(): + """ + Display version information + :return: + """ + return jsonify({ + 'version': str(__version__) + }) + + @app.route('/ping', methods=['GET']) def ping(): """ diff --git a/setup.py b/setup.py index 19e1ec5..9cd5087 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre1' +VERSION = '0.2.0-pre2' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) diff --git a/tests/test_auth.py b/tests/test_auth.py index 5570e55..606bb3b 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -33,11 +33,13 @@ def auth_required(): response.status_code = 200 return response + @orlo.app.route('/test/user') @conditional_auth(user_auth.login_required) def get_resource(): return jsonify({'data': 'Hello, %s!' % g.current_user}) + class OrloAuthTest(TestCase): """ Base test class to setup the app @@ -57,6 +59,7 @@ def create_app(self): self.app.config['DEBUG'] = True self.app.config['TRAP_HTTP_EXCEPTIONS'] = True self.app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False + print(self.app.logger.handlers) return self.app diff --git a/tests/test_contract.py b/tests/test_contract.py index d5caa09..be69bc2 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -368,6 +368,16 @@ def test_ping(self): self.assert200(response) self.assertEqual('pong', response.json['message']) + def test_version(self): + """ + Test the version url + """ + from orlo import __version__ + print("VERSION: " + str(__version__)) + response = self.client.get('/version') + self.assert200(response) + self.assertEqual(__version__, response.json['version']) + def test_get_single_release(self): """ Fetch a single release From 18984bcd0fac4b29055c5e049ccd2e9c17c3bf15 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 20 Apr 2016 12:04:59 +0100 Subject: [PATCH 07/36] Add --version option to command line --- orlo/cli.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/orlo/cli.py b/orlo/cli.py index 7af90ae..168b4b5 100644 --- a/orlo/cli.py +++ b/orlo/cli.py @@ -1,6 +1,13 @@ from __future__ import print_function import argparse +try: + from orlo import __version__ +except ImportError: + # _version.py doesn't exist + __version__ = "TEST_BUILD" + + __author__ = 'alforbes' """ @@ -11,10 +18,14 @@ def parse_args(): - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + prog='orlo', + version=__version__, + ) p_config = argparse.ArgumentParser(add_help=False) - p_config.add_argument('--file', '-f', dest='filepath', help="File to write to", + p_config.add_argument('--file', '-f', dest='file_path', + help="File to write to", default='/etc/orlo/orlo.ini') p_database = argparse.ArgumentParser(add_help=False) @@ -45,7 +56,7 @@ def parse_args(): def write_config(args): from orlo import config - config_file = open(args.filepath, 'w') + config_file = open(args.file_path, 'w') config.write(config_file) From 32d1158594f88acf53cbabe3c03e6b4ca60994e7 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 21 Apr 2016 12:15:30 +0100 Subject: [PATCH 08/36] Handle getting an invalid release id gracefully --- orlo/queries.py | 101 +++++++++++++++++++++++--------------- orlo/route_releases.py | 107 +++++++++++++++++++++++++---------------- orlo/util.py | 37 ++++++++++---- tests/__init__.py | 2 +- tests/test_contract.py | 7 +++ 5 files changed, 163 insertions(+), 91 deletions(-) diff --git a/orlo/queries.py b/orlo/queries.py index 3b87e26..88a92f0 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -1,11 +1,10 @@ from __future__ import print_function -import calendar import datetime import arrow from orlo import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage -from collections import OrderedDict +from sqlalchemy import exc __author__ = 'alforbes' @@ -14,11 +13,22 @@ """ +def get_release(release_id): + """ + Fetch a single release + + :param release_id: + :return: + """ + return db.session.query(Release).filter(Release.id == release_id) + + def filter_release_status(query, status): """ Filter the given query by the given release status - Release status is special, because it's actually determined by the package status + Release status is special, because it's actually determined by the + package status :param query: Query object :param status: The status to filter on @@ -27,15 +37,16 @@ def filter_release_status(query, status): enums = Package.status.property.columns[0].type.enums if status not in enums: raise InvalidUsage("Invalid package status, {} is not in {}".format( - status, str(enums))) + status, str(enums))) if status in ["SUCCESSFUL", "NOT_STARTED"]: # ALL packages must match this status for it to apply to the release - # Query logic translates to "Releases which do not have any packages which satisfy + # Query logic translates to "Releases which do not have any packages + # which satisfy # the condition 'Package.status != status'". I.E, all match. query = query.filter( - ~Release.packages.any( - Package.status != status - )) + ~Release.packages.any( + Package.status != status + )) elif status in ["FAILED", "IN_PROGRESS"]: # ANY package can match for this status to apply to the release query = query.filter(Release.packages.any(Package.status == status)) @@ -53,15 +64,16 @@ def filter_release_rollback(query, rollback): if rollback is True: # Only count releases which have a rollback package query = query.filter( - Release.packages.any(Package.rollback == True) + Release.packages.any(Package.rollback == True) ) elif rollback is False: # Only count releases which do not have any rollback packages query = query.filter( - ~Release.packages.any(Package.rollback == True) + ~Release.packages.any(Package.rollback == True) ) else: # What the hell did you pass? - raise TypeError("Bad rollback parameter: '{}', type {}. Boolean expected.".format( + raise TypeError( + "Bad rollback parameter: '{}', type {}. Boolean expected.".format( rollback, type(rollback))) return query @@ -142,7 +154,8 @@ def apply_filters(query, args): value = arrow.get(value) # Do comparisons - app.logger.debug("Filtering: {} {} {}".format(filter_field, comparison, value)) + app.logger.debug( + "Filtering: {} {} {}".format(filter_field, comparison, value)) if comparison == '==': query = query.filter(filter_field == value) if comparison == '<': @@ -169,7 +182,8 @@ def releases(**kwargs): if any(field.startswith('package_') for field in kwargs.keys()) \ or "status" in kwargs.keys(): - # Package attributes need the join, as does status as it's really a package + # Package attributes need the join, as does status as it's really a + # package # attribute query = db.session.query(Release).join(Package) else: @@ -179,7 +193,8 @@ def releases(**kwargs): try: query = apply_filters(query, kwargs) except AttributeError as e: - raise InvalidUsage("An invalid field was specified: {}".format(e.message)) + raise InvalidUsage( + "An invalid field was specified: {}".format(e.message)) if desc: stime_field = Release.stime.desc @@ -227,7 +242,7 @@ def user_info(username): :return: """ query = db.session.query( - Release.user, db.func.count(Release.id)) \ + Release.user, db.func.count(Release.id)) \ .filter(Release.user == username) \ .group_by(Release.user) @@ -270,7 +285,7 @@ def team_info(team_name): :return: """ query = db.session.query( - Release.user, db.func.count(Release.id)) \ + Release.user, db.func.count(Release.id)) \ .filter(Release.team == team_name) \ .group_by(Release.team) @@ -322,7 +337,7 @@ def package_info(package_name): :return: """ query = db.session.query( - Package.name, db.func.count(Package.id)) \ + Package.name, db.func.count(Package.id)) \ .filter(Package.name == package_name) \ .group_by(Package.name) @@ -337,7 +352,8 @@ def package_list(platform=None): """ query = db.session.query(Package.name).distinct() if platform: - query = query.join(Release).filter(Release.platforms.any(Platform.name == platform)) + query = query.join(Release).filter( + Release.platforms.any(Platform.name == platform)) return query @@ -346,15 +362,17 @@ def package_versions(platform=None): """ List the current version of all packages - It is not sufficient to just return the highest version of each successful package, + It is not sufficient to just return the highest version of each + successful package, as they can be rolled back, so we determine the version by last release time :param platform: Platform to filter on """ - # Sub query gets a list of successful packages by last successful release time + # Sub query gets a list of successful packages by last successful release + # time sub_q = db.session.query( - Package.name, db.func.max(Package.stime).label('max_stime')) \ + Package.name, db.func.max(Package.stime).label('max_stime')) \ .filter(Package.status == 'SUCCESSFUL') if platform: # filter releases not on this platform sub_q = sub_q \ @@ -366,15 +384,16 @@ def package_versions(platform=None): # q inner-joins Package table with sub_q to get the version q = db.session.query( - Package.name, - Package.version) \ + Package.name, + Package.version) \ .join(sub_q, sub_q.c.max_stime == Package.stime) \ .group_by(Package.name, Package.version) return q -def count_releases(user=None, package=None, team=None, platform=None, status=None, +def count_releases(user=None, package=None, team=None, platform=None, + status=None, rollback=None, stime=None, ftime=None): """ Return the number of releases with the attributes specified @@ -386,25 +405,28 @@ def count_releases(user=None, package=None, team=None, platform=None, status=Non :param string platform: Filter by platform :param string stime: Filter by releases that started after :param string ftime: Filter by releases that started before - :param boolean rollback: Filter on whether or not the release contains a rollback + :param boolean rollback: Filter on whether or not the release contains a \ + rollback :return: Query - Note that rollback and status are special fields when applied to a release, as they are - Package attributes. + Note that rollback and status are special fields when applied to a + release, as they are Package attributes. - A "successful" or "in progress" release is defined as a release where all packages match the - status. Conversely, a "failed" or "not started" release is defined as a release where any - package matches. + A "successful" or "in progress" release is defined as a release where all + packages match the status. Conversely, a "failed" or "not started" release + is defined as a release where any package matches. - For rollbacks, if any package is a rollback the release is included, otherwise if all - packages are not rollbacks the release obviously isn't either. + For rollbacks, if any package is a rollback the release is included, + otherwise if all packages are not rollbacks the release obviously isn't + either. - Implication of this is that a release can be both "failed" and "in progress". + Implication of this is that a release can be both "failed" and "in + progress". """ args = { - 'user': user, 'package': package, 'team': team, 'platform': platform, 'status': status, - 'rollback': rollback, 'stime': stime, 'ftime': ftime, + 'user': user, 'package': package, 'team': team, 'platform': platform, + 'status': status, 'rollback': rollback, 'stime': stime, 'ftime': ftime, } app.logger.debug("Entered count_releases with args: {}".format(str(args))) @@ -432,7 +454,8 @@ def count_releases(user=None, package=None, team=None, platform=None, status=Non return query -def count_packages(user=None, team=None, platform=None, status=None, rollback=None): +def count_packages(user=None, team=None, platform=None, status=None, + rollback=None): """ Return the number of packages with the attributes specified @@ -468,7 +491,7 @@ def platform_summary(): """ query = db.session.query( - Platform.name, db.func.count(Platform.id)) \ + Platform.name, db.func.count(Platform.id)) \ .join(release_platform) \ .group_by(Platform.name) @@ -484,7 +507,7 @@ def platform_info(platform_name): """ query = db.session.query( - Platform.name, db.func.count(Platform.id)) \ + Platform.name, db.func.count(Platform.id)) \ .filter(Platform.name == platform_name) \ .group_by(Platform.name) @@ -502,5 +525,3 @@ def platform_list(): .join(release_platform) return query - - diff --git a/orlo/route_releases.py b/orlo/route_releases.py index 9062c6b..dbf6a9e 100644 --- a/orlo/route_releases.py +++ b/orlo/route_releases.py @@ -2,10 +2,11 @@ from orlo import app, queries, config from orlo.exceptions import InvalidUsage from orlo.user_auth import token_auth -from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, ReleaseMetadata, Platform -from orlo.util import validate_request_json, create_release, validate_release_input, \ - validate_package_input, fetch_release, create_package, fetch_package, stream_json_list, \ - str_to_bool +from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, \ + ReleaseMetadata, Platform +from orlo.util import validate_request_json, create_release, \ + validate_release_input, validate_package_input, fetch_release, \ + create_package, fetch_package, stream_json_list, str_to_bool, is_uuid from orlo.deploy import ShellDeploy from orlo.user_auth import conditional_auth @@ -34,8 +35,8 @@ def post_releases(): curl -H "Content-Type: application/json" \\ -X POST \\ http://127.0.0.1/releases \\ - -d '{"note": "blah", "platforms": ["site1"], "references": ["ticket"], "team": "A-Team", - "user": "aforbes"}' + -d '{"note": "blah", "platforms": ["site1"], "references": ["ticket"], + "team": "A-Team", "user": "aforbes"}' """ validate_release_input(request) release = create_release(request) @@ -51,7 +52,8 @@ def post_releases(): app.logger.info( 'Create release {}, references: {}, platforms: {}'.format( - release.id, release.notes, release.references, release.platforms, release.metadata) + release.id, release.notes, release.references, release.platforms, + release.metadata) ) release.start() @@ -81,8 +83,7 @@ def post_packages(release_id): .. sourcecode:: shell curl -H "Content-Type: application/json" \\ - -X POST \\ - http://127.0.0.1/releases/${RELEASE_ID}/packages \\ + -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages \\ -d '{"name": "test-package", "version": "1.0.1"}' """ validate_package_input(request, release_id) @@ -91,9 +92,9 @@ def post_packages(release_id): package = create_package(release.id, request) app.logger.info( - 'Create package {}, release {}, name {}, version {}'.format( - package.id, release.id, request.json['name'], - request.json['version'])) + 'Create package {}, release {}, name {}, version {}'.format( + package.id, release.id, request.json['name'], + request.json['version'])) db.session.add(package) db.session.commit() @@ -115,7 +116,7 @@ def post_results(release_id, package_id): """ results = PackageResult(package_id, str(request.json)) app.logger.info("Post results, release {}, package {}".format( - release_id, package_id)) + release_id, package_id)) db.session.add(results) db.session.commit() return '', 204 @@ -193,11 +194,12 @@ def post_packages_start(release_id, package_id): .. sourcecode:: shell - curl -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages/${PACKAGE_ID}/start + curl -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages/${ + PACKAGE_ID}/start """ package = fetch_package(release_id, package_id) app.logger.info("Package start, release {}, package {}".format( - release_id, package_id)) + release_id, package_id)) package.start() db.session.add(package) @@ -217,7 +219,8 @@ def post_packages_stop(release_id, package_id): .. sourcecode:: shell curl -H "Content-Type: application/json" \\ - -X POST http://127.0.0.1/releases/${RELEASE_ID}/packages/${PACKAGE_ID}/stop \\ + -X POST \\ + http://127.0.0.1/releases/${RELEASE_ID}/packages/${PACKAGE_ID}/stop \\ -d '{"success": "true"}' :param string package_id: Package UUID @@ -228,7 +231,7 @@ def post_packages_stop(release_id, package_id): package = fetch_package(release_id, package_id) app.logger.info("Package stop, release {}, package {}, success {}".format( - release_id, package_id, success)) + release_id, package_id, success)) package.stop(success=success) db.session.add(package) @@ -271,7 +274,9 @@ def post_releases_metadata(release_id): validate_request_json(request) meta = request if not meta: - raise InvalidUsage("Must include metadata in posted document: es {\"key\" : \"value\"}") + raise InvalidUsage( + "Must include metadata in posted document: es {\"key\" : " + "\"value\"}") for key, value in request.json.items(): app.logger.info("Adding Metadata to release {}".format(release_id)) @@ -288,25 +293,34 @@ def get_releases(release_id=None): """ Return a list of releases to the client, filters optional - :param string release_id: Optionally specify a single release UUID to fetch. \ + :param string release_id: Optionally specify a single release UUID to + fetch. \ This does not disable filters. - :query int desc: Normally results are returned ordered by stime ascending, setting + :query int desc: Normally results are returned ordered by stime + ascending, setting desc to true will reverse this and sort by stime descending :query int limit: Limit the results by int :query int offset: Offset the results by int :query string package_name: Filter releases by package name :query string user: Filter releases by user the that performed the release :query string platform: Filter releases by platform - :query string stime_before: Only include releases that started before timestamp given - :query string stime_after: Only include releases that started after timestamp given - :query string ftime_before: Only include releases that finished before timestamp given - :query string ftime_after: Only include releases that finished after timestamp given + :query string stime_before: Only include releases that started before \ + timestamp given + :query string stime_after: Only include releases that started after \ + timestamp given + :query string ftime_before: Only include releases that finished before \ + timestamp given + :query string ftime_after: Only include releases that finished after \ + timestamp given :query string team: Filter releases by team - :query string status: Filter by release status. This field is calculated from the package. \ - status, see special note below. - :query int duration_lt: Only include releases that took less than (int) seconds - :query int duration_gt: Only include releases that took more than (int) seconds - :query boolean package_rollback: Filter on whether or not the releases contain a rollback + :query string status: Filter by release status. This field is calculated \ + from the package status, see special note below. + :query int duration_lt: Only include releases that took less than (int) \ + seconds + :query int duration_gt: Only include releases that took more than (int) \ + seconds + :query boolean package_rollback: Filter on whether or not the releases \ + contain a rollback :query string package_name: Filter by package name :query string package_version: Filter by package version :query int package_duration_gt: Filter by packages of duration greater than @@ -315,26 +329,32 @@ def get_releases(release_id=None): "NOT_STARTED", "IN_PROGRESS", "SUCCESSFUL", "FAILED" **Note for time arguments**: - The timestamp format you must use is specified in /etc/orlo/orlo.ini. All times are UTC. + The timestamp format you must use is specified in /etc/orlo/orlo.ini. + All times are UTC. **Note on status**: - The release status is calculated from the packages it contains. The possible values are - the same as a package. For a release to be considered "SUCCESSFUL" or "NOT_STARTED", - all packages must have this value. If any one package has the value "IN_PROGRESS" or - "FAILED", that status applies to the whole release, with "FAILED" overriding "IN_PROGRESS". + The release status is calculated from the packages it contains. The + possible values are the same as a package. For a release to be + considered "SUCCESSFUL" or "NOT_STARTED", all packages must have this + value. If any one package has the value "IN_PROGRESS" or "FAILED", + that status applies to the whole release, with "FAILED" overriding + "IN_PROGRESS". """ - booleans = ('rollback', 'package_rollback', ) + booleans = ('rollback', 'package_rollback',) - if release_id: # Simple - query = db.session.query(Release).filter(Release.id == release_id) + if release_id: # Simple, just fetch one release + if not is_uuid(release_id): + raise InvalidUsage("Release ID given is not a valid UUID") + query = queries.get_release(release_id) elif len(request.args.keys()) == 0: raise InvalidUsage("Please specify a filter. See " - "http://orlo.readthedocs.org/en/latest/rest.html#get--releases for " - "more info") + "http://orlo.readthedocs.org/en/latest/rest.html" + "#get--releases for more info") else: # Bit more complex - # Flatten args, as the ImmutableDict puts some values in a list when expanded + # Flatten args, as the ImmutableDict puts some values in a list when + # expanded args = {} for k in request.args.keys(): if k in booleans: @@ -343,4 +363,9 @@ def get_releases(release_id=None): args[k] = request.args.get(k) query = queries.releases(**args) - return Response(stream_json_list('releases', query), content_type='application/json') + # Execute eagerly to avoid confusing stack traces within the Response on + # error + db.session.execute(query) + + return Response(stream_json_list('releases', query), + content_type='application/json') diff --git a/orlo/util.py b/orlo/util.py index 7d478c9..e3421dd 100644 --- a/orlo/util.py +++ b/orlo/util.py @@ -1,19 +1,19 @@ from __future__ import print_function, unicode_literals -import arrow -import datetime from flask import json from orlo import app from orlo.orm import db, Release, Package, Platform from orlo.exceptions import InvalidUsage from sqlalchemy.orm import exc from six import string_types +import uuid __author__ = 'alforbes' def append_or_create_platforms(request_platforms): """ - Create the platforms if they don't exist, and return a list of Platform objects + Create the platforms if they don't exist, and return a list of Platform + objects :param list request_platforms: List of strings denoting platform names """ @@ -106,7 +106,8 @@ def validate_request_json(request): try: request.json except Exception: - # This is pretty ugly, but we want something more user friendly than "Bad Request" + # This is pretty ugly, but we want something more user friendly than + # "Bad Request" raise InvalidUsage("Could not parse JSON document") if not request.json: @@ -125,7 +126,8 @@ def validate_package_input(request, release_id): if not 'name' in request.json or not 'version' in request.json: raise InvalidUsage("Missing name / version in request body.") - app.logger.debug("Package request validated, release_id {}".format(release_id)) + app.logger.debug( + "Package request validated, release_id {}".format(release_id)) return True @@ -150,10 +152,12 @@ def list_to_string(array): def stream_json_list(heading, iterator): """ - A lagging generator to stream JSON so we don't have to hold everything in memory + A lagging generator to stream JSON so we don't have to hold everything in + memory - This is a little tricky, as we need to omit the last comma to make valid JSON, - thus we use a lagging generator, similar to http://stackoverflow.com/questions/1630320/ + This is a little tricky, as we need to omit the last comma to make valid + JSON, thus we use a lagging generator, similar to + http://stackoverflow.com/questions/1630320/ :param heading: The title of the set, e.g. "releases" :param iterator: Any object with __iter__(), e.g. SQLAlchemy Query @@ -162,7 +166,8 @@ def stream_json_list(heading, iterator): try: prev_release = next(iterator) # get first result except StopIteration: - # StopIteration here means the length was zero, so yield a valid releases doc and stop + # StopIteration here means the length was zero, so yield a valid + # releases doc and stop yield '{{"{}": []}}'.format(heading) raise StopIteration @@ -190,3 +195,17 @@ def str_to_bool(value): if isinstance(value, int): return True if value > 0 else False raise ValueError("Value {} can not be cast as boolean".format(value)) + + +def is_uuid(string): + """ + Test whether a string is a valid UUID + + :param string: + :return: + """ + try: + uuid.UUID(string) + except ValueError: + return False + return True diff --git a/tests/__init__.py b/tests/__init__.py index 5ad64c1..05b4ea8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -13,7 +13,7 @@ def create_app(self): app.config['SQLALCHEMY_DATABASE_URI'] = 'postgres://orlo:password@localhost:5432/orlo' # app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://' app.config['TESTING'] = True - app.config['DEBUG'] = True + app.config['DEBUG'] = False app.config['TRAP_HTTP_EXCEPTIONS'] = True app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False diff --git a/tests/test_contract.py b/tests/test_contract.py index be69bc2..b56c233 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -388,6 +388,13 @@ def test_get_single_release(self): self.assertEqual(1, len(r['releases'])) self.assertEqual(release_id, r['releases'][0]['id']) + def test_get_single_release_invalid(self): + """ + Test that we return 400 with an invalid release ID + """ + r = self._get_releases(release_id="not_a_valid_uuid", + expected_status=400) + def test_get_releases(self): """ Test the list of releases From 535b873ccbe30411729bd8d7543faf58deae343d Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 21 Apr 2016 12:17:11 +0100 Subject: [PATCH 09/36] Reduce test verbosity by disabling debug and removing prints + bump pre-release build --- orlo/__init__.py | 2 -- setup.py | 2 +- tests/test_auth.py | 3 +-- tests/test_contract.py | 1 - tests/test_orm.py | 2 -- tests/test_stats.py | 1 - tests/test_stats_empirical.py | 2 +- 7 files changed, 3 insertions(+), 10 deletions(-) diff --git a/orlo/__init__.py b/orlo/__init__.py index 8769304..98587ff 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -32,8 +32,6 @@ if config.getboolean('logging', 'debug'): app.logger.setLevel(logging.DEBUG) -app.logger.debug('Debug enabled') - if not config.getboolean('main', 'strict_slashes'): app.url_map.strict_slashes = False diff --git a/setup.py b/setup.py index 9cd5087..0f415dc 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre2' +VERSION = '0.2.0-pre3' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) diff --git a/tests/test_auth.py b/tests/test_auth.py index 606bb3b..b604085 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -56,10 +56,9 @@ def create_app(self): self.app = orlo.app self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://' self.app.config['TESTING'] = True - self.app.config['DEBUG'] = True + self.app.config['DEBUG'] = False self.app.config['TRAP_HTTP_EXCEPTIONS'] = True self.app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False - print(self.app.logger.handlers) return self.app diff --git a/tests/test_contract.py b/tests/test_contract.py index b56c233..5ceead5 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -373,7 +373,6 @@ def test_version(self): Test the version url """ from orlo import __version__ - print("VERSION: " + str(__version__)) response = self.client.get('/version') self.assert200(response) self.assertEqual(__version__, response.json['version']) diff --git a/tests/test_orm.py b/tests/test_orm.py index 130ff50..3bbe6b4 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -164,8 +164,6 @@ def test_package_types(self): """ p = db.session.query(Package).first() x = db.session.query(Package).all() - print(x) - print(len(x)) self.assertIs(type(p.id), uuid.UUID) self.assertIs(type(p.name), unicode) self.assertIs(type(p.stime), arrow.arrow.Arrow) diff --git a/tests/test_stats.py b/tests/test_stats.py index a774043..ac8ff8f 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -135,7 +135,6 @@ def test_package_time_month(self): year = str(arrow.utcnow().year) month = str(arrow.utcnow().month) self.assertEqual(7, result[year][month]['test-package']['normal']['successful']) - print(result) def test_package_time_week(self): """ diff --git a/tests/test_stats_empirical.py b/tests/test_stats_empirical.py index 1a03302..4225f7e 100644 --- a/tests/test_stats_empirical.py +++ b/tests/test_stats_empirical.py @@ -64,7 +64,7 @@ def setUp(self): self.create_release(False, False) db.session.commit() - print("Total releases created: {}".format(self.releases)) + # print("Total releases created: {}".format(self.releases)) def create_release(self, normal, successful, user='test_user', team='test_team', platform='test_platform'): From 4a52d9f38412da795a348a5f6cfc852b5f656bb8 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 21 Apr 2016 14:45:38 +0100 Subject: [PATCH 10/36] [debian] Add prerm script to stop service --- debian/prerm | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 debian/prerm diff --git a/debian/prerm b/debian/prerm new file mode 100644 index 0000000..a6f26fc --- /dev/null +++ b/debian/prerm @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +systemctl stop orlo.service \ No newline at end of file From 725ba023905f4acd20225cc90abf70986ea9f6b6 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 21 Apr 2016 17:19:07 +0100 Subject: [PATCH 11/36] Separate post_releases_start and post_releases_deploy WIP --- orlo/deploy.py | 44 +++++++++++++++++++++++++----------------- orlo/exceptions.py | 4 ++++ orlo/orm.py | 3 ++- orlo/route_releases.py | 27 +++++++++++++++++++++----- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/orlo/deploy.py b/orlo/deploy.py index 4e617f5..6174f7e 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -2,10 +2,9 @@ import json import subprocess from threading import Timer -from orlo import app from orlo.config import config -from orlo.exceptions import OrloError - +from orlo.exceptions import OrloDeployError +from orlo import app __author__ = 'alforbes' @@ -82,7 +81,7 @@ def start(self): args = [config.get('deploy_shell', 'command_path')] for p in self.release.packages: args.append("{}={}".format(p.name, p.version)) - print("Args: {}".format(str(args))) + app.logger.debug("Args: {}".format(str(args))) env = { 'ORLO_URL': self.server_url, @@ -92,7 +91,7 @@ def start(self): my_key = "ORLO_" + key.upper() env[my_key] = str(value) - print("Env: {}".format(json.dumps(env))) + app.logger.debug("Env: {}".format(json.dumps(env))) metadata = {} for m in self.release.metadata: @@ -121,13 +120,17 @@ def run_command(args, env, in_data, timeout_sec=3600): :param timeout_sec: Timeout in seconds, 1 hour by default :return: """ - proc = subprocess.Popen( - args, - env=env, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + try: + proc = subprocess.Popen( + args, + env=env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except OSError as e: + m = "OSError starting deployer process: {}".format(e.strerror) + raise OrloDeployError(m) timer = Timer(timeout_sec, proc.kill) out = err = " " @@ -136,11 +139,16 @@ def run_command(args, env, in_data, timeout_sec=3600): out, err = proc.communicate(in_data) finally: timer.cancel() - print("Out:\n{}".format(out)) - print("Err:\n{}".format(err)) + app.logger.debug("Out:\n{}".format(out)) + app.logger.debug("Err:\n{}".format(err)) if proc.returncode is not 0: - raise OrloError("Subprocess exited with code {}".format( - proc.returncode), status_code=500) - print("end run") - + raise OrloDeployError( + message="Subprocess exited with code {}".format( + proc.returncode), + payload={ + 'stdout': out, + 'stderr': err, + }, + status_code=500) + app.logger.debug("end run") diff --git a/orlo/exceptions.py b/orlo/exceptions.py index 85c4fc2..c25ddc9 100644 --- a/orlo/exceptions.py +++ b/orlo/exceptions.py @@ -45,3 +45,7 @@ class OrloConfigError(Exception): def __init__(self, message): Exception.__init__(self) print("Configuration Error: " + message) + + +class OrloDeployError(OrloError): + status_code = 500 diff --git a/orlo/orm.py b/orlo/orm.py index 2e968e3..80f7775 100644 --- a/orlo/orm.py +++ b/orlo/orm.py @@ -161,7 +161,8 @@ def stop(self, success): :param success: Whether or not the package deploy succeeded """ if self.stime is None: - raise OrloWorkflowError("Can not stop a package which has not been started") + raise OrloWorkflowError( + "Can not stop a package which has not been started") self.ftime = arrow.now(config.get('main', 'time_zone')) td = self.ftime - self.stime diff --git a/orlo/route_releases.py b/orlo/route_releases.py index dbf6a9e..f3f4ad4 100644 --- a/orlo/route_releases.py +++ b/orlo/route_releases.py @@ -124,11 +124,9 @@ def post_results(release_id, package_id): @app.route('/releases//deploy', methods=['POST']) @conditional_auth(token_auth.token_required) -def post_releases_start(release_id): +def post_releases_deploy(release_id): """ - Indicate that a release is starting - - This trigger the start of the deploy + Deploy a Release :param string release_id: Release UUID @@ -140,17 +138,36 @@ def post_releases_start(release_id): -X POST http://127.0.0.1/releases/${RELEASE_ID}/deploy """ release = fetch_release(release_id) - # TODO call deploy Class start Method app.logger.info("Release start, release {}".format(release_id)) + release.start() db.session.add(release) db.session.commit() + # TODO call deploy Class start Method, i.e. pure python rather than shell deploy = ShellDeploy(release) deploy.start() return '', 204 +@app.route('/releases//start', methods=['POST']) +@conditional_auth(token_auth.token_required) +def post_releases_start(release_id): + """ + Indicate that a release is starting + + :param release_id: + :return: + """ + release = fetch_release(release_id) + app.logger.info("Release start, release {}".format(release_id)) + release.start() + + db.session.add(release) + db.session.commit() + return '', 204 + + @app.route('/releases//stop', methods=['POST']) @conditional_auth(token_auth.token_required) def post_releases_stop(release_id): From 4ecd03c1841760462f8a9c800bb0acf44f498fd4 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Fri, 6 May 2016 11:05:31 +0100 Subject: [PATCH 12/36] Log output of deploy --- orlo/deploy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orlo/deploy.py b/orlo/deploy.py index 6174f7e..6f0e9bc 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -151,4 +151,7 @@ def run_command(args, env, in_data, timeout_sec=3600): 'stderr': err, }, status_code=500) + else: + app.logger.info( + "Deploy completed successfully. Output:\n{}".format(out)) app.logger.debug("end run") From 4ea03e561d6850eef21c12c525382b0bede56aa9 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Fri, 6 May 2016 11:23:37 +0100 Subject: [PATCH 13/36] Use json.dumps instead of str() for reliable conversion --- orlo/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orlo/deploy.py b/orlo/deploy.py index 6f0e9bc..9c6cd33 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -89,7 +89,7 @@ def start(self): } for key, value in self.release.to_dict().items(): my_key = "ORLO_" + key.upper() - env[my_key] = str(value) + env[my_key] = json.dumps(value) app.logger.debug("Env: {}".format(json.dumps(env))) From d1c3bf4f3e2b1b3155a604d284346bb20543d5b4 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Fri, 6 May 2016 11:28:02 +0100 Subject: [PATCH 14/36] Return output of deploy to client --- debian/changelog | 11 ++++++++++- orlo/deploy.py | 12 +++++++++--- orlo/route_releases.py | 4 ++-- setup.py | 2 +- tests/test_deploy.py | 18 ++++++++++++++++++ 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index eb3ceef..38d6506 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -orlo (0.2.0-pre2) UNRELEASED; urgency=medium +orlo (0.2.0-pre4) UNRELEASED; urgency=medium [ Alex Forbes ] * Add dependant python virtualenv package to Vagrantfile @@ -66,6 +66,15 @@ orlo (0.2.0-pre2) UNRELEASED; urgency=medium * Move debian packaging files in to debian dir * Explicitly set synced_folder to virtualbox provider * [lintian] fix description + * Add /version url and test + * Add --version option to command line + * Handle getting an invalid release id gracefully + * Reduce test verbosity by disabling debug and removing prints + * [debian] Add prerm script to stop service + * Separate post_releases_start and post_releases_deploy + * Log output of deploy + * Use json.dumps instead of str() for reliable conversion + * Return output of deploy to client -- Alex Forbes Thu, 14 Apr 2016 16:17:34 +0000 diff --git a/orlo/deploy.py b/orlo/deploy.py index 9c6cd33..59c33d5 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -98,8 +98,8 @@ def start(self): metadata.update(m.to_dict()) in_data = json.dumps(metadata) - self.run_command(args, env, in_data, - timeout_sec=config.getint('deploy', 'timeout')) + return self.run_command( + args, env, in_data, timeout_sec=config.getint('deploy', 'timeout')) def kill(self): """ @@ -116,7 +116,8 @@ def run_command(args, env, in_data, timeout_sec=3600): :param env: Dict of environment variables :param in_data: String to pass to stdin - :param args: List of arguments + :param list args: Arguments (i.e. full list including command, + as you would pass to subprocess) :param timeout_sec: Timeout in seconds, 1 hour by default :return: """ @@ -155,3 +156,8 @@ def run_command(args, env, in_data, timeout_sec=3600): app.logger.info( "Deploy completed successfully. Output:\n{}".format(out)) app.logger.debug("end run") + + return { + 'stdout': out, + 'stderr': err, + } diff --git a/orlo/route_releases.py b/orlo/route_releases.py index f3f4ad4..b027eb8 100644 --- a/orlo/route_releases.py +++ b/orlo/route_releases.py @@ -146,8 +146,8 @@ def post_releases_deploy(release_id): # TODO call deploy Class start Method, i.e. pure python rather than shell deploy = ShellDeploy(release) - deploy.start() - return '', 204 + output = deploy.start() + return jsonify(output), 200 @app.route('/releases//start', methods=['POST']) diff --git a/setup.py b/setup.py index 0f415dc..b2f76cd 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre3' +VERSION = '0.2.0-pre4' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 8539147..4efc391 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -160,3 +160,21 @@ def test_start_example_deployer(self): deploy.start() + def test_output(self): + """ + Test that we return the output of the deploy + + Not a good test, as it relies on the test-package being an argument, + and simply echoing it back. This is the "spec", but this test could + break if the arguments change. + """ + with ConfigChange('deploy', 'timeout', '3'), \ + ConfigChange('deploy_shell', 'command_path', '/bin/echo'): + deploy = ShellDeploy(self.release) + + # Override server_url, normally it is set by config: + deploy.server_url = self.get_server_url() + + output = deploy.start() + self.assertEqual(output['stdout'], 'test-package=1.2.3\n') + From 2c9ac93e82ca51123161ee9b37ccd6f7e2a2a206 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 12 May 2016 12:54:01 +0100 Subject: [PATCH 15/36] Add .DS_Store to gitignore --- .gitignore | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 02c5e8d..fd51a5d 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ orlo/_version.py .vagrant *.swp *.deb + +# OSX +.DS_Store diff --git a/setup.py b/setup.py index b2f76cd..ffbd61f 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre4' +VERSION = '0.2.0-pre5' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) From 30e8ccbc725e540f4445875a98906a2aa71abb09 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 17 May 2016 11:47:52 +0100 Subject: [PATCH 16/36] Add payload of arguments to error --- orlo/deploy.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/orlo/deploy.py b/orlo/deploy.py index 59c33d5..3050e09 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -130,8 +130,11 @@ def run_command(args, env, in_data, timeout_sec=3600): stderr=subprocess.PIPE, ) except OSError as e: - m = "OSError starting deployer process: {}".format(e.strerror) - raise OrloDeployError(m) + raise OrloDeployError( + message="OSError starting process: {}".format( + e.strerror), + payload={'arguments': args} + ) timer = Timer(timeout_sec, proc.kill) out = err = " " From 1dac4c93931e6592ecc549fa62003a2ae504caf4 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 24 May 2016 12:57:23 +0100 Subject: [PATCH 17/36] Add configuration to documentation --- docs/config.rst | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 7 ++--- docs/install.rst | 22 ++++++++++++--- 3 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 docs/config.rst diff --git a/docs/config.rst b/docs/config.rst new file mode 100644 index 0000000..5a12d41 --- /dev/null +++ b/docs/config.rst @@ -0,0 +1,70 @@ +Configuration +============= + +[main] +`````` + +:debug_mode: `true` or `false`. Default `false`. Enables Flask's debug mode. +:propagate_exceptions: `true` or `false`. Default `true`. Sets + 'PROPAGATE_EXCEPTIONS` in Flask. See + `Flask documentation `_ + for details. +:time_format: A `strftime `_ + string. Default `%Y-%m-%dT%H:%M:%SZ`. +:time_zone: Local time zone, as understood by pytz. Internally, + all timestamps are stored in UTC. The timestamp is interpreted by Arrow when + timestamps are given by the user (e.g. on recording a release), this + setting merely reflects what time zone is given back to GET requests. + Default `UTC`. +:strict_slashes: `true` or `false`. Default `false`. By default, Werkzeug + (what Flask uses underneath), will automatically "handle" trailing slashes, + with the result that /foo/ and /foo are the same url. This disables that + behaviour. It is recommended that you leave this set to false. See the + `Werkzeug documentation `_ + for more information. +:base_url: The external url which points to your web app. Required for + callbacks. Default `http://localhost:8080`. + +[security] +`````````` + +:enabled: `true` or `false`. Enables security. +:passwd_file: Path to a `htpasswd `_ + file to use for authentication. +:secret_key: A secret key to use for token encryption. Default `change_me`. + If security is enabled and this is still set to `change_me`, Orlo will + refuse to start. +:token_ttl: The length of times that tokens should live for in seconds. + Tokens automatically expire after this. Default `3600`. +:ldap_server: Ldap server to use for ldap requests. Default `localhost + .localdomain` +:ldap_port: Ldap port to use for ldap requests. Default `389`. +:user_base_dn: Ldap dn in which to search for users. Default `ou=people, + ou=example,ou=test` + + +[db] +```` + +:uri: Database uri for SQLAlchemy. See `Flask-SQLAlchemy `_ + docs for details. Default `postgres://orlo:password@localhost:5432/orlo` +:echo_queries: Whether or not to echo sql queries to log. Default `false`. + +[logging] +````````` + +:level: Logging level, valid values `debug`, `info`, `warning`, `error`. + Default `info`. +:file: Log file to use. If not set, logging only goes to stdout. + +[deploy] +```````` + +:timeout: How long to wait for the deploy script to complete in seconds. If it + does not complete within this time, the deploy is considered failed and + an exception is raised. Default `3600`. + +[deploy_shell] +`````````````` + +:command_path: Path to the deployment script. Defaults to Orlo's test deployer. diff --git a/docs/index.rst b/docs/index.rst index 3c3412b..3beb449 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. Orlo -====== +==== Orlo is an API for tracking deployments, written with Python, Flask and SqlAlchemy. @@ -18,11 +18,12 @@ Contents :maxdepth: 2 install + config rest About Orlo ------------- +---------- Orlo aims to cover the needs of all eCG platforms with respect to gathering information about deployments, while being simple to integrate with existing deployment software and scripts. This currently includes: @@ -39,7 +40,7 @@ With this information, it will be possible to build dashboards and more intellig The API should also be agnostic to release process, server container or packaging format - all platforms do things differently. It should be forgiving and "do the right thing" in the case of missing data, as not all platforms will use every field. Why "orlo"? -------------- +----------- Originally this project was called "Sponge", because sponges are absorbent. But it turns out that name was already in use on readthedocs.org, so it was renamed. In English, orlo means "a plinth supporting the base of a column". diff --git a/docs/install.rst b/docs/install.rst index eeade3f..6019b53 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -4,7 +4,15 @@ Installation **This is a rough draft** This documentation is in an early state and your mileage may vary, please raise issues on github for any problems you encounter. -Currently, we do not publish packages to pip, thus the only way is currently from source, but this may change when the project reaches maturity. +From pip +-------- +:: + + pip install orlo + + +The pip build is not always up to date as things are still in flux. At this early stage, it is recommended to install from source. + From source ----------- @@ -18,6 +26,14 @@ Clone the github repo: python setup.py install +Or: + +:: + + pip install https://github.com/eBayClassifiedsGroup/orlo.git + + + Developing in Vagrant --------------------- @@ -111,7 +127,7 @@ If installed from the debian package run orlo setup_database -If you are running from source the orlo script can be found in ./bin, but you may need to invoke it explicity from your python interpreter, as it is hard-coded to the dh-virtualenv path. +If you are running from source the orlo script can be found in ./bin, but you may need to invoke it explicitly from your python interpreter, as it is hard-coded to the dh-virtualenv path. Running under Gunicorn @@ -133,4 +149,4 @@ And it should return 'pong' Nginx Setup ----------- -We strongly recommend running orlo behind a proxy such as nginx. An example configuration is provided under ./etc/ +We strongly recommend running orlo behind a proxy such as nginx, with TLS if you plan to use authentication. An example configuration is provided under ./etc/ From b5f3c5a9d769fe50fd1a1dd12bbd6edbdb98b8b8 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 24 May 2016 13:04:32 +0100 Subject: [PATCH 18/36] Break out ExecStart command into multiple lines for clarity --- debian/systemd/orlo.service | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/systemd/orlo.service b/debian/systemd/orlo.service index 73ee3bd..46860b2 100644 --- a/debian/systemd/orlo.service +++ b/debian/systemd/orlo.service @@ -9,7 +9,13 @@ ConditionPathExists=/usr/share/python/orlo/bin/gunicorn Type=simple User=orlo Group=orlo -ExecStart=/usr/share/python/orlo/bin/gunicorn -w 4 -b 127.0.0.1:8080 orlo:app --access-logfile /var/log/orlo/gunicorn-access.log --log-level debug --error-logfile /var/log/orlo/gunicorn-error.log --log-file /var/log/orlo/gunicorn.log +ExecStart=/usr/share/python/orlo/bin/gunicorn \ + -w 4 -b 127.0.0.1:8080 \ + --access-logfile /var/log/orlo/gunicorn-access.log \ + --log-level debug \ + --error-logfile /var/log/orlo/gunicorn-error.log \ + --log-file /var/log/orlo/gunicorn.log \ + orlo:app [Install] WantedBy=multi-user.target From f39c0fe1f00fff1146ff7b27101a253ec07b80e7 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 24 May 2016 13:05:08 +0100 Subject: [PATCH 19/36] Add note to test_auth file Can have trouble finding the passwd file when using an installed module --- tests/test_auth.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_auth.py b/tests/test_auth.py index b604085..cef174f 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -17,6 +17,16 @@ PASSWORD = 'blah' +""" +IF YOU GET FAILURES IN THIS FILE + +E.g: +IOError: [Errno 2] No such file or directory: '/home/vagrant/virtualenv/orlo/local/lib/python2.7/site-packages/orlo-0.2.0_pre5-py2.7.egg/orlo/../etc/passwd' + +This is because you're using an installed module and not actually testing +your changes! Uninstall the installed module before testing. +""" + # A couple of test routes which require auth, and return 200 if it succeeds @orlo.app.route('/test/token_required') @conditional_auth(token_auth.token_required) From 57e115edcd746662083362428bcba8b58e0b5763 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 24 May 2016 13:07:13 +0100 Subject: [PATCH 20/36] Make config file configurable by environment variable Add config for logging format --- orlo/__init__.py | 46 +++++++++++++++++++++++++++++++--------------- orlo/cli.py | 6 +++--- orlo/config.py | 16 ++++++++++++---- orlo/queries.py | 6 +++--- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/orlo/__init__.py b/orlo/__init__.py index 98587ff..5fb1677 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -1,8 +1,9 @@ from flask import Flask import logging from logging.handlers import RotatingFileHandler +from logging import Formatter -from orlo.config import config +from orlo.config import config, CONFIG_FILE from orlo.exceptions import OrloStartupError try: @@ -23,30 +24,45 @@ if config.getboolean('db', 'echo_queries'): app.config['SQLALCHEMY_ECHO'] = True +if not config.getboolean('main', 'strict_slashes'): + app.url_map.strict_slashes = False + # Debug mode ignores all custom logging and should only be used in # local testing... if config.getboolean('main', 'debug_mode'): app.debug = True -# ...as opposed to loglevel debug, which can be used anywhere -if config.getboolean('logging', 'debug'): - app.logger.setLevel(logging.DEBUG) +if not app.debug: + log_level = config.get('logging', 'level') + if log_level == 'debug': + app.logger.setLevel(logging.DEBUG) + elif log_level == 'info': + app.logger.setLevel(logging.INFO) + elif log_level == 'warning': + app.logger.setLevel(logging.WARNING) + elif log_level == 'error': + app.logger.setLevel(logging.ERROR) -if not config.getboolean('main', 'strict_slashes'): - app.url_map.strict_slashes = False + logfile = config.get('logging', 'file') + if logfile != 'disabled': + file_handler = RotatingFileHandler( + logfile, + maxBytes=1048576, + backupCount=1, + ) + log_format = config.get('logging', 'format') + formatter = Formatter(log_format) -logfile = config.get('logging', 'file') -if logfile != 'disabled': - handler = RotatingFileHandler( - logfile, - maxBytes=1048576, - backupCount=1, - ) - app.logger.addHandler(handler) + file_handler.setFormatter(formatter) + app.logger.addHandler(file_handler) if config.getboolean('security', 'enabled') and \ config.get('security', 'secret_key') == 'change_me': - raise OrloStartupError("Security is enabled, please configure the secret key") + raise OrloStartupError( + "Security is enabled, please configure the secret key") + +app.logger.info("Startup completed with configuration from {}".format( + CONFIG_FILE)) # Must be imported last diff --git a/orlo/cli.py b/orlo/cli.py index 168b4b5..59678c1 100644 --- a/orlo/cli.py +++ b/orlo/cli.py @@ -25,7 +25,7 @@ def parse_args(): p_config = argparse.ArgumentParser(add_help=False) p_config.add_argument('--file', '-f', dest='file_path', - help="File to write to", + help="Config file to read/write", default='/etc/orlo/orlo.ini') p_database = argparse.ArgumentParser(add_help=False) @@ -43,12 +43,12 @@ def parse_args(): sp_database = subparsers.add_parser( 'setup_database', help="Initialise the configured DB", - parents=[p_database]) + parents=[p_database, p_config]) sp_database.set_defaults(func=setup_database) sp_run_server = subparsers.add_parser( 'run_server', help="Run a test server", - parents=[p_server]) + parents=[p_server, p_config]) sp_run_server.set_defaults(func=run_server) return parser.parse_args() diff --git a/orlo/config.py b/orlo/config.py index 03829d0..d9d7877 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -4,10 +4,15 @@ __author__ = 'alforbes' -config = ConfigParser.ConfigParser() +try: + CONFIG_FILE = os.environ['ORLO_CONFIG'] +except KeyError: + CONFIG_FILE = '/etc/orlo/orlo.ini' + +config = ConfigParser.RawConfigParser() config.add_section('main') -config.set('main', 'debug_mode', 'true') +config.set('main', 'debug_mode', 'false') config.set('main', 'propagate_exceptions', 'true') config.set('main', 'time_format', '%Y-%m-%dT%H:%M:%SZ') config.set('main', 'time_zone', 'UTC') @@ -32,8 +37,11 @@ config.set('db', 'echo_queries', 'false') config.add_section('logging') -config.set('logging', 'debug', 'false') +config.set('logging', 'level', 'info') config.set('logging', 'file', 'disabled') +config.set('logging', 'format', '%(asctime)s [main] %(levelname)s %(' + 'module)s:%(funcName)s:%(lineno)d - %(' + 'message)s') config.add_section('deploy') config.set('deploy', 'timeout', @@ -44,4 +52,4 @@ os.path.dirname(os.path.abspath(__file__)) + '/../deployer.py') -config.read('/etc/orlo/orlo.ini') +config.read(CONFIG_FILE) diff --git a/orlo/queries.py b/orlo/queries.py index 88a92f0..5f09c62 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -362,9 +362,9 @@ def package_versions(platform=None): """ List the current version of all packages - It is not sufficient to just return the highest version of each - successful package, - as they can be rolled back, so we determine the version by last release time + It is not sufficient to just return the highest version of each successful + package, as they can be rolled back, so we determine the version by last + release time :param platform: Platform to filter on """ From 366753d87e46179dfcb192e5fd86e4b1dd580011 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 24 May 2016 15:56:01 +0100 Subject: [PATCH 21/36] Include logger name in logs --- orlo/__init__.py | 7 ++++--- orlo/config.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/orlo/__init__.py b/orlo/__init__.py index 5fb1677..e2d26c7 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -61,9 +61,7 @@ raise OrloStartupError( "Security is enabled, please configure the secret key") -app.logger.info("Startup completed with configuration from {}".format( - CONFIG_FILE)) - +app.logger.debug("Log level: {}".format(config.get('logging', 'level'))) # Must be imported last import orlo.error_handlers @@ -74,3 +72,6 @@ import orlo.route_stats import orlo.user_auth +app.logger.info("Startup completed with configuration from {}".format( + CONFIG_FILE)) + diff --git a/orlo/config.py b/orlo/config.py index d9d7877..69a3a1f 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -39,7 +39,7 @@ config.add_section('logging') config.set('logging', 'level', 'info') config.set('logging', 'file', 'disabled') -config.set('logging', 'format', '%(asctime)s [main] %(levelname)s %(' +config.set('logging', 'format', '%(asctime)s [%(name)s] %(levelname)s %(' 'module)s:%(funcName)s:%(lineno)d - %(' 'message)s') From 5a2730acfd8312ec8fc421f0f4321d69dd00d1e5 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 25 May 2016 15:12:07 +0100 Subject: [PATCH 22/36] Update requirements for py3, use >= where possible --- requirements.txt | 1 + requirements_testing.txt | 26 +++++++++++++------------- tests/test_auth.py | 3 ++- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/requirements.txt b/requirements.txt index d774bb7..7f1b1e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ passlib==1.6.1 psycopg2==2.6.1 python-ldap>=2.4.25 pytz +six diff --git a/requirements_testing.txt b/requirements_testing.txt index 70066f0..8d51661 100644 --- a/requirements_testing.txt +++ b/requirements_testing.txt @@ -1,24 +1,24 @@ Flask-HTTPAuth>=2.7.1 Flask-Migrate>=1.5.1 Flask-SQLAlchemy>=2.0 -Flask-Testing>=0.4.2 Flask-TokenAuth>=0.0.2 Flask>=0.10.1 -Jinja2==2.7.1 -MarkupSafe==0.18 -SQLAlchemy==1.0.8 -Werkzeug==0.9.4 -arrow==0.7.0 +Jinja2>=2.7.1 +MarkupSafe>=0.18 +SQLAlchemy>=1.0.8 +Werkzeug>=0.9.4 +arrow>=0.7.0 gunicorn -itsdangerous==0.23 +itsdangerous>=0.23 orloclient>=0.1.1 -passlib==1.6.1 -psycopg2==2.6.1 +passlib>=1.6.1 +psycopg2>=2.6.1 pytest pytz requests>=2.4.2 -sqlalchemy-utils==0.31.0 +sqlalchemy-utils>=0.31.0 mock==1.3.0 -mockldap==0.2.6 -python-ldap==2.4.25 -six==1.10.0 +mockldap>=0.2.6 +pyldap>=2.4.25 +six>=1.10.0 +git+https://github.com/jarus/flask-testing.git diff --git a/tests/test_auth.py b/tests/test_auth.py index cef174f..7bf3d8c 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -24,7 +24,8 @@ IOError: [Errno 2] No such file or directory: '/home/vagrant/virtualenv/orlo/local/lib/python2.7/site-packages/orlo-0.2.0_pre5-py2.7.egg/orlo/../etc/passwd' This is because you're using an installed module and not actually testing -your changes! Uninstall the installed module before testing. +your changes! Uninstall the installed module before testing, and use python +setup.py develop """ # A couple of test routes which require auth, and return 200 if it succeeds From e130941d006431b6dd5f0d035256c7ab2a2d7337 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 25 May 2016 15:13:45 +0100 Subject: [PATCH 23/36] Remove indirect dependencies --- requirements.txt | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7f1b1e9..546c0b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,14 @@ gunicorn -arrow==0.7.0 +arrow>=0.7.0 Flask>=0.10.1 Flask-SQLAlchemy>=2.0 Flask-Migrate>=1.5.1 Flask-HTTPAuth>=2.7.1 Flask-TokenAuth>=0.0.2 requests>=2.4.2 -SQLAlchemy==1.0.8 -sqlalchemy-utils==0.31.0 -Jinja2==2.7.1 -MarkupSafe==0.18 -Werkzeug==0.9.4 -itsdangerous==0.23 -passlib==1.6.1 -psycopg2==2.6.1 +SQLAlchemy>=1.0.8 +sqlalchemy-utils>=0.31.0 +psycopg2>=2.6.1 python-ldap>=2.4.25 pytz six From f067f6f6aa77960cdcc0224863dc58cbb8c77eed Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 25 May 2016 15:14:01 +0100 Subject: [PATCH 24/36] Update Vagrantfile to Ubuntu Xenial This is still a bit early, xenial isn't brilliant on vagrant --- Vagrantfile | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 8dbae63..73ec6fa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,7 +12,7 @@ Vagrant.configure(2) do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://atlas.hashicorp.com/search. - config.vm.box = "ubuntu/trusty64" + config.vm.box = "ubuntu/xenial64" # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs @@ -71,6 +71,9 @@ Vagrant.configure(2) do |config| # sudo apt-get install -y apache2 # SHELL config.vm.provision "shell", inline: <<-SHELL + # ubuntu 16.04 fix see https://github.com/mitchellh/vagrant/issues/7288 + echo 127.0.0.1 `hostname` |sudo tee -a /etc/hosts + # sudo sed -i 's/archive.ubuntu.com/nl.archive.ubuntu.com/g' /etc/apt/sources.list apt-get update apt-get -y install python-pip python-dev postgresql postgresql-server-dev-all @@ -81,32 +84,31 @@ Vagrant.configure(2) do |config| apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev # Build tools - apt-get -y install build-essential git-buildpackage debhelper python-dev dh-systemd python-virtualenv + apt-get -y install build-essential git-buildpackage debhelper python-dev \ + python3-dev dh-systemd python-virtualenv wget -P /tmp/ \ 'https://launchpad.net/ubuntu/+archive/primary/+files/dh-virtualenv_0.11-1_all.deb' dpkg -i /tmp/dh-virtualenv_0.11-1_all.deb apt-get -f install -y - pip install --upgrade pip - pip install virtualenv + pip install --upgrade pip setuptools + pip install --upgrade virtualenv # Virtualenv is to avoid conflict with Debian's python-six - virtualenv /home/vagrant/virtualenv/orlo - source /home/vagrant/virtualenv/orlo/bin/activate - echo "source ~/virtualenv/orlo/bin/activate" >> /home/vagrant/.profile + virtualenv /home/ubuntu/virtualenv/orlo + source /home/ubuntu/virtualenv/orlo/bin/activate + echo "source ~/virtualenv/orlo/bin/activate" >> /home/ubuntu/.profile pip install -r /vagrant/requirements.txt pip install -r /vagrant/requirements_testing.txt pip install -r /vagrant/docs/requirements.txt - # Flask-Testing hasn't been released to pip in ages :( - pip install --upgrade git+https://github.com/jarus/flask-testing.git - - sudo chown -R vagrant:vagrant /home/vagrant/virtualenv + sudo chown -R ubuntu:ubuntu /home/ubuntu/virtualenv # Create the database python /vagrant/create_db.py + python /vagrant/setup.py develop mkdir /etc/orlo - chown vagrant:root /etc/orlo + chown ubuntu:root /etc/orlo SHELL end From 4422e81875fba20578ee88ef64c16352172cad86 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 25 May 2016 15:47:12 +0100 Subject: [PATCH 25/36] Remove indirect imports --- requirements.txt | 4 ++-- requirements_testing.txt | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 546c0b4..591387a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ requests>=2.4.2 SQLAlchemy>=1.0.8 sqlalchemy-utils>=0.31.0 psycopg2>=2.6.1 -python-ldap>=2.4.25 +pyldap>=2.4.25 pytz -six +six>=1.10.0 diff --git a/requirements_testing.txt b/requirements_testing.txt index 8d51661..9500f28 100644 --- a/requirements_testing.txt +++ b/requirements_testing.txt @@ -3,13 +3,9 @@ Flask-Migrate>=1.5.1 Flask-SQLAlchemy>=2.0 Flask-TokenAuth>=0.0.2 Flask>=0.10.1 -Jinja2>=2.7.1 -MarkupSafe>=0.18 SQLAlchemy>=1.0.8 -Werkzeug>=0.9.4 arrow>=0.7.0 gunicorn -itsdangerous>=0.23 orloclient>=0.1.1 passlib>=1.6.1 psycopg2>=2.6.1 From 36820425e79cea9a85039fd1b3552f300401fda5 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 25 May 2016 15:47:34 +0100 Subject: [PATCH 26/36] [py3] fix import errors Use six for RawConfigParser --- orlo/__init__.py | 2 ++ orlo/cli.py | 8 ++++---- orlo/config.py | 4 ++-- orlo/route_stats.py | 3 +-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/orlo/__init__.py b/orlo/__init__.py index e2d26c7..cde1cd7 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -1,3 +1,5 @@ +from __future__ import print_function, division, absolute_import +from __future__ import unicode_literals from flask import Flask import logging from logging.handlers import RotatingFileHandler diff --git a/orlo/cli.py b/orlo/cli.py index 59678c1..723be70 100644 --- a/orlo/cli.py +++ b/orlo/cli.py @@ -18,10 +18,10 @@ def parse_args(): - parser = argparse.ArgumentParser( - prog='orlo', - version=__version__, - ) + parser = argparse.ArgumentParser(prog='orlo') + + parser.add_argument('--version', '-v', action='version', + version='%(prog)s {}'.format(__version__)) p_config = argparse.ArgumentParser(add_help=False) p_config.add_argument('--file', '-f', dest='file_path', diff --git a/orlo/config.py b/orlo/config.py index 69a3a1f..148faf1 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -1,6 +1,6 @@ from __future__ import print_function -import ConfigParser import os +from six.moves.configparser import RawConfigParser __author__ = 'alforbes' @@ -9,7 +9,7 @@ except KeyError: CONFIG_FILE = '/etc/orlo/orlo.ini' -config = ConfigParser.RawConfigParser() +config = RawConfigParser() config.add_section('main') config.set('main', 'debug_mode', 'false') diff --git a/orlo/route_stats.py b/orlo/route_stats.py index 8d40230..41f4019 100644 --- a/orlo/route_stats.py +++ b/orlo/route_stats.py @@ -1,8 +1,7 @@ -from __future__ import print_function import arrow from flask import request, jsonify -import stats +from orlo import stats from orlo import app from orlo.exceptions import InvalidUsage import orlo.queries as queries From 1483dad2f4b373395dcccdd320d846f3c8c315a2 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 26 May 2016 10:56:07 +0100 Subject: [PATCH 27/36] Fix tests to run under python3 --- orlo/deploy.py | 2 +- orlo/orm.py | 8 ++---- orlo/queries.py | 4 +-- orlo/route_import.py | 3 +- orlo/route_releases.py | 2 +- tests/test_auth.py | 58 +++++++++++++------------------------- tests/test_contract.py | 6 ++-- tests/test_deploy.py | 2 +- tests/test_orm.py | 11 ++++---- tests/test_route_import.py | 2 +- 10 files changed, 39 insertions(+), 59 deletions(-) diff --git a/orlo/deploy.py b/orlo/deploy.py index 3050e09..166f9a0 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -140,7 +140,7 @@ def run_command(args, env, in_data, timeout_sec=3600): out = err = " " try: timer.start() - out, err = proc.communicate(in_data) + out, err = proc.communicate(in_data.encode('utf-8')) finally: timer.cancel() app.logger.debug("Out:\n{}".format(out)) diff --git a/orlo/orm.py b/orlo/orm.py index 80f7775..30d0702 100644 --- a/orlo/orm.py +++ b/orlo/orm.py @@ -1,5 +1,3 @@ -from __future__ import print_function, unicode_literals - from flask.ext.sqlalchemy import SQLAlchemy from sqlalchemy_utils.types.uuid import UUIDType from sqlalchemy_utils.types.arrow import ArrowType @@ -82,7 +80,7 @@ def __init__(self, platforms, user, team=None, references=None): self.start() def __str__(self): - return unicode(self.to_dict()) + return self.to_dict() def to_dict(self): time_format = config.get('main', 'time_format') @@ -92,7 +90,7 @@ def to_dict(self): metadata.update(m.to_dict()) return { - 'id': unicode(self.id), + 'id': str(self.id), 'packages': [p.to_dict() for p in self.packages], 'platforms': [platform.name for platform in self.platforms], 'references': string_to_list(self.references), @@ -176,7 +174,7 @@ def stop(self, success): def to_dict(self): time_format = config.get('main', 'time_format') return { - 'id': unicode(self.id), + 'id': str(self.id), 'name': self.name, 'version': self.version, 'stime': self.stime.strftime(config.get('main', 'time_format')) if self.stime else None, diff --git a/orlo/queries.py b/orlo/queries.py index 5f09c62..02d8e8f 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -88,7 +88,7 @@ def apply_filters(query, args): :return: filtered query object """ - for field, value in args.iteritems(): + for field, value in args.items(): if field == 'latest': # this is not a comparison continue @@ -194,7 +194,7 @@ def releases(**kwargs): query = apply_filters(query, kwargs) except AttributeError as e: raise InvalidUsage( - "An invalid field was specified: {}".format(e.message)) + "An invalid field was specified: {}".format(e.args[0])) if desc: stime_field = Release.stime.desc diff --git a/orlo/route_import.py b/orlo/route_import.py index fd8838a..360fee4 100644 --- a/orlo/route_import.py +++ b/orlo/route_import.py @@ -1,4 +1,3 @@ -from __future__ import print_function import arrow import json from flask import jsonify, request @@ -146,4 +145,4 @@ def post_import(): releases.append(release.id) - return jsonify({'releases': [unicode(x) for x in releases]}), 200 + return jsonify({'releases': [str(x) for x in releases]}), 200 diff --git a/orlo/route_releases.py b/orlo/route_releases.py index b027eb8..4d79c29 100644 --- a/orlo/route_releases.py +++ b/orlo/route_releases.py @@ -365,7 +365,7 @@ def get_releases(release_id=None): if not is_uuid(release_id): raise InvalidUsage("Release ID given is not a valid UUID") query = queries.get_release(release_id) - elif len(request.args.keys()) == 0: + elif len([x for x in request.args.keys()]) == 0: raise InvalidUsage("Please specify a filter. See " "http://orlo.readthedocs.org/en/latest/rest.html" "#get--releases for more info") diff --git a/tests/test_auth.py b/tests/test_auth.py index 7bf3d8c..ccbc606 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -17,17 +17,6 @@ PASSWORD = 'blah' -""" -IF YOU GET FAILURES IN THIS FILE - -E.g: -IOError: [Errno 2] No such file or directory: '/home/vagrant/virtualenv/orlo/local/lib/python2.7/site-packages/orlo-0.2.0_pre5-py2.7.egg/orlo/../etc/passwd' - -This is because you're using an installed module and not actually testing -your changes! Uninstall the installed module before testing, and use python -setup.py develop -""" - # A couple of test routes which require auth, and return 200 if it succeeds @orlo.app.route('/test/token_required') @conditional_auth(token_auth.token_required) @@ -58,7 +47,8 @@ class OrloAuthTest(TestCase): top = ('o=test', {'o': ['test']}) example = ('ou=example,o=test', {'ou': ['example']}) people = ('ou=people,ou=example,o=test', {'ou': ['other']}) - ldapuser = ('uid=ldapuser,ou=people,ou=example,o=test', {'uid': ['ldapuser'], 'userPassword': ['ldapuserpw']}) + ldapuser = ('uid=ldapuser,ou=people,ou=example,o=test', + {'uid': ['ldapuser'], 'userPassword': ['ldapuserpw']}) # This is the content of our mock LDAP directory. It takes the form # {dn: {attr: [value, ...], ...}, ...}. directory = dict([top, example, people, ldapuser]) @@ -88,15 +78,21 @@ def setUp(self): self.mockldap.start() self.ldapobj = self.mockldap['ldap://localhost/'] self.orig_security_enabled = orlo.config.get('security', 'enabled') - self.orig_security_secret_key = orlo.config.set('security', 'secret_key') - self.orig_security_ldap_server = orlo.config.set('security', 'ldap_server') + self.orig_security_secret_key = orlo.config.set('security', + 'secret_key') + self.orig_security_ldap_server = orlo.config.set('security', + 'ldap_server') self.orig_security_ldap_port = orlo.config.set('security', 'ldap_port') - self.orig_security_user_base_dn = orlo.config.set('security', 'user_base_dn') + self.orig_security_user_base_dn = orlo.config.set('security', + 'user_base_dn') orlo.config.set('security', 'enabled', 'true') - orlo.config.set('security', 'secret_key', 'It does not matter how slowly you go so long as you do not stop') + orlo.config.set('security', 'secret_key', 'It does not matter how ' + 'slowly you go so long as ' + 'you do not stop') orlo.config.set('security', 'ldap_server', 'localhost') orlo.config.set('security', 'ldap_port', '389') - orlo.config.set('security', 'user_base_dn', 'ou=people,ou=example,o=test') + orlo.config.set('security', 'user_base_dn', + 'ou=people,ou=example,o=test') def tearDown(self): db.session.remove() @@ -107,21 +103,6 @@ def tearDown(self): orlo.config.set('security', 'secret_key', self.orig_security_secret_key) def get_with_basic_auth(self, path, username='testuser', password='blah'): - """ - Do a request with basic auth - - :param path: - :param username: - :param password: - """ - h = Headers() - h.add('Authorization', 'Basic ' + base64.b64encode( - '{u}:{p}'.format(u=username, p=password) - )) - response = Client.open(self.client, path=path, headers=h) - return response - - def get_with_ldap_auth(self, path, username='ldapuser', password='ldapuserpw'): """ Do a request with ldap auth @@ -130,9 +111,9 @@ def get_with_ldap_auth(self, path, username='ldapuser', password='ldapuserpw'): :param password: """ h = Headers() - h.add('Authorization', 'Basic ' + base64.b64encode( - '{u}:{p}'.format(u=username, p=password) - )) + s_auth = base64.b64encode('{u}:{p}'.format( + u=username, p=password).encode('utf-8')) + h.add('Authorization', 'Basic ' + s_auth.decode('utf-8')) response = Client.open(self.client, path=path, headers=h) return response @@ -158,7 +139,8 @@ def post_with_token_auth(self, path, token, data): h = Headers() h.add('X-Auth-Token', token) h.add('Content-Type', 'application/json') - response = Client.open(self.client, method='POST', data=data, path=path, headers=h) + response = Client.open(self.client, method='POST', data=data, path=path, + headers=h) return response def get_token(self): @@ -227,7 +209,7 @@ def test_with_login(self): self.assert200(response) def test_with_ldap_login(self): - response = self.get_with_ldap_auth( + response = self.get_with_basic_auth( '/test/auth_required', username='ldapuser', password='ldapuserpw' ) self.assert200(response) @@ -271,4 +253,4 @@ def test_post_releases_with_token_returns_400(self): self.URL_PATH, token=token, data={'foo': 'bar'}, ) self.assert400(response) - self.assertIn('message', response.data) + self.assertIn(b'message', response.data) diff --git a/tests/test_contract.py b/tests/test_contract.py index 5ceead5..1950b94 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -181,7 +181,7 @@ def test_create_release(self): Create a release """ release_id = self._create_release() - self.assertEqual(uuid.UUID(release_id).get_version(), 4) + self.assertEqual(uuid.UUID(release_id).version, 4) def test_create_package(self): """ @@ -189,7 +189,7 @@ def test_create_package(self): """ release_id = self._create_release() package_id = self._create_package(release_id) - self.assertEqual(uuid.UUID(package_id).get_version(), 4) + self.assertEqual(uuid.UUID(package_id).version, 4) def test_add_results(self): """ @@ -357,7 +357,7 @@ def _get_releases(self, release_id=None, filters=None, expected_status=200): except AssertionError as err: print(results_response.data) raise - r_json = json.loads(results_response.data) + r_json = json.loads(results_response.data.decode('utf-8')) return r_json def test_ping(self): diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 4efc391..cc23d59 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -176,5 +176,5 @@ def test_output(self): deploy.server_url = self.get_server_url() output = deploy.start() - self.assertEqual(output['stdout'], 'test-package=1.2.3\n') + self.assertEqual(output['stdout'], b'test-package=1.2.3\n') diff --git a/tests/test_orm.py b/tests/test_orm.py index 3bbe6b4..0dd83db 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -9,6 +9,7 @@ import arrow import datetime import uuid +from six import string_types __author__ = 'alforbes' @@ -151,12 +152,12 @@ def test_release_types(self): self.assertIs(type(r.id), uuid.UUID) self.assertIs(hasattr(r.notes, '__iter__'), True) self.assertIs(hasattr(r.platforms, '__iter__'), True) - self.assertIs(type(r.references), unicode) + self.assertIn(type(r.references), string_types) self.assertIs(type(list(r.references)), list) self.assertIs(type(r.stime), arrow.arrow.Arrow) self.assertIs(type(r.ftime), arrow.arrow.Arrow) self.assertIs(type(r.duration), datetime.timedelta) - self.assertIs(type(r.team), unicode) + self.assertIn(type(r.team), string_types) def test_package_types(self): """ @@ -165,9 +166,9 @@ def test_package_types(self): p = db.session.query(Package).first() x = db.session.query(Package).all() self.assertIs(type(p.id), uuid.UUID) - self.assertIs(type(p.name), unicode) + self.assertIn(type(p.name), string_types) self.assertIs(type(p.stime), arrow.arrow.Arrow) self.assertIs(type(p.ftime), arrow.arrow.Arrow) self.assertIs(type(p.duration), datetime.timedelta) - self.assertIs(type(p.status), unicode) - self.assertIs(type(p.version), unicode) + self.assertIn(type(p.status), string_types) + self.assertIn(type(p.version), string_types) diff --git a/tests/test_route_import.py b/tests/test_route_import.py index 22e1340..fc8f259 100644 --- a/tests/test_route_import.py +++ b/tests/test_route_import.py @@ -84,7 +84,7 @@ def test_import_param_user(self): """ Test import user matches """ - self.assertEqual(self.release.user, unicode(self.doc_dict[0]['user'])) + self.assertEqual(self.release.user, self.doc_dict[0]['user']) def test_import_param_package_name(self): """ From 134e18ed3c4c3b6cb4ef0d9c303df50d842e61fe Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 26 May 2016 11:05:51 +0100 Subject: [PATCH 28/36] Fix for python2 again Convert to assertIsInstance, much more sensible --- tests/test_orm.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_orm.py b/tests/test_orm.py index 0dd83db..01ce3b0 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -149,15 +149,15 @@ def test_release_types(self): Test the types returned by Release objects are OK """ r = db.session.query(Release).first() - self.assertIs(type(r.id), uuid.UUID) + self.assertIsInstance(r.id, uuid.UUID) self.assertIs(hasattr(r.notes, '__iter__'), True) self.assertIs(hasattr(r.platforms, '__iter__'), True) - self.assertIn(type(r.references), string_types) + self.assertIsInstance(r.references, string_types) self.assertIs(type(list(r.references)), list) - self.assertIs(type(r.stime), arrow.arrow.Arrow) - self.assertIs(type(r.ftime), arrow.arrow.Arrow) - self.assertIs(type(r.duration), datetime.timedelta) - self.assertIn(type(r.team), string_types) + self.assertIsInstance(r.stime, arrow.arrow.Arrow) + self.assertIsInstance(r.ftime, arrow.arrow.Arrow) + self.assertIsInstance(r.duration, datetime.timedelta) + self.assertIsInstance(r.team, string_types) def test_package_types(self): """ @@ -165,10 +165,10 @@ def test_package_types(self): """ p = db.session.query(Package).first() x = db.session.query(Package).all() - self.assertIs(type(p.id), uuid.UUID) - self.assertIn(type(p.name), string_types) - self.assertIs(type(p.stime), arrow.arrow.Arrow) - self.assertIs(type(p.ftime), arrow.arrow.Arrow) - self.assertIs(type(p.duration), datetime.timedelta) - self.assertIn(type(p.status), string_types) - self.assertIn(type(p.version), string_types) + self.assertIsInstance(p.id, uuid.UUID) + self.assertIsInstance(p.name, string_types) + self.assertIsInstance(p.stime, arrow.arrow.Arrow) + self.assertIsInstance(p.ftime, arrow.arrow.Arrow) + self.assertIsInstance(p.duration, datetime.timedelta) + self.assertIsInstance(p.status, string_types) + self.assertIsInstance(p.version, string_types) From 55ac4d1dad4528805e87af9b4a360ce2b956cf0b Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 26 May 2016 14:46:15 +0100 Subject: [PATCH 29/36] Add own fork of flask-testing --- requirements_testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_testing.txt b/requirements_testing.txt index 9500f28..0382455 100644 --- a/requirements_testing.txt +++ b/requirements_testing.txt @@ -17,4 +17,4 @@ mock==1.3.0 mockldap>=0.2.6 pyldap>=2.4.25 six>=1.10.0 -git+https://github.com/jarus/flask-testing.git +git+https://github.com/al4/flask-testing.git From 370a7a6c2778bcda77cdbde738be47eb8e5c360e Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 26 May 2016 17:44:48 +0100 Subject: [PATCH 30/36] Fix /versions sometimes returning wrong version The subquery was only matching on stime, so if any packages were released at the same time they could match, thus potentially (but consistently) returning an old version. --- orlo/__init__.py | 5 +++-- orlo/exceptions.py | 2 ++ orlo/queries.py | 17 +++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/orlo/__init__.py b/orlo/__init__.py index cde1cd7..49bd07a 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -4,9 +4,11 @@ import logging from logging.handlers import RotatingFileHandler from logging import Formatter +import sys from orlo.config import config, CONFIG_FILE -from orlo.exceptions import OrloStartupError +from orlo.exceptions import OrloStartupError, OrloError, OrloAuthError, \ + OrloConfigError try: # _version is created by setup.py @@ -76,4 +78,3 @@ app.logger.info("Startup completed with configuration from {}".format( CONFIG_FILE)) - diff --git a/orlo/exceptions.py b/orlo/exceptions.py index c25ddc9..c3fb4c2 100644 --- a/orlo/exceptions.py +++ b/orlo/exceptions.py @@ -1,4 +1,6 @@ from __future__ import print_function +import logging +import sys __author__ = 'alforbes' diff --git a/orlo/queries.py b/orlo/queries.py index 02d8e8f..30d4261 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -4,7 +4,7 @@ from orlo import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage -from sqlalchemy import exc +from sqlalchemy import and_, exc __author__ = 'alforbes' @@ -372,9 +372,10 @@ def package_versions(platform=None): # Sub query gets a list of successful packages by last successful release # time sub_q = db.session.query( - Package.name, db.func.max(Package.stime).label('max_stime')) \ + Package.name.label('name'), + db.func.max(Package.stime).label('max_stime')) \ .filter(Package.status == 'SUCCESSFUL') - if platform: # filter releases not on this platform + if platform: # filter by platform sub_q = sub_q \ .join(Release) \ .filter(Release.platforms.any(Platform.name == platform)) @@ -384,17 +385,17 @@ def package_versions(platform=None): # q inner-joins Package table with sub_q to get the version q = db.session.query( - Package.name, - Package.version) \ - .join(sub_q, sub_q.c.max_stime == Package.stime) \ + Package.name, + Package.version) \ + .join(sub_q, and_(sub_q.c.max_stime == Package.stime, + sub_q.c.name == Package.name)) \ .group_by(Package.name, Package.version) return q def count_releases(user=None, package=None, team=None, platform=None, - status=None, - rollback=None, stime=None, ftime=None): + status=None, rollback=None, stime=None, ftime=None): """ Return the number of releases with the attributes specified From 76238dda70bf0cb1d983d606b4915ffb1a15436f Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 26 May 2016 16:50:12 +0000 Subject: [PATCH 31/36] Update changelog for 0.2.0-pre4 release --- debian/changelog | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/debian/changelog b/debian/changelog index 38d6506..37c4ce6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -75,6 +75,24 @@ orlo (0.2.0-pre4) UNRELEASED; urgency=medium * Log output of deploy * Use json.dumps instead of str() for reliable conversion * Return output of deploy to client + * Add .DS_Store to gitignore + * Add payload of arguments to error + * Add configuration to documentation + * Break out ExecStart command into multiple lines for clarity + * Add note to test_auth file + * Make config file configurable by environment variable + * Include logger name in logs + * Update requirements for py3, use >= where possible + * Remove indirect dependencies + * Update Vagrantfile to Ubuntu Xenial + * Remove indirect imports + * [py3] fix import errors + * Fix tests to run under python3 + * Fix for python2 again + * Add own fork of flask-testing + * Fix /versions sometimes returning wrong version + + [ Ubuntu ] -- Alex Forbes Thu, 14 Apr 2016 16:17:34 +0000 From fc29e8c734baf185c78344faa0a115cfe839a941 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 26 May 2016 18:03:16 +0100 Subject: [PATCH 32/36] Use python3 in debian package --- debian/changelog | 2 +- debian/rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 37c4ce6..7e67d69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -orlo (0.2.0-pre4) UNRELEASED; urgency=medium +orlo (0.2.0.pre5) UNRELEASED; urgency=medium [ Alex Forbes ] * Add dependant python virtualenv package to Vagrantfile diff --git a/debian/rules b/debian/rules index 1d28e71..764c311 100755 --- a/debian/rules +++ b/debian/rules @@ -4,4 +4,4 @@ dh $@ --with systemd --with python-virtualenv override_dh_virtualenv: - dh_virtualenv --no-test + dh_virtualenv --no-test --python /usr/bin/python3 From 7af60af3f042e390e96f0ef1f9343878cc3ab144 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 16 Jun 2016 15:19:07 +0100 Subject: [PATCH 33/36] Add pyldap dependency --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ffbd61f..97c5934 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre5' +VERSION = '0.2.0-pre6' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) @@ -35,6 +35,7 @@ 'arrow', 'gunicorn', 'psycopg2', + 'pyldap', 'pytz', 'sphinxcontrib-httpdomain', 'sqlalchemy-utils', From b4515267e26aca1044b7234f0e18c313b0ae4ef1 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 16 Jun 2016 15:19:18 +0100 Subject: [PATCH 34/36] Set back to python2 in debian build --- debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 764c311..d22413c 100755 --- a/debian/rules +++ b/debian/rules @@ -4,4 +4,4 @@ dh $@ --with systemd --with python-virtualenv override_dh_virtualenv: - dh_virtualenv --no-test --python /usr/bin/python3 + dh_virtualenv --no-test --python /usr/bin/python From 356edcaeba0157d997de7846c56b75c63a82ea7f Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 16 Jun 2016 15:27:50 +0100 Subject: [PATCH 35/36] Bump version to 0.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 97c5934..7994e6c 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import os -VERSION = '0.2.0-pre6' +VERSION = '0.2.0' my_path = os.path.dirname(os.path.realpath(__file__)) version_file = open('{}/orlo/_version.py'.format(my_path), 'w') version_file.write("__version__ = '{}'".format(VERSION)) From 1cb59ccf096bbc5ab8e53d3d97f585fdd2eda7b3 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Fri, 17 Jun 2016 11:19:00 +0100 Subject: [PATCH 36/36] Fix /info/packages tests failing Seems to be logic error introduced by sqlalchemy update --- orlo/route_info.py | 7 ++++--- tests/test_queries.py | 44 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/orlo/route_info.py b/orlo/route_info.py index 97647a4..8086e39 100644 --- a/orlo/route_info.py +++ b/orlo/route_info.py @@ -84,15 +84,16 @@ def info_packages(package=None): platform = request.args.get('platform') if package: - packages = queries.package_info(package) + packages = queries.package_info(package).all() else: - packages = queries.package_summary(platform=platform) + packages = queries.package_summary(platform=platform).all() d = {} for package, count in packages: d[package] = {'releases': count} - return jsonify(packages), 200 + + return jsonify(d), 200 @app.route('/info/packages/list', methods=['GET']) diff --git a/tests/test_queries.py b/tests/test_queries.py index dab958d..68b2cbc 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -6,6 +6,7 @@ import orlo.exceptions import orlo.stats from time import sleep +import sqlalchemy.orm __author__ = 'alforbes' @@ -183,6 +184,13 @@ def test_package_summary(self): self.assertIn('packageOne', packages) self.assertIn('packageTwo', packages) + def test_package_summary_returns_query(self): + rid1 = self._create_release() + self._create_package(rid1, name='packageOne') + + result = orlo.queries.package_summary() + self.assertIsInstance(result, sqlalchemy.orm.query.Query) + def test_package_summary_with_platform(self): """ Test package_summary @@ -320,20 +328,40 @@ def test_package_versions_with_rollback(self): self.assertIn(('packageOne', '1.0.1'), versions) +class TestInfo(OrloQueryTest): + """ + Test the _info functions + """ + + def _create_test_package(self): + rid = self._create_release(platforms=['platformOne']) + pid = self._create_package(rid, name='packageOne', version='1.0.1') + return pid + + def test_returns_query(self): + """ + Assert that package_info should return a query + """ + self._create_test_package() + result = orlo.queries.package_info('packageOne') + self.assertIsInstance(result, sqlalchemy.orm.query.Query) + + class TestCountReleases(OrloQueryTest): """ Parent class for testing the CountReleases function - By subclassing it and overriding ARGS, we can test different combinations of arguments - with the same test code. + By subclassing it and overriding ARGS, we can test different combinations of + arguments with the same test code. - INCLUSIVE_ARGS represents a set of arguments that will match the releases created (see - the functions in OrloQueryTest for what those are) - EXCLUSIVE_ARGS represents a set of arguments that will not match any releases created, - i.e. should return a count of zero + INCLUSIVE_ARGS represents a set of arguments that will match the releases + created (see the functions in OrloQueryTest for what those are) + EXCLUSIVE_ARGS represents a set of arguments that will not match any + releases created, i.e. should return a count of zero - This parent class has tests that should be the same result no matter what the arguments - (except the exclusive case which must always a count of zero so we define it here) + This parent class has tests that should be the same result no matter what + the arguments (except the exclusive case which must always a count of zero + so we define it here) """ INCLUSIVE_ARGS = {} # Args we are testing, result should include these