From 502f4b37e94617f6742e4f8cc60db18a1fed4e81 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 23 Nov 2016 17:39:06 +0000 Subject: [PATCH 01/38] =?UTF-8?q?Fix=20locale=20in=20vagrantfile=20(for=20?= =?UTF-8?q?me=20anyway=20=F0=9F=98=81),=20use=20tox=20for=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 4 ++-- Vagrantfile | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 68449f5..b61ce47 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # Uses git buildpackage, which from debian rules will call dh_virtualenv test: - python setup.py test + tox sdist: python setup.py sdist @@ -12,7 +12,7 @@ clean: python setup.py clean debuild clean -debian: +deb: debuild -us -uc changelog: diff --git a/Vagrantfile b/Vagrantfile index efcf28c..01d58f4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,6 +13,7 @@ Vagrant.configure(2) do |config| end config.vm.provision "shell", inline: <<-SHELL + echo 'en_GB.UTF-8 UTF-8' | tee -a /etc/locale.gen sudo localedef -i en_GB -f UTF-8 en_GB.UTF-8 sudo locale-gen en_GB.UTF-8 sudo sed -i 's/us.archive.ubuntu.com/nl.archive.ubuntu.com/g' /etc/apt/sources.list From 3977d215a662f1125ed9defc2b1a17ded949fe34 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 23 Nov 2016 17:39:53 +0000 Subject: [PATCH 02/38] Update changelog for 0.3.1-1 release We released an earlier build internally --- debian/changelog | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/debian/changelog b/debian/changelog index 02d24cd..bca3ce2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,38 @@ +orlo (0.3.1-1) jessie; urgency=medium + + [ Alex Forbes ] + * Fix test + * Rename __init__ to test_base and update imports + * Create multiple vagrant machines for testing + * Add jessie and xenial vagrant boxes for build/testing + * Run python setup.py clean in make clean + * Change deb install location to /opt/venvs + * Add initial tox.ini + * Default internal config should be sqlite + * Configure tests to use postgres in vagrant + * Remove trusty vm, fix IP + * Use main repos, upgrade dh-virtualenv, fix missing \ + * Add db clients and restart postgres + * Small doc fix + * Remove dependency on passwd file by mocking the verification function + * Use tox to run tests in travis + * mock has moved under py3 + * Set test database to localhost if running in Travis + * Set posargs for tox and maxfail for pytest + * Pass travis env car through to tox + * Rename DeployTest as LiveDbTest, move to test_base + * Add stress test + * Add liveserver config to app, refactor db test with classmethods + * Close the session in our json streamer to avoid leaking connections + * Test refactor + * Fix test failure caused by previous tests not clearing db connection + * Bump version to 0.3.1 + * Fix locale in vagrantfile (for me anyway 😁), use tox for testing + + [ vagrant ] + + -- Alex Forbes Wed, 23 Nov 2016 17:39:48 +0000 + orlo (0.3.1) trusty; urgency=medium * Add support for filtering by reference From c8d23044569d49a876be525bc91902bb6ee7aeec Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 23 Nov 2016 18:02:18 +0000 Subject: [PATCH 03/38] Fix path to gunicorn in systemd unit --- debian/systemd/orlo.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/systemd/orlo.service b/debian/systemd/orlo.service index 46860b2..4ce4d86 100644 --- a/debian/systemd/orlo.service +++ b/debian/systemd/orlo.service @@ -3,7 +3,7 @@ [Unit] Description=orlo After=network.target -ConditionPathExists=/usr/share/python/orlo/bin/gunicorn +ConditionPathExists=/opt/venvs/orlo/bin/gunicorn [Service] Type=simple From 05c657eb6b0186901e00357c2c3e374fb3bcd07d Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Wed, 23 Nov 2016 18:02:43 +0000 Subject: [PATCH 04/38] Update changelog for 0.3.1-2 release --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index bca3ce2..681052f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +orlo (0.3.1-2) jessie; urgency=medium + + [ Alex Forbes ] + * Fix path to gunicorn in systemd unit + + [ vagrant ] + + -- Alex Forbes Wed, 23 Nov 2016 18:02:42 +0000 + orlo (0.3.1-1) jessie; urgency=medium [ Alex Forbes ] From 138b7bac31ee9f092d1f754f944b041cef5ea090 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 10:08:59 +0000 Subject: [PATCH 05/38] Fix exec path --- debian/systemd/orlo.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/systemd/orlo.service b/debian/systemd/orlo.service index 4ce4d86..8ffff86 100644 --- a/debian/systemd/orlo.service +++ b/debian/systemd/orlo.service @@ -9,7 +9,7 @@ ConditionPathExists=/opt/venvs/orlo/bin/gunicorn Type=simple User=orlo Group=orlo -ExecStart=/usr/share/python/orlo/bin/gunicorn \ +ExecStart=/opt/venvs/orlo/bin/gunicorn \ -w 4 -b 127.0.0.1:8080 \ --access-logfile /var/log/orlo/gunicorn-access.log \ --log-level debug \ From 95e8f432a735af5b0ddb3e7eec0c046a0a5fbfde Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 10:09:29 +0000 Subject: [PATCH 06/38] Update changelog for 0.3.1-3 release --- debian/changelog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/changelog b/debian/changelog index 681052f..c9014cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +orlo (0.3.1-3) UNRELEASED; urgency=medium + + [ Alex Forbes ] + * Fix exec path + + [ vagrant ] + + -- Alex Forbes Thu, 24 Nov 2016 10:09:28 +0000 + orlo (0.3.1-2) jessie; urgency=medium [ Alex Forbes ] From 75fdc041d7f0df50aa08b94105388aa25cd29e27 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 10:47:43 +0000 Subject: [PATCH 07/38] Restrict gbp dch to debian dir, add debuild artifacts to gitignore --- .gitignore | 4 ++++ Makefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fd51a5d..9764dac 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,10 @@ var/ *.egg-info/ .installed.cfg *.egg +debian/*.debhelper +debian/*.substvars +debian/orlo +debian/files # PyInstaller # Usually these files are written by a python script from a template diff --git a/Makefile b/Makefile index b61ce47..9cb9e5d 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,6 @@ deb: debuild -us -uc changelog: - gbp dch --ignore-branch --auto --commit + gbp dch --ignore-branch --auto --commit debian .PHONY: debian sdist test clean changelog From 8c4d1da92826e96f089970a4303af0a39b8a585b Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 10:52:26 +0000 Subject: [PATCH 08/38] Set universal wheel in setup.cfg --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 From 8ff7b46fba1a0e298191fc8bc3f5c2b7a61c853a Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 15:30:06 +0000 Subject: [PATCH 09/38] Add orloclient as dependency Makes sense for this to be installed whenever the server is --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d9a9a1c..d2629a3 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ ], include_package_data=True, install_requires=[ + 'orloclient', 'Flask', 'Flask-Migrate', 'Flask-SQLAlchemy', From 824b5f554b77d3805fa7f44a090d4886ea426832 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 15:38:15 +0000 Subject: [PATCH 10/38] Add /internal/version endpoint --- orlo/__init__.py | 1 + orlo/route_internal.py | 15 +++++++++++++++ tests/test_route_internal.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 orlo/route_internal.py create mode 100644 tests/test_route_internal.py diff --git a/orlo/__init__.py b/orlo/__init__.py index 9a111c8..dfd6617 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -80,6 +80,7 @@ import orlo.route_packages import orlo.route_import import orlo.route_info +import orlo.route_internal import orlo.route_stats import orlo.user_auth diff --git a/orlo/route_internal.py b/orlo/route_internal.py new file mode 100644 index 0000000..eabe3e3 --- /dev/null +++ b/orlo/route_internal.py @@ -0,0 +1,15 @@ +from __future__ import print_function +from flask import jsonify +from orlo import app +from orlo import __version__ + +__author__ = 'alforbes' + + +@app.route('/internal/version', methods=['GET']) +def internal_version(): + """ + Get the running version of Orlo + :return: + """ + return jsonify({'version': __version__}) diff --git a/tests/test_route_internal.py b/tests/test_route_internal.py new file mode 100644 index 0000000..c2c67ca --- /dev/null +++ b/tests/test_route_internal.py @@ -0,0 +1,14 @@ +from __future__ import print_function +from test_route_base import OrloHttpTest + +__author__ = 'alforbes' + + +class TestInternalRoute(OrloHttpTest): + def test_version(self): + """ + Test /version returns 200 + """ + response = self.client.get('/internal/version') + self.assert200(response) + self.assertIn('version', response.json) From 59b7b6f3e7c12194f6f844c9dee2f7344abb1023 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 15:47:59 +0000 Subject: [PATCH 11/38] Move routes to subdirectory --- orlo/__init__.py | 18 ++++++------------ orlo/routes/__init__.py | 13 +++++++++++++ orlo/{route_base.py => routes/base.py} | 0 orlo/{route_import.py => routes/import_.py} | 0 orlo/{route_info.py => routes/info.py} | 0 orlo/{route_internal.py => routes/internal.py} | 0 orlo/{route_packages.py => routes/packages.py} | 0 orlo/{route_releases.py => routes/releases.py} | 0 orlo/{route_stats.py => routes/stats.py} | 0 9 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 orlo/routes/__init__.py rename orlo/{route_base.py => routes/base.py} (100%) rename orlo/{route_import.py => routes/import_.py} (100%) rename orlo/{route_info.py => routes/info.py} (100%) rename orlo/{route_internal.py => routes/internal.py} (100%) rename orlo/{route_packages.py => routes/packages.py} (100%) rename orlo/{route_releases.py => routes/releases.py} (100%) rename orlo/{route_stats.py => routes/stats.py} (100%) diff --git a/orlo/__init__.py b/orlo/__init__.py index dfd6617..cc0535c 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -1,14 +1,14 @@ from __future__ import print_function, division, absolute_import from __future__ import unicode_literals -from flask import Flask + import logging -from logging.handlers import RotatingFileHandler from logging import Formatter -import sys +from logging.handlers import RotatingFileHandler + +from flask import Flask from orlo.config import config, CONFIG_FILE -from orlo.exceptions import OrloStartupError, OrloError, OrloAuthError, \ - OrloConfigError +from orlo.exceptions import OrloStartupError try: # _version is created by setup.py @@ -75,13 +75,7 @@ # Must be imported last import orlo.error_handlers -import orlo.route_base -import orlo.route_releases -import orlo.route_packages -import orlo.route_import -import orlo.route_info -import orlo.route_internal -import orlo.route_stats +import orlo.routes import orlo.user_auth app.logger.info("Startup completed with configuration from {}".format( diff --git a/orlo/routes/__init__.py b/orlo/routes/__init__.py new file mode 100644 index 0000000..d75cdfd --- /dev/null +++ b/orlo/routes/__init__.py @@ -0,0 +1,13 @@ +from __future__ import print_function + +import orlo.routes.base +import orlo.routes.import_ +import orlo.routes.info +import orlo.routes.internal +import orlo.routes.packages +import orlo.routes.releases +import orlo.routes.stats + + +__author__ = 'alforbes' + diff --git a/orlo/route_base.py b/orlo/routes/base.py similarity index 100% rename from orlo/route_base.py rename to orlo/routes/base.py diff --git a/orlo/route_import.py b/orlo/routes/import_.py similarity index 100% rename from orlo/route_import.py rename to orlo/routes/import_.py diff --git a/orlo/route_info.py b/orlo/routes/info.py similarity index 100% rename from orlo/route_info.py rename to orlo/routes/info.py diff --git a/orlo/route_internal.py b/orlo/routes/internal.py similarity index 100% rename from orlo/route_internal.py rename to orlo/routes/internal.py diff --git a/orlo/route_packages.py b/orlo/routes/packages.py similarity index 100% rename from orlo/route_packages.py rename to orlo/routes/packages.py diff --git a/orlo/route_releases.py b/orlo/routes/releases.py similarity index 100% rename from orlo/route_releases.py rename to orlo/routes/releases.py diff --git a/orlo/route_stats.py b/orlo/routes/stats.py similarity index 100% rename from orlo/route_stats.py rename to orlo/routes/stats.py From d39b335be33890406b400ac3be968991c460f620 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 16:58:08 +0000 Subject: [PATCH 12/38] Refactor orlo command line, embed gunicorn app, add db migrations Some import paths have changed. It is still possible to use the app object directly with external gunicorn or another wsgi compliant server Flask-script and flask-alembic added for DB migrations. --- create_db.py | 12 --- etc/orlo.ini | 9 ++- orlo/__init__.py | 81 ++----------------- orlo/__main__.py | 175 ++++++++++++++++++++++++++++++++++++++++ orlo/app.py | 103 +++++++++++++++++++++++ orlo/cli.py | 11 +-- orlo/config.py | 16 ++-- orlo/deploy.py | 2 +- orlo/error_handlers.py | 2 +- orlo/orm.py | 3 +- orlo/queries.py | 2 +- orlo/routes/base.py | 2 +- orlo/routes/import_.py | 2 +- orlo/routes/info.py | 2 +- orlo/routes/internal.py | 2 +- orlo/routes/packages.py | 3 +- orlo/routes/releases.py | 4 +- orlo/routes/stats.py | 2 +- orlo/stats.py | 2 +- orlo/user_auth.py | 2 +- orlo/util.py | 2 +- runserver.py | 6 -- setup.py | 8 +- tests/test_orm.py | 2 +- 24 files changed, 327 insertions(+), 128 deletions(-) delete mode 100644 create_db.py create mode 100644 orlo/__main__.py create mode 100644 orlo/app.py delete mode 100755 runserver.py diff --git a/create_db.py b/create_db.py deleted file mode 100644 index 92a57ba..0000000 --- a/create_db.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import print_function - -__author__ = 'alforbes' - -from orlo import app -from orlo.orm import db - -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///orlo.db' -app.debug = True -db.create_all() - - diff --git a/etc/orlo.ini b/etc/orlo.ini index 18b2b2c..8a48152 100644 --- a/etc/orlo.ini +++ b/etc/orlo.ini @@ -5,11 +5,14 @@ echo_queries = false [main] time_zone = UTC -propagate_exceptions = true ; Format to represent date/time as time_format = %Y-%m-%dT%H:%M:%SZ base_url = http://localhost:8080 +[flask] +propagate_exceptions = true +debug = true + [security] enabled = false method = file @@ -19,5 +22,5 @@ secret_key = change_me token_ttl = 3600 [logging] -file = /var/log/orlo/app.log -debug = false +directory = /var/log/orlo +level = debug diff --git a/orlo/__init__.py b/orlo/__init__.py index cc0535c..122d9e5 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -1,82 +1,11 @@ from __future__ import print_function, division, absolute_import from __future__ import unicode_literals +from pkg_resources import get_distribution -import logging -from logging import Formatter -from logging.handlers import RotatingFileHandler +__version__ = get_distribution(__name__).version -from flask import Flask +from orlo.config import config +from orlo.app import app as app -from orlo.config import config, CONFIG_FILE -from orlo.exceptions import OrloStartupError -try: - # _version is created by setup.py - from orlo._version import __version__ -except ImportError: - __version__ = "TEST_BUILD" - - -app = Flask(__name__) - -app.config['SQLALCHEMY_DATABASE_URI'] = config.get('db', 'uri') -app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - -if 'sqlite' not in app.config['SQLALCHEMY_DATABASE_URI']: - # SQLite doesn't support these - app.config['SQLALCHEMY_POOL_SIZE'] = config.getint('db', 'pool_size') - app.config['SQLALCHEMY_POOL_RECYCLE'] = 1 - app.config['SQLALCHEMY_MAX_OVERFLOW'] = 10 - -if config.getboolean('main', 'propagate_exceptions'): - app.config['PROPAGATE_EXCEPTIONS'] = True - -if config.getboolean('db', 'echo_queries'): - app.config['SQLALCHEMY_ECHO'] = True - -if not config.getboolean('main', 'strict_slashes'): - app.url_map.strict_slashes = False - -# Debug mode ignores all custom logging and should only be used in -# local testing... -if config.getboolean('main', 'debug_mode'): - app.debug = True - -if not app.debug: - log_level = config.get('logging', 'level') - if log_level == 'debug': - app.logger.setLevel(logging.DEBUG) - elif log_level == 'info': - app.logger.setLevel(logging.INFO) - elif log_level == 'warning': - app.logger.setLevel(logging.WARNING) - elif log_level == 'error': - app.logger.setLevel(logging.ERROR) - - logfile = config.get('logging', 'file') - if logfile != 'disabled': - file_handler = RotatingFileHandler( - logfile, - maxBytes=1048576, - backupCount=1, - ) - log_format = config.get('logging', 'format') - formatter = Formatter(log_format) - - file_handler.setFormatter(formatter) - app.logger.addHandler(file_handler) - -if config.getboolean('security', 'enabled') and \ - config.get('security', 'secret_key') == 'change_me': - raise OrloStartupError( - "Security is enabled, please configure the secret key") - -app.logger.debug("Log level: {}".format(config.get('logging', 'level'))) - -# Must be imported last -import orlo.error_handlers -import orlo.routes -import orlo.user_auth - -app.logger.info("Startup completed with configuration from {}".format( - CONFIG_FILE)) +app.logger.info("Initialisation completed") diff --git a/orlo/__main__.py b/orlo/__main__.py new file mode 100644 index 0000000..47a6f35 --- /dev/null +++ b/orlo/__main__.py @@ -0,0 +1,175 @@ +from __future__ import print_function + +import logging +import os +import traceback + +import alembic +import alembic.util.exc as alembic_exc +import sqlalchemy.exc +from logging.handlers import RotatingFileHandler +from flask_alembic import alembic_script +from flask_script import Manager, Command, Option, Server +from orlo.config import config +from orlo.app import app, OrloApplication +from orlo.orm import db + + +logger = logging.getLogger('orlo') + +__author__ = 'alforbes' + + +class Start(Command): + """ + Run the Gunicorn API server + """ + + option_list = ( + Option('-l', '--loglevel', default=config.get('logging', 'level'), + help='Set logging level, config means use what\'s in config ' + 'file API:gunicorn_loglevel', + choices=['debug', 'info', 'warning', 'error', 'critical']), + Option('-c', '--log-console', default=False, dest='console', + action='store_true', + help="Log to console instead of configured log files"), + Option('--workers', default=config.get('gunicorn', 'workers'), + help="Number of gunicorn workers to start"), + ) + + def run(self, loglevel, console, workers): + """ + Start the production server + + @param loglevel: + @param console: + @return: + """ + log_level = getattr(logging, loglevel.upper()) + logger.setLevel(log_level) + + formatter = logging.Formatter( + config.get('logging', 'format', raw=True)) + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + + if console: + # Stream handler should use configured log level + stream_handler.setLevel(log_level) + else: + # Only print critical errors from now on + stream_handler.setLevel(logging.CRITICAL) + + # File logging + log_dir = config.get('logging', 'directory') + if '/' in log_dir: + file_handler = RotatingFileHandler( + os.path.join('log_dir', 'app.log')) + file_handler.setFormatter(formatter) + file_handler.setLevel(log_level) + logger.addHandler(file_handler) + + logger.debug('Debug logging enabled') + + gunicorn_options = { + 'accesslog': config.get('API', 'gunicorn_accesslog') if not + console else '-', + 'bind': '%s:%s' % ('0.0.0.0', '5000'), + 'capture_output': True, + 'errorlog': config.get('API', 'gunicorn_errorlog') if not + console else '-', + 'logfile': config.get('API', 'gunicorn_logfile') if not + console else '-', + 'loglevel': loglevel or config.get('API', 'gunicorn_loglevel'), + 'on_exit': on_exit, + 'on_starting': on_starting, + 'workers': workers or config.get('API', 'gunicorn_workers'), + } + try: + OrloApplication(app, gunicorn_options).run() + except KeyboardInterrupt: + logger.info('Caught KeyboardInterrupt') + logger.debug('__main__ done') + + +class WriteConfig(Command): + option_list = ( + Option('file', dest='file_path', help='Config file to write', + default='/etc/orlo/orlo.ini') + ) + + def run(self, file_path): + """ Write out configuration """ + config_file = open(file_path, 'w') + config.write(config_file) + + +script_manager = Manager(app) +script_manager.add_command('db', alembic_script) +script_manager.add_command('start', Start) +script_manager.add_command('run_dev_server', Server(host='0.0.0.0', port=5000)) + + +def on_starting(server): + """ Start our managers """ + logger.debug('on_starting called') + check_database() + + +def on_exit(server): + logger.debug('on_exit called') + + +def check_database(): + logger.debug('Checking database "{}"'.format( + config.get('API', 'db_uri') + )) + + try: + with app.app_context(): + current_head = alembic.script_directory.get_current_head() + current_rev = alembic.migration_context.get_current_revision() + logger.debug('Current revision: {}'.format(current_rev)) + logger.debug('Current head: {}'.format(current_head)) + if current_head is None: + logger.warning('No alembic revisions, initialising') + alembic.revision('Initial revision') + alembic.upgrade() + elif current_rev is None: + logger.info('Database not configured, calling alembic upgrade') + alembic.upgrade() + elif current_rev != current_head: + logger.info('New database revision available, calling alembic ' + 'upgrade') + alembic.upgrade() + except sqlalchemy.exc.OperationalError: + logger.warning('Database is not configured, creating tables') + db.create_all() + except alembic_exc.CommandError: + logger.error( + 'Alembic raised an exception, please check the state of the ' + 'database, and that there aren\'t any extra files in ' + '/opt/venvs/gumtree-deployer/local/lib/python2.7/site-packages' + '/gumtreeDeployer/rest/migrations. Exception ' + 'was:\n{}'.format(traceback.format_exc())) + raise SystemExit(1) + + logger.info('Database is configured') + + +def main(): + """ + Entry point for setuptools to call + """ + try: + script_manager.run() + except StandardError: + # Should print here as there is no guarantee logging is working + tb = traceback.format_exc() + print('Exception on execution:\n{}'.format(tb)) + + +if __name__ == "__main__": + main() diff --git a/orlo/app.py b/orlo/app.py new file mode 100644 index 0000000..2901174 --- /dev/null +++ b/orlo/app.py @@ -0,0 +1,103 @@ +from __future__ import print_function + +import os +import logging +import gunicorn.app.base +from gunicorn.six import iteritems + +from logging import Formatter +from logging.handlers import RotatingFileHandler + +from flask import Flask +from flask_alembic import Alembic + +from orlo.config import config, CONFIG_FILE +from orlo.exceptions import OrloStartupError + + +__author__ = 'alforbes' + +app = Flask(__name__) + +alembic = Alembic() +alembic.init_app(app) + +app.config['SQLALCHEMY_DATABASE_URI'] = config.get('db', 'uri') +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +if 'sqlite' not in app.config['SQLALCHEMY_DATABASE_URI']: + # SQLite doesn't support these + app.config['SQLALCHEMY_POOL_SIZE'] = config.getint('db', 'pool_size') + app.config['SQLALCHEMY_POOL_RECYCLE'] = 2 + app.config['SQLALCHEMY_MAX_OVERFLOW'] = 10 + +if config.getboolean('flask', 'propagate_exceptions'): + app.config['PROPAGATE_EXCEPTIONS'] = True + +if config.getboolean('db', 'echo_queries'): + app.config['SQLALCHEMY_ECHO'] = True + +if not config.getboolean('main', 'strict_slashes'): + app.url_map.strict_slashes = False + +# Debug mode ignores all custom logging and should only be used in +# local testing... +if config.getboolean('flask', 'debug'): + app.debug = True + +if not app.debug: + log_level = config.get('logging', 'level') + if log_level == 'debug': + app.logger.setLevel(logging.DEBUG) + elif log_level == 'info': + app.logger.setLevel(logging.INFO) + elif log_level == 'warning': + app.logger.setLevel(logging.WARNING) + elif log_level == 'error': + app.logger.setLevel(logging.ERROR) + + log_dir = config.get('logging', 'directory') + logfile = os.path.join(log_dir, 'app.log') + if log_dir != 'disabled': + file_handler = RotatingFileHandler( + logfile, + maxBytes=1048576, + backupCount=1, + ) + log_format = config.get('logging', 'format') + formatter = Formatter(log_format) + + file_handler.setFormatter(formatter) + app.logger.addHandler(file_handler) + + +class OrloApplication(gunicorn.app.base.BaseApplication): + """ Gunicorn application """ + def __init__(self, application, options=None): + self.options = options or {} + self.application = application + super(OrloApplication, self).__init__() + + def load_config(self): + _config = dict([(key, value) for key, value in iteritems(self.options) + if key in self.cfg.settings and value is not None]) + for key, value in iteritems(_config): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + +if config.getboolean('security', 'enabled') and \ + config.get('security', 'secret_key') == 'change_me': + raise OrloStartupError( + "Security is enabled, please configure security:secret_key in orlo.ini") + +app.logger.debug("Log level: {}".format(config.get('logging', 'level'))) + + +# Must be imported last + +import orlo.error_handlers +import orlo.routes +import orlo.user_auth diff --git a/orlo/cli.py b/orlo/cli.py index 723be70..7d1b07c 100644 --- a/orlo/cli.py +++ b/orlo/cli.py @@ -1,12 +1,7 @@ from __future__ import print_function import argparse -try: - from orlo import __version__ -except ImportError: - # _version.py doesn't exist - __version__ = "TEST_BUILD" - +from orlo import __version__ __author__ = 'alforbes' @@ -55,7 +50,7 @@ def parse_args(): def write_config(args): - from orlo import config + from orlo.config import config config_file = open(args.file_path, 'w') config.write(config_file) @@ -75,7 +70,7 @@ def run_server(args): print("Warning: this is a development server and not suitable " "for production, we recommend running under gunicorn.") - from orlo import app + from orlo.app import app app.config['DEBUG'] = True app.config['TRAP_HTTP_EXCEPTIONS'] = True app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False diff --git a/orlo/config.py b/orlo/config.py index bfac44e..17d5a83 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -12,13 +12,15 @@ config = RawConfigParser() config.add_section('main') -config.set('main', 'debug_mode', 'false') -config.set('main', 'propagate_exceptions', 'true') config.set('main', 'time_format', '%Y-%m-%dT%H:%M:%SZ') config.set('main', 'time_zone', 'UTC') config.set('main', 'strict_slashes', 'false') config.set('main', 'base_url', 'http://localhost:8080') +config.add_section('gunicorn') +config.set('gunicorn', 'workers', '4') +config.set('gunicorn', 'loglevel', '4') + config.add_section('security') config.set('security', 'enabled', 'false') config.set('security', 'passwd_file', 'none') @@ -35,16 +37,20 @@ config.set('db', 'echo_queries', 'false') config.set('db', 'pool_size', '50') +config.add_section('flask') +config.set('flask', 'propagate_exceptions', 'true') +config.set('flask', 'debug', 'false') + config.add_section('logging') config.set('logging', 'level', 'info') -config.set('logging', 'file', 'disabled') config.set('logging', 'format', '%(asctime)s [%(name)s] %(levelname)s %(' 'module)s:%(funcName)s:%(lineno)d - %(' 'message)s') +config.set('logging', 'directory', '/var/log/orlo') config.add_section('deploy') -config.set('deploy', 'timeout', - '3600') # How long to timeout external deployer calls +# How long to timeout external deployer calls +config.set('deploy', 'timeout', '3600') config.add_section('deploy_shell') config.set('deploy_shell', 'command_path', diff --git a/orlo/deploy.py b/orlo/deploy.py index 166f9a0..e87d445 100644 --- a/orlo/deploy.py +++ b/orlo/deploy.py @@ -4,7 +4,7 @@ from threading import Timer from orlo.config import config from orlo.exceptions import OrloDeployError -from orlo import app +from orlo.app import app __author__ = 'alforbes' diff --git a/orlo/error_handlers.py b/orlo/error_handlers.py index 00d8947..45d3c9d 100644 --- a/orlo/error_handlers.py +++ b/orlo/error_handlers.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import jsonify, request -from orlo import app +from orlo.app import app from orlo.exceptions import InvalidUsage, OrloError __author__ = 'alforbes' diff --git a/orlo/orm.py b/orlo/orm.py index c63bfa2..756b703 100644 --- a/orlo/orm.py +++ b/orlo/orm.py @@ -2,7 +2,8 @@ from sqlalchemy_utils.types.uuid import UUIDType from sqlalchemy_utils.types.arrow import ArrowType -from orlo import app, config +from orlo.app import app +from orlo.config import config from orlo.exceptions import OrloWorkflowError import pytz import uuid diff --git a/orlo/queries.py b/orlo/queries.py index df886f5..73643ed 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -1,7 +1,7 @@ from __future__ import print_function import datetime import arrow -from orlo import app +from orlo.app import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage from sqlalchemy import and_, exc diff --git a/orlo/routes/base.py b/orlo/routes/base.py index 22fa32a..ae3cfb4 100644 --- a/orlo/routes/base.py +++ b/orlo/routes/base.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import jsonify, request -from orlo import app +from orlo.app import app from orlo import __version__ __author__ = 'alforbes' diff --git a/orlo/routes/import_.py b/orlo/routes/import_.py index 360fee4..0ff0445 100644 --- a/orlo/routes/import_.py +++ b/orlo/routes/import_.py @@ -1,7 +1,7 @@ import arrow import json from flask import jsonify, request -from orlo import app +from orlo.app import app from orlo.orm import db, Package, Release, PackageResult, ReleaseNote, Platform from orlo.util import validate_request_json from orlo.user_auth import token_auth diff --git a/orlo/routes/info.py b/orlo/routes/info.py index 8086e39..30470bd 100644 --- a/orlo/routes/info.py +++ b/orlo/routes/info.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import request, jsonify, url_for -from orlo import app +from orlo.app import app import orlo.queries as queries __author__ = 'alforbes' diff --git a/orlo/routes/internal.py b/orlo/routes/internal.py index eabe3e3..504547f 100644 --- a/orlo/routes/internal.py +++ b/orlo/routes/internal.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import jsonify -from orlo import app +from orlo.app import app from orlo import __version__ __author__ = 'alforbes' diff --git a/orlo/routes/packages.py b/orlo/routes/packages.py index 927c88d..1fa8417 100644 --- a/orlo/routes/packages.py +++ b/orlo/routes/packages.py @@ -1,6 +1,7 @@ from __future__ import print_function from flask import jsonify, request, Response, json, g -from orlo import app, queries, config +from orlo.app import app +from orlo import queries from orlo.exceptions import InvalidUsage from orlo.user_auth import token_auth from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, \ diff --git a/orlo/routes/releases.py b/orlo/routes/releases.py index 879afa6..f19576f 100644 --- a/orlo/routes/releases.py +++ b/orlo/routes/releases.py @@ -1,5 +1,7 @@ from flask import jsonify, request, Response, json, g -from orlo import app, queries, config +from orlo.app import app +from orlo import queries +from orlo.config import config from orlo.exceptions import InvalidUsage from orlo.user_auth import token_auth from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, \ diff --git a/orlo/routes/stats.py b/orlo/routes/stats.py index 5153911..cf96e4b 100644 --- a/orlo/routes/stats.py +++ b/orlo/routes/stats.py @@ -2,7 +2,7 @@ from flask import request, jsonify from orlo import stats -from orlo import app +from orlo.app import app from orlo.exceptions import InvalidUsage import orlo.queries as queries diff --git a/orlo/stats.py b/orlo/stats.py index 43c0450..eb1ff2b 100644 --- a/orlo/stats.py +++ b/orlo/stats.py @@ -1,6 +1,6 @@ from __future__ import print_function from orlo.queries import apply_filters, filter_release_rollback, filter_release_status -from orlo import app +from orlo.app import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage from collections import OrderedDict diff --git a/orlo/user_auth.py b/orlo/user_auth.py index 7892a3b..74ff51a 100644 --- a/orlo/user_auth.py +++ b/orlo/user_auth.py @@ -1,4 +1,4 @@ -from orlo import app +from orlo.app import app from orlo.config import config from orlo.exceptions import OrloAuthError from flask_httpauth import HTTPBasicAuth diff --git a/orlo/util.py b/orlo/util.py index 4f5f9b7..a633616 100644 --- a/orlo/util.py +++ b/orlo/util.py @@ -1,6 +1,6 @@ from __future__ import print_function, unicode_literals from flask import json -from orlo import app +from orlo.app import app from orlo.orm import db, Release, Package, Platform from orlo.exceptions import InvalidUsage from sqlalchemy.orm import exc diff --git a/runserver.py b/runserver.py deleted file mode 100755 index 69cb434..0000000 --- a/runserver.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -from orlo import app - -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///orlo.db' - -app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/setup.py b/setup.py index d2629a3..d84c650 100755 --- a/setup.py +++ b/setup.py @@ -29,14 +29,16 @@ ], include_package_data=True, install_requires=[ - 'orloclient', 'Flask', + 'Flask-Alembic >= 2.0.1', + 'Flask-HTTPAuth', 'Flask-Migrate', 'Flask-SQLAlchemy', - 'Flask-HTTPAuth', + 'Flask-Script >= 2.0.5', 'Flask-TokenAuth', 'arrow', 'gunicorn', + 'orloclient', 'psycopg2', 'pyldap', 'pytz', @@ -50,6 +52,6 @@ test_suite='tests', # Creates a script in /usr/local/bin entry_points={ - 'console_scripts': ['orlo=orlo.cli:main'] + 'console_scripts': ['orlo=orlo.__main__:main'] } ) diff --git a/tests/test_orm.py b/tests/test_orm.py index b5e3912..66b8bb8 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -3,7 +3,7 @@ from random import randrange from orlo.orm import db from orlo.orm import Release, Package, PackageResult, Platform -from orlo import app +from orlo.app import app from sqlalchemy.orm import exc import arrow import datetime From 4ce168d256ae4017c43e4f333b93b1e360a5cce2 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 17:05:07 +0000 Subject: [PATCH 13/38] Commit initial database version --- MANIFEST.in | 1 + .../e60a77e44da8_initial_db_revision.py | 100 ++++++++++++++++++ orlo/migrations/script.py.mako | 23 ++++ 3 files changed, 124 insertions(+) create mode 100644 orlo/migrations/e60a77e44da8_initial_db_revision.py create mode 100644 orlo/migrations/script.py.mako diff --git a/MANIFEST.in b/MANIFEST.in index bb3ec5f..b268a86 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include README.md +recursive-include orlo/migrations * diff --git a/orlo/migrations/e60a77e44da8_initial_db_revision.py b/orlo/migrations/e60a77e44da8_initial_db_revision.py new file mode 100644 index 0000000..0d64bff --- /dev/null +++ b/orlo/migrations/e60a77e44da8_initial_db_revision.py @@ -0,0 +1,100 @@ +"""Initial DB revision + +Revision ID: e60a77e44da8 +Revises: +Create Date: 2016-11-24 17:03:25.249133 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e60a77e44da8' +down_revision = None +branch_labels = ('default',) +depends_on = None + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('platform', + sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('name', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('release', + sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('references', sa.String(), nullable=True), + sa.Column('stime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('ftime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('duration', sa.Interval(), nullable=True), + sa.Column('user', sa.String(), nullable=False), + sa.Column('team', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id') + ) + op.create_index(op.f('ix_release_stime'), 'release', ['stime'], unique=False) + op.create_table('package', + sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('stime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('ftime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('duration', sa.Interval(), nullable=True), + sa.Column('status', sa.Enum('NOT_STARTED', 'IN_PROGRESS', 'SUCCESSFUL', 'FAILED', name='status_types'), nullable=True), + sa.Column('version', sa.String(length=32), nullable=False), + sa.Column('diff_url', sa.String(), nullable=True), + sa.Column('rollback', sa.Boolean(), nullable=True), + sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id') + ) + op.create_index(op.f('ix_package_stime'), 'package', ['stime'], unique=False) + op.create_table('release_metadata', + sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('key', sa.Text(), nullable=False), + sa.Column('value', sa.Text(), nullable=False), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id') + ) + op.create_table('release_note', + sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id') + ) + op.create_table('release_platform', + sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('platform_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.ForeignKeyConstraint(['platform_id'], ['platform.id'], ), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ) + ) + op.create_table('package_result', + sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('content', sa.Text(), nullable=True), + sa.Column('package_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('package_result') + op.drop_table('release_platform') + op.drop_table('release_note') + op.drop_table('release_metadata') + op.drop_index(op.f('ix_package_stime'), table_name='package') + op.drop_table('package') + op.drop_index(op.f('ix_release_stime'), table_name='release') + op.drop_table('release') + op.drop_table('platform') + ### end Alembic commands ### diff --git a/orlo/migrations/script.py.mako b/orlo/migrations/script.py.mako new file mode 100644 index 0000000..8f45a34 --- /dev/null +++ b/orlo/migrations/script.py.mako @@ -0,0 +1,23 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} From a61a6f64ea11981e8b90b88ce0f8ca9c9027e558 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 17:46:40 +0000 Subject: [PATCH 14/38] WIP --- orlo/__main__.py | 76 +++++++++++++++++++++++++----------------------- orlo/app.py | 25 +++++++++------- orlo/config.py | 3 +- 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/orlo/__main__.py b/orlo/__main__.py index 47a6f35..d6515f6 100644 --- a/orlo/__main__.py +++ b/orlo/__main__.py @@ -10,16 +10,19 @@ from logging.handlers import RotatingFileHandler from flask_alembic import alembic_script from flask_script import Manager, Command, Option, Server +from orlo.exceptions import OrloStartupError + from orlo.config import config from orlo.app import app, OrloApplication from orlo.orm import db -logger = logging.getLogger('orlo') - __author__ = 'alforbes' +logger = logging.getLogger('orlo') + + class Start(Command): """ Run the Gunicorn API server @@ -27,13 +30,12 @@ class Start(Command): option_list = ( Option('-l', '--loglevel', default=config.get('logging', 'level'), - help='Set logging level, config means use what\'s in config ' - 'file API:gunicorn_loglevel', - choices=['debug', 'info', 'warning', 'error', 'critical']), + choices=['debug', 'info', 'warning', 'error', 'critical'], + help='Set logging level'), Option('-c', '--log-console', default=False, dest='console', action='store_true', help="Log to console instead of configured log files"), - Option('--workers', default=config.get('gunicorn', 'workers'), + Option('-w', '--workers', default=config.get('gunicorn', 'workers'), help="Number of gunicorn workers to start"), ) @@ -46,14 +48,14 @@ def run(self, loglevel, console, workers): @return: """ log_level = getattr(logging, loglevel.upper()) - logger.setLevel(log_level) + app.logger.setLevel(log_level) formatter = logging.Formatter( config.get('logging', 'format', raw=True)) stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) - logger.addHandler(stream_handler) + app.logger.addHandler(stream_handler) if console: # Stream handler should use configured log level @@ -62,39 +64,48 @@ def run(self, loglevel, console, workers): # Only print critical errors from now on stream_handler.setLevel(logging.CRITICAL) - # File logging log_dir = config.get('logging', 'directory') - if '/' in log_dir: + logfile = os.path.join(log_dir, 'flask.log') + if log_dir != 'disabled': file_handler = RotatingFileHandler( - os.path.join('log_dir', 'app.log')) + logfile, + maxBytes=1048576, + backupCount=1, + ) + log_format = config.get('logging', 'format') + formatter = logging.Formatter(log_format) + file_handler.setFormatter(formatter) - file_handler.setLevel(log_level) + file_handler.setLevel(logging.DEBUG) logger.addHandler(file_handler) - logger.debug('Debug logging enabled') + app.logger.debug('Debug logging enabled') + log_dir = config.get('logging', 'directory') gunicorn_options = { - 'accesslog': config.get('API', 'gunicorn_accesslog') if not + 'accesslog': os.path.join(log_dir, 'access.log') if not console else '-', 'bind': '%s:%s' % ('0.0.0.0', '5000'), 'capture_output': True, - 'errorlog': config.get('API', 'gunicorn_errorlog') if not + 'errorlog': os.path.join(log_dir, 'gunicorn_error.log') if not console else '-', - 'logfile': config.get('API', 'gunicorn_logfile') if not + 'logfile': os.path.join(log_dir, 'gunicorn.log') if not console else '-', - 'loglevel': loglevel or config.get('API', 'gunicorn_loglevel'), - 'on_exit': on_exit, + 'loglevel': loglevel or config.get('logging', 'level'), 'on_starting': on_starting, - 'workers': workers or config.get('API', 'gunicorn_workers'), + 'workers': workers, } try: OrloApplication(app, gunicorn_options).run() except KeyboardInterrupt: - logger.info('Caught KeyboardInterrupt') - logger.debug('__main__ done') + app.logger.info('Caught KeyboardInterrupt') + app.logger.debug('__main__ done') class WriteConfig(Command): + """ + Write out the Orlo configuration file + """ option_list = ( Option('file', dest='file_path', help='Config file to write', default='/etc/orlo/orlo.ini') @@ -113,18 +124,13 @@ def run(self, file_path): def on_starting(server): - """ Start our managers """ - logger.debug('on_starting called') + app.logger.debug('on_starting called') check_database() -def on_exit(server): - logger.debug('on_exit called') - - def check_database(): - logger.debug('Checking database "{}"'.format( - config.get('API', 'db_uri') + logger.info('Checking database "{}"'.format( + config.get('db', 'uri') )) try: @@ -134,13 +140,11 @@ def check_database(): logger.debug('Current revision: {}'.format(current_rev)) logger.debug('Current head: {}'.format(current_head)) if current_head is None: - logger.warning('No alembic revisions, initialising') - alembic.revision('Initial revision') - alembic.upgrade() + raise OrloStartupError('No alembic revisions, this is a bug') elif current_rev is None: logger.info('Database not configured, calling alembic upgrade') alembic.upgrade() - elif current_rev != current_head: + elif current_head != current_rev: logger.info('New database revision available, calling alembic ' 'upgrade') alembic.upgrade() @@ -151,9 +155,9 @@ def check_database(): logger.error( 'Alembic raised an exception, please check the state of the ' 'database, and that there aren\'t any extra files in ' - '/opt/venvs/gumtree-deployer/local/lib/python2.7/site-packages' - '/gumtreeDeployer/rest/migrations. Exception ' - 'was:\n{}'.format(traceback.format_exc())) + '/opt/venvs/orlo/local/lib/python2.7/site-packages/orlo/' + 'migrations. Exception was:\n{}'.format( + traceback.format_exc())) raise SystemExit(1) logger.info('Database is configured') diff --git a/orlo/app.py b/orlo/app.py index 2901174..64e9678 100644 --- a/orlo/app.py +++ b/orlo/app.py @@ -46,18 +46,21 @@ app.debug = True if not app.debug: - log_level = config.get('logging', 'level') - if log_level == 'debug': - app.logger.setLevel(logging.DEBUG) - elif log_level == 'info': - app.logger.setLevel(logging.INFO) - elif log_level == 'warning': - app.logger.setLevel(logging.WARNING) - elif log_level == 'error': - app.logger.setLevel(logging.ERROR) + # If you are calling orlo.app directly rather than with the embedded + # gunicorn, you may want to add a streamHandler + try: + _level = config.get('logging', 'level') + log_level = getattr(logging, _level.upper()) + except AttributeError: + app.logger.error( + 'Failed to set log level to {}, see ' + 'https://docs.python.org/3.6/library/logging.html#logging-levels ' + 'for valid levels.'.format(_level)) + log_level = logging.INFO + app.logger.setLevel(log_level) log_dir = config.get('logging', 'directory') - logfile = os.path.join(log_dir, 'app.log') + logfile = os.path.join(log_dir, 'flask.log') if log_dir != 'disabled': file_handler = RotatingFileHandler( logfile, @@ -68,6 +71,8 @@ formatter = Formatter(log_format) file_handler.setFormatter(formatter) + file_handler.setLevel(logging.DEBUG) + app.logger.addHandler(file_handler) diff --git a/orlo/config.py b/orlo/config.py index 17d5a83..aa6e1a8 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -19,7 +19,6 @@ config.add_section('gunicorn') config.set('gunicorn', 'workers', '4') -config.set('gunicorn', 'loglevel', '4') config.add_section('security') config.set('security', 'enabled', 'false') @@ -46,7 +45,7 @@ config.set('logging', 'format', '%(asctime)s [%(name)s] %(levelname)s %(' 'module)s:%(funcName)s:%(lineno)d - %(' 'message)s') -config.set('logging', 'directory', '/var/log/orlo') +config.set('logging', 'directory', '/var/log/orlo') # disabled for no log files config.add_section('deploy') # How long to timeout external deployer calls From c4d80141836bd18038359f4484637d4ae48464b5 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 19:06:40 +0000 Subject: [PATCH 15/38] Fix migration script Sqlalchemy-utils' UUIDType doesn't work with Alembic :( Please enter the commit message for your changes. Lines starting --- .../e60a77e44da8_initial_db_revision.py | 56 +++++++++++++------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/orlo/migrations/e60a77e44da8_initial_db_revision.py b/orlo/migrations/e60a77e44da8_initial_db_revision.py index 0d64bff..66e76ea 100644 --- a/orlo/migrations/e60a77e44da8_initial_db_revision.py +++ b/orlo/migrations/e60a77e44da8_initial_db_revision.py @@ -7,6 +7,9 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy_utils.types.uuid import UUIDType +from sqlalchemy_utils.types.arrow import ArrowType +from orlo.config import config # revision identifiers, used by Alembic. @@ -15,20 +18,41 @@ branch_labels = ('default',) depends_on = None + +class HackyUUIDType(UUIDType): + """ Horrible hack for SQLAlchemy-utils' UUID, which doesn't support + Alembic yet + + For Postgres, we return the UUID dialect, for others CHAR(36) + See https://github.com/kvesteri/sqlalchemy-utils/issues/129 + """ + def __repr__(self): + """ + :return: + """ + uri = config.get('db', 'uri') + if uri.startswith('postgresql'): + return "sa.dialects.postgresql.UUID()" + elif uri.startswith('mysql'): + return "sa.dialects.mysql.CHAR(32)" + else: + return "sa.types.CHAR(32)" + + def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.create_table('platform', - sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('id', HackyUUIDType(), nullable=False), sa.Column('name', sa.Text(), nullable=False), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), sa.UniqueConstraint('name') ) op.create_table('release', - sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('id', HackyUUIDType(), nullable=False), sa.Column('references', sa.String(), nullable=True), - sa.Column('stime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), - sa.Column('ftime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('stime', ArrowType(), nullable=True), + sa.Column('ftime', ArrowType(), nullable=True), sa.Column('duration', sa.Interval(), nullable=True), sa.Column('user', sa.String(), nullable=False), sa.Column('team', sa.String(), nullable=True), @@ -37,24 +61,24 @@ def upgrade(): ) op.create_index(op.f('ix_release_stime'), 'release', ['stime'], unique=False) op.create_table('package', - sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('id', HackyUUIDType(), nullable=False), sa.Column('name', sa.String(length=120), nullable=False), - sa.Column('stime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), - sa.Column('ftime', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column('stime', ArrowType(), nullable=True), + sa.Column('ftime', ArrowType(), nullable=True), sa.Column('duration', sa.Interval(), nullable=True), sa.Column('status', sa.Enum('NOT_STARTED', 'IN_PROGRESS', 'SUCCESSFUL', 'FAILED', name='status_types'), nullable=True), sa.Column('version', sa.String(length=32), nullable=False), sa.Column('diff_url', sa.String(), nullable=True), sa.Column('rollback', sa.Boolean(), nullable=True), - sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('release_id', HackyUUIDType(), nullable=True), sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id') ) op.create_index(op.f('ix_package_stime'), 'package', ['stime'], unique=False) op.create_table('release_metadata', - sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), - sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('release_id', HackyUUIDType(), nullable=True), sa.Column('key', sa.Text(), nullable=False), sa.Column('value', sa.Text(), nullable=False), sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), @@ -62,23 +86,23 @@ def upgrade(): sa.UniqueConstraint('id') ) op.create_table('release_note', - sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('id', HackyUUIDType(), nullable=False), sa.Column('content', sa.Text(), nullable=False), - sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('release_id', HackyUUIDType(), nullable=True), sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id') ) op.create_table('release_platform', - sa.Column('release_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), - sa.Column('platform_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.Column('platform_id', HackyUUIDType(), nullable=True), sa.ForeignKeyConstraint(['platform_id'], ['platform.id'], ), sa.ForeignKeyConstraint(['release_id'], ['release.id'], ) ) op.create_table('package_result', - sa.Column('id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=False), + sa.Column('id', HackyUUIDType(), nullable=False), sa.Column('content', sa.Text(), nullable=True), - sa.Column('package_id', sqlalchemy_utils.types.uuid.UUIDType(length=16), nullable=True), + sa.Column('package_id', HackyUUIDType(), nullable=True), sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id') From dd1cad5a15d64ac3c9713b635849d717ba08eede Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 19:07:08 +0000 Subject: [PATCH 16/38] Fix up log handling Flask logs shouldn't go through gunicorn --- orlo/__main__.py | 71 +++++++++++++++++++----------------------------- orlo/app.py | 10 +++++-- 2 files changed, 35 insertions(+), 46 deletions(-) diff --git a/orlo/__main__.py b/orlo/__main__.py index d6515f6..6ef9a2f 100644 --- a/orlo/__main__.py +++ b/orlo/__main__.py @@ -4,7 +4,7 @@ import os import traceback -import alembic +import flask import alembic.util.exc as alembic_exc import sqlalchemy.exc from logging.handlers import RotatingFileHandler @@ -13,16 +13,13 @@ from orlo.exceptions import OrloStartupError from orlo.config import config -from orlo.app import app, OrloApplication +from orlo.app import app, OrloApplication, alembic from orlo.orm import db __author__ = 'alforbes' -logger = logging.getLogger('orlo') - - class Start(Command): """ Run the Gunicorn API server @@ -49,41 +46,18 @@ def run(self, loglevel, console, workers): """ log_level = getattr(logging, loglevel.upper()) app.logger.setLevel(log_level) - - formatter = logging.Formatter( - config.get('logging', 'format', raw=True)) - - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - app.logger.addHandler(stream_handler) - - if console: - # Stream handler should use configured log level - stream_handler.setLevel(log_level) - else: - # Only print critical errors from now on - stream_handler.setLevel(logging.CRITICAL) - - log_dir = config.get('logging', 'directory') - logfile = os.path.join(log_dir, 'flask.log') - if log_dir != 'disabled': - file_handler = RotatingFileHandler( - logfile, - maxBytes=1048576, - backupCount=1, - ) - log_format = config.get('logging', 'format') - formatter = logging.Formatter(log_format) - - file_handler.setFormatter(formatter) - file_handler.setLevel(logging.DEBUG) - logger.addHandler(file_handler) + app.logger.propagate = False + # Flask's ProductionHandler is locked at error unless debug mode is + # enabled. We don't necessarily want to enable debug mode whenever we + # capture debug logs, as it's a security risk. + for h in app.logger.handlers: + h.level = log_level app.logger.debug('Debug logging enabled') log_dir = config.get('logging', 'directory') gunicorn_options = { - 'accesslog': os.path.join(log_dir, 'access.log') if not + 'accesslog': os.path.join(log_dir, 'gunicorn_access.log') if not console else '-', 'bind': '%s:%s' % ('0.0.0.0', '5000'), 'capture_output': True, @@ -95,6 +69,7 @@ def run(self, loglevel, console, workers): 'on_starting': on_starting, 'workers': workers, } + app.logger.critical('test logger') try: OrloApplication(app, gunicorn_options).run() except KeyboardInterrupt: @@ -129,30 +104,40 @@ def on_starting(server): def check_database(): - logger.info('Checking database "{}"'.format( + app.logger.info('Checking database "{}"'.format( config.get('db', 'uri') )) + try: + # Can we connect to the DB + db.engine.execute('select 1') + except sqlalchemy.exc.OperationalError: + app.logger.error("Cannot connect to the database, please check the " + "database and configuration.") + raise SystemExit(1) + try: with app.app_context(): current_head = alembic.script_directory.get_current_head() current_rev = alembic.migration_context.get_current_revision() - logger.debug('Current revision: {}'.format(current_rev)) - logger.debug('Current head: {}'.format(current_head)) + app.logger.debug('Current revision: {}'.format(current_rev)) + app.logger.debug('Current head: {}'.format(current_head)) if current_head is None: raise OrloStartupError('No alembic revisions, this is a bug') elif current_rev is None: - logger.info('Database not configured, calling alembic upgrade') + app.logger.info('Database not configured, calling alembic ' + 'upgrade') alembic.upgrade() elif current_head != current_rev: - logger.info('New database revision available, calling alembic ' + app.logger.info('New database revision available, calling ' + 'alembic ' 'upgrade') alembic.upgrade() except sqlalchemy.exc.OperationalError: - logger.warning('Database is not configured, creating tables') + app.logger.warning('Database is not configured, creating tables') db.create_all() except alembic_exc.CommandError: - logger.error( + app.logger.error( 'Alembic raised an exception, please check the state of the ' 'database, and that there aren\'t any extra files in ' '/opt/venvs/orlo/local/lib/python2.7/site-packages/orlo/' @@ -160,7 +145,7 @@ def check_database(): traceback.format_exc())) raise SystemExit(1) - logger.info('Database is configured') + app.logger.info('Database is configured') def main(): diff --git a/orlo/app.py b/orlo/app.py index 64e9678..4f4abb6 100644 --- a/orlo/app.py +++ b/orlo/app.py @@ -59,16 +59,20 @@ log_level = logging.INFO app.logger.setLevel(log_level) + log_format = config.get('logging', 'format') + formatter = Formatter(log_format) + + for h in app.logger.handlers: + h.setFormatter(formatter) + log_dir = config.get('logging', 'directory') - logfile = os.path.join(log_dir, 'flask.log') + logfile = os.path.join(log_dir, 'orlo.log') if log_dir != 'disabled': file_handler = RotatingFileHandler( logfile, maxBytes=1048576, backupCount=1, ) - log_format = config.get('logging', 'format') - formatter = Formatter(log_format) file_handler.setFormatter(formatter) file_handler.setLevel(logging.DEBUG) From e40b2d4625ae771b88c6f1e7ffd1508c9584cf4c Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 24 Nov 2016 19:30:55 +0000 Subject: [PATCH 17/38] Alembic migration WIP --- orlo/__main__.py | 4 +- .../e60a77e44da8_initial_db_revision.py | 130 ++++++++++-------- 2 files changed, 74 insertions(+), 60 deletions(-) diff --git a/orlo/__main__.py b/orlo/__main__.py index 6ef9a2f..484159f 100644 --- a/orlo/__main__.py +++ b/orlo/__main__.py @@ -69,7 +69,6 @@ def run(self, loglevel, console, workers): 'on_starting': on_starting, 'workers': workers, } - app.logger.critical('test logger') try: OrloApplication(app, gunicorn_options).run() except KeyboardInterrupt: @@ -130,8 +129,7 @@ def check_database(): alembic.upgrade() elif current_head != current_rev: app.logger.info('New database revision available, calling ' - 'alembic ' - 'upgrade') + 'alembic upgrade') alembic.upgrade() except sqlalchemy.exc.OperationalError: app.logger.warning('Database is not configured, creating tables') diff --git a/orlo/migrations/e60a77e44da8_initial_db_revision.py b/orlo/migrations/e60a77e44da8_initial_db_revision.py index 66e76ea..23a9394 100644 --- a/orlo/migrations/e60a77e44da8_initial_db_revision.py +++ b/orlo/migrations/e60a77e44da8_initial_db_revision.py @@ -41,71 +41,87 @@ def __repr__(self): def upgrade(): ### commands auto generated by Alembic - please adjust! ### - op.create_table('platform', - sa.Column('id', HackyUUIDType(), nullable=False), - sa.Column('name', sa.Text(), nullable=False), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('id'), - sa.UniqueConstraint('name') + op.create_table( + 'platform', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('name', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + sa.UniqueConstraint('name'), + checkfirst=True ) - op.create_table('release', - sa.Column('id', HackyUUIDType(), nullable=False), - sa.Column('references', sa.String(), nullable=True), - sa.Column('stime', ArrowType(), nullable=True), - sa.Column('ftime', ArrowType(), nullable=True), - sa.Column('duration', sa.Interval(), nullable=True), - sa.Column('user', sa.String(), nullable=False), - sa.Column('team', sa.String(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('id') + op.create_table( + 'release', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('references', sa.String(), nullable=True), + sa.Column('stime', ArrowType(), nullable=True), + sa.Column('ftime', ArrowType(), nullable=True), + sa.Column('duration', sa.Interval(), nullable=True), + sa.Column('user', sa.String(), nullable=False), + sa.Column('team', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + checkfirst=True ) op.create_index(op.f('ix_release_stime'), 'release', ['stime'], unique=False) - op.create_table('package', - sa.Column('id', HackyUUIDType(), nullable=False), - sa.Column('name', sa.String(length=120), nullable=False), - sa.Column('stime', ArrowType(), nullable=True), - sa.Column('ftime', ArrowType(), nullable=True), - sa.Column('duration', sa.Interval(), nullable=True), - sa.Column('status', sa.Enum('NOT_STARTED', 'IN_PROGRESS', 'SUCCESSFUL', 'FAILED', name='status_types'), nullable=True), - sa.Column('version', sa.String(length=32), nullable=False), - sa.Column('diff_url', sa.String(), nullable=True), - sa.Column('rollback', sa.Boolean(), nullable=True), - sa.Column('release_id', HackyUUIDType(), nullable=True), - sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('id') + op.create_table( + 'package', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('stime', ArrowType(), nullable=True), + sa.Column('ftime', ArrowType(), nullable=True), + sa.Column('duration', sa.Interval(), nullable=True), + sa.Column('status', sa.Enum('NOT_STARTED', 'IN_PROGRESS', 'SUCCESSFUL', 'FAILED', name='status_types'), nullable=True), + sa.Column('version', sa.String(length=32), nullable=False), + sa.Column('diff_url', sa.String(), nullable=True), + sa.Column('rollback', sa.Boolean(), nullable=True), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + checkfirst=True ) + op.create_index(op.f('ix_package_stime'), 'package', ['stime'], unique=False) - op.create_table('release_metadata', - sa.Column('id', HackyUUIDType(), nullable=False), - sa.Column('release_id', HackyUUIDType(), nullable=True), - sa.Column('key', sa.Text(), nullable=False), - sa.Column('value', sa.Text(), nullable=False), - sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('id') + op.create_table( + 'release_metadata', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.Column('key', sa.Text(), nullable=False), + sa.Column('value', sa.Text(), nullable=False), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + checkfirst=True ) - op.create_table('release_note', - sa.Column('id', HackyUUIDType(), nullable=False), - sa.Column('content', sa.Text(), nullable=False), - sa.Column('release_id', HackyUUIDType(), nullable=True), - sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('id') + op.create_table( + 'release_note', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + checkfirst=True ) - op.create_table('release_platform', - sa.Column('release_id', HackyUUIDType(), nullable=True), - sa.Column('platform_id', HackyUUIDType(), nullable=True), - sa.ForeignKeyConstraint(['platform_id'], ['platform.id'], ), - sa.ForeignKeyConstraint(['release_id'], ['release.id'], ) + op.create_table( + 'release_platform', + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.Column('platform_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['platform_id'], ['platform.id'], ), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + checkfirst=True ) - op.create_table('package_result', - sa.Column('id', HackyUUIDType(), nullable=False), - sa.Column('content', sa.Text(), nullable=True), - sa.Column('package_id', HackyUUIDType(), nullable=True), - sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('id') + + op.create_table( + 'package_result', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('content', sa.Text(), nullable=True), + sa.Column('package_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + checkfirst=True ) ### end Alembic commands ### From 9cd770fd80a44726377320f1233bb8f1b0be254b Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Thu, 30 Mar 2017 11:28:48 +0100 Subject: [PATCH 18/38] Use extras hack for tests_require test dependencies now declared only in one place --- setup.py | 17 ++++++++++++----- tox.ini | 6 +----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index d84c650..61b2ebf 100755 --- a/setup.py +++ b/setup.py @@ -14,6 +14,13 @@ version_file.write("__version__ = '{}'".format(VERSION)) version_file.close() +tests_require = [ + 'Flask-Testing', + 'orloclient>=0.3.0', + 'mockldap', + 'pytest', +] + setup( name='orlo', @@ -38,17 +45,17 @@ 'Flask-TokenAuth', 'arrow', 'gunicorn', - 'orloclient', + 'orloclient>=0.3.0', 'psycopg2', 'pyldap', 'pytz', 'sphinxcontrib-httpdomain', 'sqlalchemy-utils', ], - tests_require=[ - 'Flask-Testing', - 'orloclient>=0.1.1', - ], + extras_require={ + 'test': tests_require, + }, + tests_require=tests_require, test_suite='tests', # Creates a script in /usr/local/bin entry_points={ diff --git a/tox.ini b/tox.ini index beadf0b..feda0ea 100644 --- a/tox.ini +++ b/tox.ini @@ -9,8 +9,4 @@ envlist = py27, py34 [testenv] commands = py.test {posargs} passenv = TRAVIS -deps = - pytest - Flask-Testing - orloclient - mockldap +deps = .[test] From daf594f6ee4332096ad6c20bc02244a211c590d3 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 13:00:27 +0100 Subject: [PATCH 19/38] Update vagrantfile --- Vagrantfile | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 01d58f4..1b81e3e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -67,36 +67,24 @@ Vagrant.configure(2) do |config| source /home/vagrant/virtualenv/orlo/bin/activate echo "source ~/virtualenv/orlo/bin/activate" >> /home/vagrant/.profile - pip install -r /vagrant/orlo/requirements.txt - pip install -r /vagrant/orlo/requirements_testing.txt + cd /vagrant/orlo + pip install .[test] pip install -r /vagrant/orlo/docs/requirements.txt mkdir -p /etc/orlo /var/log/orlo chown -R vagrant:root /etc/orlo /var/log/orlo chown -R vagrant:vagrant /home/vagrant/virtualenv chown vagrant:root /vagrant - - # Create the database - #cd /vagrant/orlo - #python create_db.py - #python setup.py develop - SHELL config.vm.define "jessie" do |jessie| - jessie.vm.box = "bento/debian-8.6" + jessie.vm.box = "bento/debian-8.7" jessie.vm.network "forwarded_port", guest: 5000, host: 5000 jessie.vm.network "private_network", ip: "192.168.57.20" jessie.vm.provision "shell", inline: <<-SHELL SHELL end -# config.vm.define "trusty" do |trusty| -# trusty.vm.box = "bento/ubuntu-14.04" -# trusty.vm.network "forwarded_port", guest: 5000, host: 5100 -# trusty.vm.network "private_network", ip: "192.168.57.10" -# end - config.vm.define "xenial" do |xenial| xenial.vm.box = "bento/ubuntu-16.04" xenial.vm.network "forwarded_port", guest: 5000, host: 5200 From 522353487d9f2d286e19b25374da6dae7be12b67 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 13:03:39 +0100 Subject: [PATCH 20/38] Add libpq-dev to vagrantfile, required for psycopg2 --- Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Vagrantfile b/Vagrantfile index 1b81e3e..7b3e346 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -29,6 +29,7 @@ Vagrant.configure(2) do |config| dh-systemd \ git-buildpackage \ postgresql-client \ + libpq-dev \ mysql-client \ python-all-dev \ python-dev \ From c61de296335a9e350ec4dc222be69eab9745412a Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 13:15:10 +0100 Subject: [PATCH 21/38] Upgrade pip/setuptools in virtualenv --- Vagrantfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7b3e346..de44d61 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -51,12 +51,8 @@ Vagrant.configure(2) do |config| libsasl2-dev \ libssl-dev \ - # Updating build tooling can help - sudo pip install --upgrade \ - pip \ - setuptools \ - stdeb \ - virtualenv \ + sudo pip install --upgrade pip setuptools + sudo pip install --upgrade stdeb virtualenv wget -P /tmp/ \ 'http://launchpadlibrarian.net/291737817/dh-virtualenv_1.0-1_all.deb' @@ -68,9 +64,12 @@ Vagrant.configure(2) do |config| source /home/vagrant/virtualenv/orlo/bin/activate echo "source ~/virtualenv/orlo/bin/activate" >> /home/vagrant/.profile + pip install --upgrade pip setuptools + cd /vagrant/orlo pip install .[test] pip install -r /vagrant/orlo/docs/requirements.txt + python setup.py develop mkdir -p /etc/orlo /var/log/orlo chown -R vagrant:root /etc/orlo /var/log/orlo From 98adec1e0404d5df0b02edaf8b0c73342ef56f16 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 13:27:39 +0100 Subject: [PATCH 22/38] Depend on orloclient 0.2.0 for now, until 0.3.0 is released --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 61b2ebf..7f6572d 100755 --- a/setup.py +++ b/setup.py @@ -16,9 +16,10 @@ tests_require = [ 'Flask-Testing', - 'orloclient>=0.3.0', + 'orloclient>=0.2.0', 'mockldap', 'pytest', + 'tox', ] @@ -45,7 +46,7 @@ 'Flask-TokenAuth', 'arrow', 'gunicorn', - 'orloclient>=0.3.0', + 'orloclient>=0.2.0', 'psycopg2', 'pyldap', 'pytz', From 87d184174cec4d03ef1978d4c5496eb5349ab524 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 14:53:59 +0100 Subject: [PATCH 23/38] Add release_id to package dictionaries --- orlo/orm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/orlo/orm.py b/orlo/orm.py index 756b703..210fe70 100644 --- a/orlo/orm.py +++ b/orlo/orm.py @@ -178,12 +178,13 @@ def to_dict(self): 'id': str(self.id), 'name': self.name, 'version': self.version, - 'stime': self.stime.strftime(config.get('main', 'time_format')) if self.stime else None, - 'ftime': self.ftime.strftime(config.get('main', 'time_format')) if self.ftime else None, + 'stime': self.stime.strftime(time_format) if self.stime else None, + 'ftime': self.ftime.strftime(time_format) if self.ftime else None, 'duration': self.duration.seconds if self.duration else None, 'rollback': self.rollback, 'status': self.status, 'diff_url': self.diff_url, + 'release_id': self.release_id, } From bd1e27b10d76e6916dd2a413a21d424f1b0d5641 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 14:58:17 +0100 Subject: [PATCH 24/38] Remove /deploy functionality This, in my opinion, was an overreach given the resources available. For now, Orlo will remain a simple receptor and not a release coordinator. --- orlo/deploy.py | 166 ---------------------------------------- orlo/routes/packages.py | 1 - orlo/routes/releases.py | 29 ------- setup.py | 2 +- tests/test_deploy.py | 115 ---------------------------- 5 files changed, 1 insertion(+), 312 deletions(-) delete mode 100644 orlo/deploy.py delete mode 100644 tests/test_deploy.py diff --git a/orlo/deploy.py b/orlo/deploy.py deleted file mode 100644 index e87d445..0000000 --- a/orlo/deploy.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import print_function -import json -import subprocess -from threading import Timer -from orlo.config import config -from orlo.exceptions import OrloDeployError -from orlo.app import app - -__author__ = 'alforbes' - - -class BaseDeploy(object): - """ - A Deploy task - - Deploy tasks are simpler than releases, as they consist of just packages - and versions. The backend deployer is responsible for creating Orlo Releases - via the REST API. - - Integrations can either sub-class Deploy and override the start method and - (optionally) the kill method, or use the pre-built shell and http - integrations. - """ - - def __init__(self, release): - """ - Perform a release - """ - self.release = release - self.server_url = config.get('main', 'base_url') - - def start(self): - """ - Start the deployment - """ - raise NotImplementedError("Please override the start method") - - def kill(self): - """ - Kill a deployment in progress - """ - raise NotImplementedError("Please override the kill method") - - -class HttpDeploy(BaseDeploy): - """ - A http-based Deployment - """ - - def __init__(self, release): - super(HttpDeploy, self).__init__(release) - - def start(self): - pass - - def kill(self): - raise NotImplementedError("No kill method for HTTP deploys") - - -class ShellDeploy(BaseDeploy): - """ - Deployment by shell command - - Data is passed to the shell command given in 3 ways: - - * ORLO_URL, ORLO_RELEASE (the ID), and other Release attributes - are set as environment variables (all prefixed by ORLO_) - * The package and version sets are passed as arguments, e.g. - package-name=1.0.0 - * The metadata dictionary is passed to stdin - """ - - def __init__(self, release): - super(ShellDeploy, self).__init__(release) - self.pid = None - - def start(self): - """ - Start the deploy - """ - args = [config.get('deploy_shell', 'command_path')] - for p in self.release.packages: - args.append("{}={}".format(p.name, p.version)) - app.logger.debug("Args: {}".format(str(args))) - - env = { - 'ORLO_URL': self.server_url, - 'ORLO_RELEASE': str(self.release.id) - } - for key, value in self.release.to_dict().items(): - my_key = "ORLO_" + key.upper() - env[my_key] = json.dumps(value) - - app.logger.debug("Env: {}".format(json.dumps(env))) - - metadata = {} - for m in self.release.metadata: - metadata.update(m.to_dict()) - in_data = json.dumps(metadata) - - return self.run_command( - args, env, in_data, timeout_sec=config.getint('deploy', 'timeout')) - - def kill(self): - """ - Kill a deploy in progress - """ - raise NotImplementedError - - @staticmethod - def run_command(args, env, in_data, timeout_sec=3600): - """ - Run a command in a separate thread - - Adapted from http://stackoverflow.com/questions/1191374 - - :param env: Dict of environment variables - :param in_data: String to pass to stdin - :param list args: Arguments (i.e. full list including command, - as you would pass to subprocess) - :param timeout_sec: Timeout in seconds, 1 hour by default - :return: - """ - try: - proc = subprocess.Popen( - args, - env=env, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError as e: - raise OrloDeployError( - message="OSError starting process: {}".format( - e.strerror), - payload={'arguments': args} - ) - - timer = Timer(timeout_sec, proc.kill) - out = err = " " - try: - timer.start() - out, err = proc.communicate(in_data.encode('utf-8')) - finally: - timer.cancel() - app.logger.debug("Out:\n{}".format(out)) - app.logger.debug("Err:\n{}".format(err)) - - if proc.returncode is not 0: - raise OrloDeployError( - message="Subprocess exited with code {}".format( - proc.returncode), - payload={ - 'stdout': out, - 'stderr': err, - }, - status_code=500) - else: - app.logger.info( - "Deploy completed successfully. Output:\n{}".format(out)) - app.logger.debug("end run") - - return { - 'stdout': out, - 'stderr': err, - } diff --git a/orlo/routes/packages.py b/orlo/routes/packages.py index 1fa8417..8af718d 100644 --- a/orlo/routes/packages.py +++ b/orlo/routes/packages.py @@ -9,7 +9,6 @@ from orlo.util import validate_request_json, create_release, \ validate_release_input, validate_package_input, fetch_release, \ create_package, fetch_package, stream_json_list, str_to_bool, is_uuid -from orlo.deploy import ShellDeploy from orlo.user_auth import conditional_auth __author__ = 'alforbes' diff --git a/orlo/routes/releases.py b/orlo/routes/releases.py index f19576f..169e8cf 100644 --- a/orlo/routes/releases.py +++ b/orlo/routes/releases.py @@ -9,7 +9,6 @@ from orlo.util import validate_request_json, create_release, \ validate_release_input, validate_package_input, fetch_release, \ create_package, fetch_package, stream_json_list, str_to_bool, is_uuid -from orlo.deploy import ShellDeploy from orlo.user_auth import conditional_auth security_enabled = config.getboolean('security', 'enabled') @@ -124,34 +123,6 @@ def post_results(release_id, package_id): return '', 204 -@app.route('/releases//deploy', methods=['POST']) -@conditional_auth(token_auth.token_required) -def post_releases_deploy(release_id): - """ - Deploy a Release - - :param string release_id: Release UUID - - **Example curl**: - - .. sourcecode:: shell - - curl -H "Content-Type: application/json" \\ - -X POST http://127.0.0.1/releases/${RELEASE_ID}/deploy - """ - release = fetch_release(release_id) - app.logger.info("Release start, release {}".format(release_id)) - - release.start() - db.session.add(release) - db.session.commit() - - # TODO call deploy Class start Method, i.e. pure python rather than shell - deploy = ShellDeploy(release) - output = deploy.start() - return jsonify(output), 200 - - @app.route('/releases//start', methods=['POST']) @conditional_auth(token_auth.token_required) def post_releases_start(release_id): diff --git a/setup.py b/setup.py index 7f6572d..0e88bc6 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -VERSION = '0.3.1' +VERSION = '0.3.2' version_file = open(os.path.join(__location__, 'orlo', '_version.py'), 'w') version_file.write("__version__ = '{}'".format(VERSION)) diff --git a/tests/test_deploy.py b/tests/test_deploy.py deleted file mode 100644 index e13d854..0000000 --- a/tests/test_deploy.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import print_function -from test_base import OrloLiveTest, OrloTest, ConfigChange, ReleaseDbUtil -from orlo.deploy import BaseDeploy, HttpDeploy, ShellDeploy -from orlo.orm import db, Release, Package -from test_base import OrloLiveTest -from test_base import ReleaseDbUtil -import unittest - -__author__ = 'alforbes' - - -class DeployTest(OrloLiveTest, ReleaseDbUtil): - """ - Test the Deploy class - """ - CLASS = BaseDeploy - - def setUp(self): - super(DeployTest, self).setUp() - rid = self._create_release() - pid = self._create_package(rid) - self.release = db.session.query(Release).first() - - def test_init(self): - """ - Test that we can instantiate the class - """ - o = self.CLASS(self.release) - self.assertIsInstance(o, BaseDeploy) - - -class TestBaseDeploy(DeployTest): - def test_not_implemented(self): - """ - Base Deploy class should raise NotImplementedError on start - """ - o = self.CLASS(self.release) - with self.assertRaises(NotImplementedError): - o.start() - - -class TestHttpDeploy(DeployTest): - CLASS = HttpDeploy - - @unittest.skip("Not implemented") - def test_start(self): - """ - Test that start emits an http call - """ - pass - - @unittest.skip("Not implemented") - def test_kill(self): - """ - Test that kill emits an http call - """ - pass - - -class TestShellDeploy(DeployTest): - CLASS = ShellDeploy - - def test_start(self): - """ - Test that start emits a shell command - """ - with ConfigChange('deploy', 'timeout', '3'), \ - ConfigChange('deploy_shell', 'command_path', '/bin/true'): - deploy = ShellDeploy(self.release) - - # Override server_url, normally it is set by config: - deploy.server_url = self.get_server_url() - - deploy.start() - - @unittest.skip("Not implemented") - def test_kill(self): - """ - Test that kill emits a shell command - """ - pass - - @unittest.skip("Doesn't work on travis") - def test_start_example_deployer(self): - """ - Test the example deployer completes - - DOESN'T WORK ON TRAVIS, as /bin/env python gives the system python - """ - with ConfigChange('deploy', 'timeout', '3'): - deploy = ShellDeploy(self.release) - - # Override server_url, normally it is set by config: - deploy.server_url = self.get_server_url() - - deploy.start() - - def test_output(self): - """ - Test that we return the output of the deploy - - Not a good test, as it relies on the test-package being an argument, - and simply echoing it back. This is the "spec", but this test could - break if the arguments change. - """ - with ConfigChange('deploy', 'timeout', '3'), \ - ConfigChange('deploy_shell', 'command_path', '/bin/echo'): - deploy = ShellDeploy(self.release) - - # Override server_url, normally it is set by config: - deploy.server_url = self.get_server_url() - - output = deploy.start() - self.assertEqual(output['stdout'], b'test-package=1.2.3\n') - From b6e8e357ce4945a05a34d95f7ad9880bdadb809f Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 15:08:07 +0100 Subject: [PATCH 25/38] Remove deployer-related files --- deployer.py | 121 ------------------------------------------------- deployer.rb | 110 -------------------------------------------- orlo/config.py | 9 ---- 3 files changed, 240 deletions(-) delete mode 100755 deployer.py delete mode 100755 deployer.rb diff --git a/deployer.py b/deployer.py deleted file mode 100755 index d074962..0000000 --- a/deployer.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -import argparse -import fileinput -import json -import logging -from collections import OrderedDict -from logging import error, warn, info -from orloclient import OrloClient, Release, Package -import os - -__author__ = 'alforbes' - -""" -An example deployer. This is used to run tests against, but could also be used -as a basis for Orlo integration. -""" - -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - - -class DeployerError(Exception): - def __init__(self, message): - self.message = message - error(message) - - -def get_params(): - parser = argparse.ArgumentParser(description="Orlo Test Deployer") - parser.add_argument('packages', choices=None, const=None, nargs='*', - help="Packages to deploy, in format " - "package_name=version, e.g test-package=1.0.0") - args, unknown = parser.parse_known_args() - - _packages = OrderedDict() - - for pkg in args.packages: - package, version = pkg.split('=') - _packages[package] = version - - # Fetch metadata from stdin - s_metadata = "" - for line in fileinput.input(unknown): - s_metadata += line - - try: - _metadata = json.loads(s_metadata) - except ValueError: - raise DeployerError("Could not parse json from stdin") - - return _packages, _metadata - - -def deploy(package, meta=None): - """ - Dummy deployment function - - :param Package package: Package to deploy - :param dict meta: Dictionary of metadata (unused in this dummy function) - :return: - """ - info("Package start - {}:{}".format(package.name, package.version)) - orlo_client.package_start(package) - - # Do stuff - # Determining success status is up to you - success = True - - info("Package stop - {}:{}".format(package.name, package.version)) - orlo_client.package_stop(package, success=success) - - return success - - -if __name__ == "__main__": - if os.getenv('ORLO_URL'): - orlo_url = os.environ['ORLO_URL'] - else: - # This deployer is only every supposed to accept releases from Orlo - # Other deployers could use this to detect whether they are being - # invoked by Orlo - raise DeployerError("Could not detect ORLO_URL from environment") - - if os.getenv('ORLO_RELEASE'): - orlo_release_id = os.environ['ORLO_RELEASE'] - else: - raise DeployerError("Could not detect ORLO_RELEASE in environment") - - logging.info( - "Environment: \n" + - json.dumps(os.environ, indent=2, default=lambda o: o.__dict__)) - - # Fetch packages and metadata. Packages is not used, it is just to - # demonstrate they are passed as arguments - packages, metadata = get_params() - - logging.info("Stdin: \n" + str(metadata)) - - # Create an instance of the Orlo client - orlo_client = OrloClient(uri=orlo_url) - - # The release is created in Orlo before the deployer is invoked, so fetch - # it here. If you prefer, you can to do the release creation within your - # deployer and use Orlo only for receiving data - release = orlo_client.get_release(orlo_release_id) - - # While we fetch Packages using the Orlo client, they are passed on the - # CLI as well, which is useful for non-python deployers - info("Fetching packages from Orlo") - if not release.packages: - raise DeployerError("No packages to deploy") - - info("Starting Release") - for pkg in release.packages: - info("Deploying {}".format(pkg.name)) - deploy(pkg, meta=metadata) - - info("Finishing Release") - orlo_client.release_stop(release) - - info("Done.") diff --git a/deployer.rb b/deployer.rb deleted file mode 100755 index f2dd5af..0000000 --- a/deployer.rb +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env ruby - -require 'json' -require 'net/http' -require 'openssl' -require 'uri' - -def orlo_request(method, url, body = nil) - if $http.nil? - return - end - - puts " -> %s" % [ url ] - - request = method.new(url, initheader = { - "Content-Type" => "application/json" - }) - request.basic_auth("testuser", "blabla") - - if ! body.nil? - request.body = JSON(body) - end - - response = $http.request(request) - - if ! response.kind_of?(Net::HTTPSuccess) - raise "#{request.path} returned #{response.code}" - end - - if ! response.body.nil? and response.body.length > 0 - JSON.parse(response.body) - end -end - -def orlo_get(url) - orlo_request(Net::HTTP::Get, url) -end - -def orlo_post(url, body = nil) - orlo_request(Net::HTTP::Post, url, body) -end - -puts "Dummy Deployer v0.1" - -if ENV["ORLO_URL"].nil? - $http = nil -else - $uri = URI.parse(ENV["ORLO_URL"]) - $http = Net::HTTP.new($uri.host, $uri.port) - if $uri.is_a?(URI::HTTPS) - $http.use_ssl = true -# $http.verify_mode = OpenSSL::SSL::VERIFY_NONE - end -end - -if ENV["ORLO_RELEASE"] - releases = orlo_get("/releases/#{ENV["ORLO_RELEASE"]}") - release = releases["releases"][0] - deployables = release["packages"].collect do |package| - [ package["name"], package["version"] ] - end -else - release = orlo_post("/releases", { - "user" => ENV["ORLO_USER"], - "team" => ENV["ORLO_TEAM"], - "platforms" => ENV["ORLO_PLATFORMS"], - "references" => ENV["ORLO_REFERENCES"], - }) - deployables = ARGV.collect do |deployable| - deployable.split("=") - end -end - -if release - puts "Orlo release ID #{release["id"]}" -end - -deployables.each do |deployable| - pkgname, version = deployable - - if release - package = orlo_post("/releases/#{release["id"]}/packages", { - "name" => pkgname, - "version" => version, - "rollback" => !ENV["ORLO_ROLLBACK"].nil?, - }) - end - - puts "Installing #{pkgname} version #{version}" - - if release - orlo_post("/releases/#{release["id"]}/packages/#{package["id"]}/start") - end - - puts "# apt-get install #{pkgname}=#{version}" - sleep 1 # ... wow, much install ... - - if release - orlo_post("/releases/#{release["id"]}/packages/#{package["id"]}/stop", { - "success" => true, - }) - end -end - -if release - orlo_post("/releases/#{release["id"]}/stop") -end - -puts "Finished!" -exit 0 diff --git a/orlo/config.py b/orlo/config.py index aa6e1a8..f1db405 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -47,13 +47,4 @@ 'message)s') config.set('logging', 'directory', '/var/log/orlo') # disabled for no log files -config.add_section('deploy') -# How long to timeout external deployer calls -config.set('deploy', 'timeout', '3600') - -config.add_section('deploy_shell') -config.set('deploy_shell', 'command_path', - os.path.dirname(os.path.abspath(__file__)) + - '/../deployer.py') - config.read(CONFIG_FILE) From f057aed5558bed830068b79dfeed0b58baf562a5 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 16:53:05 +0100 Subject: [PATCH 26/38] Configure postgres in vagrant image --- Vagrantfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index de44d61..b5263b8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -72,6 +72,9 @@ Vagrant.configure(2) do |config| python setup.py develop mkdir -p /etc/orlo /var/log/orlo + echo -e "[db]\nuri=postgres://orlo:password@192.168.57.100" > /etc/orlo/orlo.ini + + chown -R vagrant:root /etc/orlo /var/log/orlo chown -R vagrant:vagrant /home/vagrant/virtualenv chown vagrant:root /vagrant From bf17d5b8318ee02cd4db37516499fb87b0972b6c Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 16:53:31 +0100 Subject: [PATCH 27/38] Stamp pre-existing databases with the initial revision All installations I know about are up to date enough that this shouldn't cause problems; there have been few database changes since Orlo was conceived. --- orlo/__main__.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/orlo/__main__.py b/orlo/__main__.py index 484159f..1ad2169 100644 --- a/orlo/__main__.py +++ b/orlo/__main__.py @@ -102,6 +102,14 @@ def on_starting(server): check_database() +def stamp_initial_revision(): + """ Stamp the database with the initial revision """ + app.logger.warning("*** Stamping the database with the initial revision. \ +This can result in database inconsistencies, please check the schema if you \ +experience crashes. ***") + alembic.migration_context.stamp(alembic.script_directory, + "e60a77e44da8") + def check_database(): app.logger.info('Checking database "{}"'.format( config.get('db', 'uri') @@ -126,14 +134,30 @@ def check_database(): elif current_rev is None: app.logger.info('Database not configured, calling alembic ' 'upgrade') - alembic.upgrade() + try: + alembic.upgrade() + except sqlalchemy.exc.ProgrammingError as e: + # if this occurs on upgrade from None, it's probably because + # tables already existed + app.logger.error("Database migration failed: {}".format( + e.message)) + if e.message.endswith('relation "platform" already exists\n'): + app.logger.warning( + "This error is expected when installing an " + "alembic-enabled build for the first time, " + "continuing") + stamp_initial_revision() + else: + raise elif current_head != current_rev: app.logger.info('New database revision available, calling ' 'alembic upgrade') alembic.upgrade() except sqlalchemy.exc.OperationalError: - app.logger.warning('Database is not configured, creating tables') - db.create_all() + if 'sqlite' in app.config['SQLALCHEMY_DATABASE_URI']: + app.logger.warning('Database is not configured, creating tables') + db.create_all() + stamp_initial_revision() except alembic_exc.CommandError: app.logger.error( 'Alembic raised an exception, please check the state of the ' @@ -142,6 +166,8 @@ def check_database(): 'migrations. Exception was:\n{}'.format( traceback.format_exc())) raise SystemExit(1) + finally: + alembic.migration_context.connection.close() app.logger.info('Database is configured') From 54de9e136d6316d9d9ade3197d214a9990aa2c3c Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 16:59:50 +0100 Subject: [PATCH 28/38] Remove checkfirst from create_table statements Not a valid parameter --- orlo/migrations/e60a77e44da8_initial_db_revision.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/orlo/migrations/e60a77e44da8_initial_db_revision.py b/orlo/migrations/e60a77e44da8_initial_db_revision.py index 23a9394..5e9ce6f 100644 --- a/orlo/migrations/e60a77e44da8_initial_db_revision.py +++ b/orlo/migrations/e60a77e44da8_initial_db_revision.py @@ -48,7 +48,6 @@ def upgrade(): sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), sa.UniqueConstraint('name'), - checkfirst=True ) op.create_table( 'release', @@ -61,7 +60,6 @@ def upgrade(): sa.Column('team', sa.String(), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), - checkfirst=True ) op.create_index(op.f('ix_release_stime'), 'release', ['stime'], unique=False) op.create_table( @@ -79,7 +77,6 @@ def upgrade(): sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), - checkfirst=True ) op.create_index(op.f('ix_package_stime'), 'package', ['stime'], unique=False) @@ -92,7 +89,6 @@ def upgrade(): sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), - checkfirst=True ) op.create_table( 'release_note', @@ -102,7 +98,6 @@ def upgrade(): sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), - checkfirst=True ) op.create_table( 'release_platform', @@ -110,7 +105,6 @@ def upgrade(): sa.Column('platform_id', HackyUUIDType(), nullable=True), sa.ForeignKeyConstraint(['platform_id'], ['platform.id'], ), sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), - checkfirst=True ) op.create_table( @@ -121,7 +115,6 @@ def upgrade(): sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('id'), - checkfirst=True ) ### end Alembic commands ### From ab900ae1e7ce598499dcc1abb5763d44d7849ae1 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 17:35:08 +0100 Subject: [PATCH 29/38] Comment out postgres config for now --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index b5263b8..f707f0b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -72,7 +72,7 @@ Vagrant.configure(2) do |config| python setup.py develop mkdir -p /etc/orlo /var/log/orlo - echo -e "[db]\nuri=postgres://orlo:password@192.168.57.100" > /etc/orlo/orlo.ini + # echo -e "[db]\nuri=postgres://orlo:password@192.168.57.100" > /etc/orlo/orlo.ini chown -R vagrant:root /etc/orlo /var/log/orlo From 8fdbab18119b4180488bdd1d73f8511f4842c088 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 17:46:00 +0100 Subject: [PATCH 30/38] Suggested changes, courtesy of Carlo Bongiovanni * Make default /releases output descending order * Add default limit of 100 to releases output * Remove message if filter is not supplied --- orlo/app.py | 2 +- orlo/queries.py | 10 +++++----- orlo/routes/releases.py | 18 ++++++++---------- tests/test_route_releases.py | 24 +++++++++--------------- 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/orlo/app.py b/orlo/app.py index 4f4abb6..c921429 100644 --- a/orlo/app.py +++ b/orlo/app.py @@ -25,7 +25,7 @@ app.config['SQLALCHEMY_DATABASE_URI'] = config.get('db', 'uri') app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -if 'sqlite' not in app.config['SQLALCHEMY_DATABASE_URI']: +if not app.config['SQLALCHEMY_DATABASE_URI'].startswith('sqlite'): # SQLite doesn't support these app.config['SQLALCHEMY_POOL_SIZE'] = config.getint('db', 'pool_size') app.config['SQLALCHEMY_POOL_RECYCLE'] = 2 diff --git a/orlo/queries.py b/orlo/queries.py index 73643ed..77bf923 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -246,13 +246,13 @@ def apply_package_filters(query, args): return query -def releases(limit=None, offset=None, desc=None, **kwargs): +def releases(limit=None, offset=None, asc=None, **kwargs): """ Return whole releases, based on filters :param limit: Max number of results to return :param offset: Offset results. Provides pagination when combined with limit. - :param desc: Sort descending instead of the default ascending order + :param asc: Sort ascending instead of the default descending order :param kwargs: Request arguments :return: """ @@ -277,10 +277,10 @@ def releases(limit=None, offset=None, desc=None, **kwargs): raise InvalidUsage( "An invalid field was specified: {}".format(e.args[0])) - if desc: - stime_field = Release.stime.desc - else: + if asc: stime_field = Release.stime.asc + else: + stime_field = Release.stime.desc query = query.order_by(stime_field()) diff --git a/orlo/routes/releases.py b/orlo/routes/releases.py index 169e8cf..2d307b0 100644 --- a/orlo/routes/releases.py +++ b/orlo/routes/releases.py @@ -285,10 +285,10 @@ def get_releases(release_id=None): :param string release_id: Optionally specify a single release UUID to fetch. This does not disable filters. - :query bool desc: Normally results are returned ordered by stime - ascending, setting desc to true will reverse this and sort by stime - descending - :query int limit: Limit the results by int + :query bool asc: Normally results are returned ordered by stime + descending, setting asc to true will reverse this and sort by stime + ascending + :query int limit: Limit the results by int (default 100) :query int offset: Offset the results by int :query string package_name: Filter releases by package name :query string user: Filter releases by user the that performed the release @@ -328,7 +328,6 @@ def get_releases(release_id=None): value. If any one package has the value "IN_PROGRESS" or "FAILED", that status applies to the whole release, with "FAILED" overriding "IN_PROGRESS". - """ booleans = ('rollback', 'package_rollback',) @@ -337,14 +336,13 @@ def get_releases(release_id=None): if not is_uuid(release_id): raise InvalidUsage("Release ID given is not a valid UUID") query = queries.get_release(release_id) - elif len([x for x in request.args.keys()]) == 0: - raise InvalidUsage("Please specify a filter. See " - "http://orlo.readthedocs.org/en/latest/rest.html" - "#get--releases for more info") else: # Bit more complex + # Defaults + args = { + 'limit': 100 + } # Flatten args, as the ImmutableDict puts some values in a list when # expanded - args = {} for k in request.args.keys(): if k in booleans: args[k] = str_to_bool(request.args.get(k)) diff --git a/tests/test_route_releases.py b/tests/test_route_releases.py index d6d0c8b..6f82675 100644 --- a/tests/test_route_releases.py +++ b/tests/test_route_releases.py @@ -478,16 +478,18 @@ def test_get_release_offset(self): """ Test that offset=1 skips the first release """ - rid = None + rids = [] for _ in range(0, 2): - rid = self._create_release() + rids.append(self._create_release()) sleep(0.1) r = self._get_releases(filters=['offset=1']) self.assertEqual(len(r['releases']), 1) - self.assertEqual(r['releases'][0]['id'], rid) + # Default order is now descending, so the skip means we're skipping the + # last release, so the release given should be the first created. + self.assertEqual(r['releases'][0]['id'], rids[0]) - def test_get_release_desc(self): + def test_get_release_asc(self): """ Should return in reverse order """ @@ -497,9 +499,9 @@ def test_get_release_desc(self): rid = self._create_release() sleep(0.1) - r = self._get_releases(filters=['desc=true']) - # First in list should be last to be created - self.assertEqual(r['releases'][0]['id'], rid) + r = self._get_releases(filters=['asc=true']) + # Last in list should be last to be created + self.assertEqual(r['releases'][2]['id'], rid) def test_get_release_package_name(self): """ @@ -680,11 +682,3 @@ def test_get_release_with_bad_status(self): result = self._get_releases(filters=['status=garbage_boz'], expected_status=400) self.assertIn('message', result) - - def test_get_releases_with_no_filters(self): - """ - Test get /releases without filters returns 400 - """ - self._get_releases(expected_status=400) - - From 7949ceded9a1c3497e1b648799195f8d3ad1eda7 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 10:50:29 +0100 Subject: [PATCH 31/38] Remove obsolete and empty test file --- tests/test_deployer.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 tests/test_deployer.py diff --git a/tests/test_deployer.py b/tests/test_deployer.py deleted file mode 100644 index 89ca0b4..0000000 --- a/tests/test_deployer.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import print_function -import unittest - -""" -Test the example deployer -""" - - -class TestDeployer(unittest.TestCase): - ENV = {} - ARGS = {} - METADATA = {} - - def test_exit_0(self): - """ - Test the deployer exits 0 - """ \ No newline at end of file From ea10fcbb3c140f5a9516abc9900ac14b9f5cf9ee Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 10:50:52 +0100 Subject: [PATCH 32/38] Allow overriding of log directory by environment And disable logging for tox runs --- orlo/app.py | 2 +- orlo/config.py | 21 +++++++++++++++------ tox.ini | 1 + 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/orlo/app.py b/orlo/app.py index c921429..a389931 100644 --- a/orlo/app.py +++ b/orlo/app.py @@ -11,7 +11,7 @@ from flask import Flask from flask_alembic import Alembic -from orlo.config import config, CONFIG_FILE +from orlo.config import config from orlo.exceptions import OrloStartupError diff --git a/orlo/config.py b/orlo/config.py index f1db405..ad2ea4a 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -4,10 +4,18 @@ __author__ = 'alforbes' -try: - CONFIG_FILE = os.environ['ORLO_CONFIG'] -except KeyError: - CONFIG_FILE = '/etc/orlo/orlo.ini' + +# Defaults that can be overridden by environment variables +defaults = { + 'ORLO_CONFIG': '/etc/orlo/orlo.ini', + 'ORLO_LOGDIR': '/var/log/orlo', +} + +for var, default in defaults.items(): + try: + defaults[var] = os.environ[var] + except KeyError: + pass config = RawConfigParser() @@ -45,6 +53,7 @@ config.set('logging', 'format', '%(asctime)s [%(name)s] %(levelname)s %(' 'module)s:%(funcName)s:%(lineno)d - %(' 'message)s') -config.set('logging', 'directory', '/var/log/orlo') # disabled for no log files +config.set('logging', 'directory', defaults['ORLO_LOGDIR']) # "disabled" for no + # log files -config.read(CONFIG_FILE) +config.read(defaults['ORLO_CONFIG']) diff --git a/tox.ini b/tox.ini index feda0ea..e5c2cda 100644 --- a/tox.ini +++ b/tox.ini @@ -10,3 +10,4 @@ envlist = py27, py34 commands = py.test {posargs} passenv = TRAVIS deps = .[test] +setenv = ORLO_LOGDIR = disabled From 4c4e4d5d26b3ff865132dd31b18670d4cc260762 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 10:52:41 +0100 Subject: [PATCH 33/38] Upgrade postgres to 9.4 in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 642067b..9002e58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ addons: apt: packages: - curl - postgresql: "9.3" + postgresql: "9.4" services: - postgresql From 2760445715f4becbdc99a6f9627d66af73c087f8 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 10:57:09 +0100 Subject: [PATCH 34/38] Remove unneeded requirements.txt files Now defined in setup.py --- requirements.txt | 14 -------------- requirements_testing.txt | 20 -------------------- 2 files changed, 34 deletions(-) delete mode 100644 requirements.txt delete mode 100644 requirements_testing.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 591387a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -gunicorn -arrow>=0.7.0 -Flask>=0.10.1 -Flask-SQLAlchemy>=2.0 -Flask-Migrate>=1.5.1 -Flask-HTTPAuth>=2.7.1 -Flask-TokenAuth>=0.0.2 -requests>=2.4.2 -SQLAlchemy>=1.0.8 -sqlalchemy-utils>=0.31.0 -psycopg2>=2.6.1 -pyldap>=2.4.25 -pytz -six>=1.10.0 diff --git a/requirements_testing.txt b/requirements_testing.txt deleted file mode 100644 index 0382455..0000000 --- a/requirements_testing.txt +++ /dev/null @@ -1,20 +0,0 @@ -Flask-HTTPAuth>=2.7.1 -Flask-Migrate>=1.5.1 -Flask-SQLAlchemy>=2.0 -Flask-TokenAuth>=0.0.2 -Flask>=0.10.1 -SQLAlchemy>=1.0.8 -arrow>=0.7.0 -gunicorn -orloclient>=0.1.1 -passlib>=1.6.1 -psycopg2>=2.6.1 -pytest -pytz -requests>=2.4.2 -sqlalchemy-utils>=0.31.0 -mock==1.3.0 -mockldap>=0.2.6 -pyldap>=2.4.25 -six>=1.10.0 -git+https://github.com/al4/flask-testing.git From 56ba172d83f8d5e2a33a7e6be372bc5848a96f8b Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 12:06:04 +0100 Subject: [PATCH 35/38] Use trusty travis image rather than precise --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9002e58..14f5c5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +dist: trusty notifications: slack: ebayclassifiedsgroup:lH5V2FnojyNCh8X84Qi1FKjk From b4fceddccf255a8067c8ca4c74a67f3a72641817 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 13:08:00 +0100 Subject: [PATCH 36/38] Increase tox verbosity --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 14f5c5c..0a3b456 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,4 +25,4 @@ before_install: - pip install tox tox-travis script: - - tox -- --maxfail=2 + - tox -v -- --maxfail=2 From 5d06c3854ad53236433183ff5fea042fff350395 Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Tue, 4 Apr 2017 14:09:46 +0100 Subject: [PATCH 37/38] Fix packages entry in setup.py orlo.routes is another package, perhaps it wasn't being included sometimes? --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0e88bc6..05b04cc 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # from distutils.core import setup -from setuptools import setup +from setuptools import setup, find_packages import multiprocessing # nopep8 import os @@ -32,9 +32,9 @@ license='GPL', long_description=open(os.path.join(__location__, 'README.md')).read(), url='https://github.com/eBayClassifiedsGroup/orlo', - packages=[ - 'orlo', - ], + packages=find_packages( + exclude=['tests', 'debian', 'docs', 'etc'] + ), include_package_data=True, install_requires=[ 'Flask', From 7db0223f6a1f9a8344329c9a8f9fe2c141fc901d Mon Sep 17 00:00:00 2001 From: Alex Forbes Date: Mon, 3 Apr 2017 15:00:03 +0100 Subject: [PATCH 38/38] Bump version to 0.4.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 05b04cc..ec6ed65 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -VERSION = '0.3.2' +VERSION = '0.4.0' version_file = open(os.path.join(__location__, 'orlo', '_version.py'), 'w') version_file.write("__version__ = '{}'".format(VERSION))