diff --git a/.gitignore b/.gitignore index fd51a5d..9764dac 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,10 @@ var/ *.egg-info/ .installed.cfg *.egg +debian/*.debhelper +debian/*.substvars +debian/orlo +debian/files # PyInstaller # Usually these files are written by a python script from a template diff --git a/.travis.yml b/.travis.yml index 642067b..0a3b456 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +dist: trusty notifications: slack: ebayclassifiedsgroup:lH5V2FnojyNCh8X84Qi1FKjk @@ -11,7 +12,7 @@ addons: apt: packages: - curl - postgresql: "9.3" + postgresql: "9.4" services: - postgresql @@ -24,4 +25,4 @@ before_install: - pip install tox tox-travis script: - - tox -- --maxfail=2 + - tox -v -- --maxfail=2 diff --git a/MANIFEST.in b/MANIFEST.in index bb3ec5f..b268a86 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include README.md +recursive-include orlo/migrations * diff --git a/Makefile b/Makefile index 68449f5..9cb9e5d 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # Uses git buildpackage, which from debian rules will call dh_virtualenv test: - python setup.py test + tox sdist: python setup.py sdist @@ -12,10 +12,10 @@ clean: python setup.py clean debuild clean -debian: +deb: debuild -us -uc changelog: - gbp dch --ignore-branch --auto --commit + gbp dch --ignore-branch --auto --commit debian .PHONY: debian sdist test clean changelog diff --git a/Vagrantfile b/Vagrantfile index efcf28c..f707f0b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,6 +13,7 @@ Vagrant.configure(2) do |config| end config.vm.provision "shell", inline: <<-SHELL + echo 'en_GB.UTF-8 UTF-8' | tee -a /etc/locale.gen sudo localedef -i en_GB -f UTF-8 en_GB.UTF-8 sudo locale-gen en_GB.UTF-8 sudo sed -i 's/us.archive.ubuntu.com/nl.archive.ubuntu.com/g' /etc/apt/sources.list @@ -28,6 +29,7 @@ Vagrant.configure(2) do |config| dh-systemd \ git-buildpackage \ postgresql-client \ + libpq-dev \ mysql-client \ python-all-dev \ python-dev \ @@ -49,12 +51,8 @@ Vagrant.configure(2) do |config| libsasl2-dev \ libssl-dev \ - # Updating build tooling can help - sudo pip install --upgrade \ - pip \ - setuptools \ - stdeb \ - virtualenv \ + sudo pip install --upgrade pip setuptools + sudo pip install --upgrade stdeb virtualenv wget -P /tmp/ \ 'http://launchpadlibrarian.net/291737817/dh-virtualenv_1.0-1_all.deb' @@ -66,36 +64,30 @@ Vagrant.configure(2) do |config| source /home/vagrant/virtualenv/orlo/bin/activate echo "source ~/virtualenv/orlo/bin/activate" >> /home/vagrant/.profile - pip install -r /vagrant/orlo/requirements.txt - pip install -r /vagrant/orlo/requirements_testing.txt + pip install --upgrade pip setuptools + + cd /vagrant/orlo + pip install .[test] pip install -r /vagrant/orlo/docs/requirements.txt + python setup.py develop mkdir -p /etc/orlo /var/log/orlo + # echo -e "[db]\nuri=postgres://orlo:password@192.168.57.100" > /etc/orlo/orlo.ini + + chown -R vagrant:root /etc/orlo /var/log/orlo chown -R vagrant:vagrant /home/vagrant/virtualenv chown vagrant:root /vagrant - - # Create the database - #cd /vagrant/orlo - #python create_db.py - #python setup.py develop - SHELL config.vm.define "jessie" do |jessie| - jessie.vm.box = "bento/debian-8.6" + jessie.vm.box = "bento/debian-8.7" jessie.vm.network "forwarded_port", guest: 5000, host: 5000 jessie.vm.network "private_network", ip: "192.168.57.20" jessie.vm.provision "shell", inline: <<-SHELL SHELL end -# config.vm.define "trusty" do |trusty| -# trusty.vm.box = "bento/ubuntu-14.04" -# trusty.vm.network "forwarded_port", guest: 5000, host: 5100 -# trusty.vm.network "private_network", ip: "192.168.57.10" -# end - config.vm.define "xenial" do |xenial| xenial.vm.box = "bento/ubuntu-16.04" xenial.vm.network "forwarded_port", guest: 5000, host: 5200 diff --git a/create_db.py b/create_db.py deleted file mode 100644 index 92a57ba..0000000 --- a/create_db.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import print_function - -__author__ = 'alforbes' - -from orlo import app -from orlo.orm import db - -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///orlo.db' -app.debug = True -db.create_all() - - diff --git a/debian/changelog b/debian/changelog index 02d24cd..c9014cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,56 @@ +orlo (0.3.1-3) UNRELEASED; urgency=medium + + [ Alex Forbes ] + * Fix exec path + + [ vagrant ] + + -- Alex Forbes Thu, 24 Nov 2016 10:09:28 +0000 + +orlo (0.3.1-2) jessie; urgency=medium + + [ Alex Forbes ] + * Fix path to gunicorn in systemd unit + + [ vagrant ] + + -- Alex Forbes Wed, 23 Nov 2016 18:02:42 +0000 + +orlo (0.3.1-1) jessie; urgency=medium + + [ Alex Forbes ] + * Fix test + * Rename __init__ to test_base and update imports + * Create multiple vagrant machines for testing + * Add jessie and xenial vagrant boxes for build/testing + * Run python setup.py clean in make clean + * Change deb install location to /opt/venvs + * Add initial tox.ini + * Default internal config should be sqlite + * Configure tests to use postgres in vagrant + * Remove trusty vm, fix IP + * Use main repos, upgrade dh-virtualenv, fix missing \ + * Add db clients and restart postgres + * Small doc fix + * Remove dependency on passwd file by mocking the verification function + * Use tox to run tests in travis + * mock has moved under py3 + * Set test database to localhost if running in Travis + * Set posargs for tox and maxfail for pytest + * Pass travis env car through to tox + * Rename DeployTest as LiveDbTest, move to test_base + * Add stress test + * Add liveserver config to app, refactor db test with classmethods + * Close the session in our json streamer to avoid leaking connections + * Test refactor + * Fix test failure caused by previous tests not clearing db connection + * Bump version to 0.3.1 + * Fix locale in vagrantfile (for me anyway 😁), use tox for testing + + [ vagrant ] + + -- Alex Forbes Wed, 23 Nov 2016 17:39:48 +0000 + orlo (0.3.1) trusty; urgency=medium * Add support for filtering by reference diff --git a/debian/systemd/orlo.service b/debian/systemd/orlo.service index 46860b2..8ffff86 100644 --- a/debian/systemd/orlo.service +++ b/debian/systemd/orlo.service @@ -3,13 +3,13 @@ [Unit] Description=orlo After=network.target -ConditionPathExists=/usr/share/python/orlo/bin/gunicorn +ConditionPathExists=/opt/venvs/orlo/bin/gunicorn [Service] Type=simple User=orlo Group=orlo -ExecStart=/usr/share/python/orlo/bin/gunicorn \ +ExecStart=/opt/venvs/orlo/bin/gunicorn \ -w 4 -b 127.0.0.1:8080 \ --access-logfile /var/log/orlo/gunicorn-access.log \ --log-level debug \ diff --git a/deployer.py b/deployer.py deleted file mode 100755 index d074962..0000000 --- a/deployer.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -import argparse -import fileinput -import json -import logging -from collections import OrderedDict -from logging import error, warn, info -from orloclient import OrloClient, Release, Package -import os - -__author__ = 'alforbes' - -""" -An example deployer. This is used to run tests against, but could also be used -as a basis for Orlo integration. -""" - -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - - -class DeployerError(Exception): - def __init__(self, message): - self.message = message - error(message) - - -def get_params(): - parser = argparse.ArgumentParser(description="Orlo Test Deployer") - parser.add_argument('packages', choices=None, const=None, nargs='*', - help="Packages to deploy, in format " - "package_name=version, e.g test-package=1.0.0") - args, unknown = parser.parse_known_args() - - _packages = OrderedDict() - - for pkg in args.packages: - package, version = pkg.split('=') - _packages[package] = version - - # Fetch metadata from stdin - s_metadata = "" - for line in fileinput.input(unknown): - s_metadata += line - - try: - _metadata = json.loads(s_metadata) - except ValueError: - raise DeployerError("Could not parse json from stdin") - - return _packages, _metadata - - -def deploy(package, meta=None): - """ - Dummy deployment function - - :param Package package: Package to deploy - :param dict meta: Dictionary of metadata (unused in this dummy function) - :return: - """ - info("Package start - {}:{}".format(package.name, package.version)) - orlo_client.package_start(package) - - # Do stuff - # Determining success status is up to you - success = True - - info("Package stop - {}:{}".format(package.name, package.version)) - orlo_client.package_stop(package, success=success) - - return success - - -if __name__ == "__main__": - if os.getenv('ORLO_URL'): - orlo_url = os.environ['ORLO_URL'] - else: - # This deployer is only every supposed to accept releases from Orlo - # Other deployers could use this to detect whether they are being - # invoked by Orlo - raise DeployerError("Could not detect ORLO_URL from environment") - - if os.getenv('ORLO_RELEASE'): - orlo_release_id = os.environ['ORLO_RELEASE'] - else: - raise DeployerError("Could not detect ORLO_RELEASE in environment") - - logging.info( - "Environment: \n" + - json.dumps(os.environ, indent=2, default=lambda o: o.__dict__)) - - # Fetch packages and metadata. Packages is not used, it is just to - # demonstrate they are passed as arguments - packages, metadata = get_params() - - logging.info("Stdin: \n" + str(metadata)) - - # Create an instance of the Orlo client - orlo_client = OrloClient(uri=orlo_url) - - # The release is created in Orlo before the deployer is invoked, so fetch - # it here. If you prefer, you can to do the release creation within your - # deployer and use Orlo only for receiving data - release = orlo_client.get_release(orlo_release_id) - - # While we fetch Packages using the Orlo client, they are passed on the - # CLI as well, which is useful for non-python deployers - info("Fetching packages from Orlo") - if not release.packages: - raise DeployerError("No packages to deploy") - - info("Starting Release") - for pkg in release.packages: - info("Deploying {}".format(pkg.name)) - deploy(pkg, meta=metadata) - - info("Finishing Release") - orlo_client.release_stop(release) - - info("Done.") diff --git a/deployer.rb b/deployer.rb deleted file mode 100755 index f2dd5af..0000000 --- a/deployer.rb +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env ruby - -require 'json' -require 'net/http' -require 'openssl' -require 'uri' - -def orlo_request(method, url, body = nil) - if $http.nil? - return - end - - puts " -> %s" % [ url ] - - request = method.new(url, initheader = { - "Content-Type" => "application/json" - }) - request.basic_auth("testuser", "blabla") - - if ! body.nil? - request.body = JSON(body) - end - - response = $http.request(request) - - if ! response.kind_of?(Net::HTTPSuccess) - raise "#{request.path} returned #{response.code}" - end - - if ! response.body.nil? and response.body.length > 0 - JSON.parse(response.body) - end -end - -def orlo_get(url) - orlo_request(Net::HTTP::Get, url) -end - -def orlo_post(url, body = nil) - orlo_request(Net::HTTP::Post, url, body) -end - -puts "Dummy Deployer v0.1" - -if ENV["ORLO_URL"].nil? - $http = nil -else - $uri = URI.parse(ENV["ORLO_URL"]) - $http = Net::HTTP.new($uri.host, $uri.port) - if $uri.is_a?(URI::HTTPS) - $http.use_ssl = true -# $http.verify_mode = OpenSSL::SSL::VERIFY_NONE - end -end - -if ENV["ORLO_RELEASE"] - releases = orlo_get("/releases/#{ENV["ORLO_RELEASE"]}") - release = releases["releases"][0] - deployables = release["packages"].collect do |package| - [ package["name"], package["version"] ] - end -else - release = orlo_post("/releases", { - "user" => ENV["ORLO_USER"], - "team" => ENV["ORLO_TEAM"], - "platforms" => ENV["ORLO_PLATFORMS"], - "references" => ENV["ORLO_REFERENCES"], - }) - deployables = ARGV.collect do |deployable| - deployable.split("=") - end -end - -if release - puts "Orlo release ID #{release["id"]}" -end - -deployables.each do |deployable| - pkgname, version = deployable - - if release - package = orlo_post("/releases/#{release["id"]}/packages", { - "name" => pkgname, - "version" => version, - "rollback" => !ENV["ORLO_ROLLBACK"].nil?, - }) - end - - puts "Installing #{pkgname} version #{version}" - - if release - orlo_post("/releases/#{release["id"]}/packages/#{package["id"]}/start") - end - - puts "# apt-get install #{pkgname}=#{version}" - sleep 1 # ... wow, much install ... - - if release - orlo_post("/releases/#{release["id"]}/packages/#{package["id"]}/stop", { - "success" => true, - }) - end -end - -if release - orlo_post("/releases/#{release["id"]}/stop") -end - -puts "Finished!" -exit 0 diff --git a/etc/orlo.ini b/etc/orlo.ini index 18b2b2c..8a48152 100644 --- a/etc/orlo.ini +++ b/etc/orlo.ini @@ -5,11 +5,14 @@ echo_queries = false [main] time_zone = UTC -propagate_exceptions = true ; Format to represent date/time as time_format = %Y-%m-%dT%H:%M:%SZ base_url = http://localhost:8080 +[flask] +propagate_exceptions = true +debug = true + [security] enabled = false method = file @@ -19,5 +22,5 @@ secret_key = change_me token_ttl = 3600 [logging] -file = /var/log/orlo/app.log -debug = false +directory = /var/log/orlo +level = debug diff --git a/orlo/__init__.py b/orlo/__init__.py index 9a111c8..122d9e5 100644 --- a/orlo/__init__.py +++ b/orlo/__init__.py @@ -1,87 +1,11 @@ from __future__ import print_function, division, absolute_import from __future__ import unicode_literals -from flask import Flask -import logging -from logging.handlers import RotatingFileHandler -from logging import Formatter -import sys +from pkg_resources import get_distribution -from orlo.config import config, CONFIG_FILE -from orlo.exceptions import OrloStartupError, OrloError, OrloAuthError, \ - OrloConfigError +__version__ = get_distribution(__name__).version -try: - # _version is created by setup.py - from orlo._version import __version__ -except ImportError: - __version__ = "TEST_BUILD" +from orlo.config import config +from orlo.app import app as app -app = Flask(__name__) - -app.config['SQLALCHEMY_DATABASE_URI'] = config.get('db', 'uri') -app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - -if 'sqlite' not in app.config['SQLALCHEMY_DATABASE_URI']: - # SQLite doesn't support these - app.config['SQLALCHEMY_POOL_SIZE'] = config.getint('db', 'pool_size') - app.config['SQLALCHEMY_POOL_RECYCLE'] = 1 - app.config['SQLALCHEMY_MAX_OVERFLOW'] = 10 - -if config.getboolean('main', 'propagate_exceptions'): - app.config['PROPAGATE_EXCEPTIONS'] = True - -if config.getboolean('db', 'echo_queries'): - app.config['SQLALCHEMY_ECHO'] = True - -if not config.getboolean('main', 'strict_slashes'): - app.url_map.strict_slashes = False - -# Debug mode ignores all custom logging and should only be used in -# local testing... -if config.getboolean('main', 'debug_mode'): - app.debug = True - -if not app.debug: - log_level = config.get('logging', 'level') - if log_level == 'debug': - app.logger.setLevel(logging.DEBUG) - elif log_level == 'info': - app.logger.setLevel(logging.INFO) - elif log_level == 'warning': - app.logger.setLevel(logging.WARNING) - elif log_level == 'error': - app.logger.setLevel(logging.ERROR) - - logfile = config.get('logging', 'file') - if logfile != 'disabled': - file_handler = RotatingFileHandler( - logfile, - maxBytes=1048576, - backupCount=1, - ) - log_format = config.get('logging', 'format') - formatter = Formatter(log_format) - - file_handler.setFormatter(formatter) - app.logger.addHandler(file_handler) - -if config.getboolean('security', 'enabled') and \ - config.get('security', 'secret_key') == 'change_me': - raise OrloStartupError( - "Security is enabled, please configure the secret key") - -app.logger.debug("Log level: {}".format(config.get('logging', 'level'))) - -# Must be imported last -import orlo.error_handlers -import orlo.route_base -import orlo.route_releases -import orlo.route_packages -import orlo.route_import -import orlo.route_info -import orlo.route_stats -import orlo.user_auth - -app.logger.info("Startup completed with configuration from {}".format( - CONFIG_FILE)) +app.logger.info("Initialisation completed") diff --git a/orlo/__main__.py b/orlo/__main__.py new file mode 100644 index 0000000..1ad2169 --- /dev/null +++ b/orlo/__main__.py @@ -0,0 +1,188 @@ +from __future__ import print_function + +import logging +import os +import traceback + +import flask +import alembic.util.exc as alembic_exc +import sqlalchemy.exc +from logging.handlers import RotatingFileHandler +from flask_alembic import alembic_script +from flask_script import Manager, Command, Option, Server +from orlo.exceptions import OrloStartupError + +from orlo.config import config +from orlo.app import app, OrloApplication, alembic +from orlo.orm import db + + +__author__ = 'alforbes' + + +class Start(Command): + """ + Run the Gunicorn API server + """ + + option_list = ( + Option('-l', '--loglevel', default=config.get('logging', 'level'), + choices=['debug', 'info', 'warning', 'error', 'critical'], + help='Set logging level'), + Option('-c', '--log-console', default=False, dest='console', + action='store_true', + help="Log to console instead of configured log files"), + Option('-w', '--workers', default=config.get('gunicorn', 'workers'), + help="Number of gunicorn workers to start"), + ) + + def run(self, loglevel, console, workers): + """ + Start the production server + + @param loglevel: + @param console: + @return: + """ + log_level = getattr(logging, loglevel.upper()) + app.logger.setLevel(log_level) + app.logger.propagate = False + # Flask's ProductionHandler is locked at error unless debug mode is + # enabled. We don't necessarily want to enable debug mode whenever we + # capture debug logs, as it's a security risk. + for h in app.logger.handlers: + h.level = log_level + + app.logger.debug('Debug logging enabled') + + log_dir = config.get('logging', 'directory') + gunicorn_options = { + 'accesslog': os.path.join(log_dir, 'gunicorn_access.log') if not + console else '-', + 'bind': '%s:%s' % ('0.0.0.0', '5000'), + 'capture_output': True, + 'errorlog': os.path.join(log_dir, 'gunicorn_error.log') if not + console else '-', + 'logfile': os.path.join(log_dir, 'gunicorn.log') if not + console else '-', + 'loglevel': loglevel or config.get('logging', 'level'), + 'on_starting': on_starting, + 'workers': workers, + } + try: + OrloApplication(app, gunicorn_options).run() + except KeyboardInterrupt: + app.logger.info('Caught KeyboardInterrupt') + app.logger.debug('__main__ done') + + +class WriteConfig(Command): + """ + Write out the Orlo configuration file + """ + option_list = ( + Option('file', dest='file_path', help='Config file to write', + default='/etc/orlo/orlo.ini') + ) + + def run(self, file_path): + """ Write out configuration """ + config_file = open(file_path, 'w') + config.write(config_file) + + +script_manager = Manager(app) +script_manager.add_command('db', alembic_script) +script_manager.add_command('start', Start) +script_manager.add_command('run_dev_server', Server(host='0.0.0.0', port=5000)) + + +def on_starting(server): + app.logger.debug('on_starting called') + check_database() + + +def stamp_initial_revision(): + """ Stamp the database with the initial revision """ + app.logger.warning("*** Stamping the database with the initial revision. \ +This can result in database inconsistencies, please check the schema if you \ +experience crashes. ***") + alembic.migration_context.stamp(alembic.script_directory, + "e60a77e44da8") + +def check_database(): + app.logger.info('Checking database "{}"'.format( + config.get('db', 'uri') + )) + + try: + # Can we connect to the DB + db.engine.execute('select 1') + except sqlalchemy.exc.OperationalError: + app.logger.error("Cannot connect to the database, please check the " + "database and configuration.") + raise SystemExit(1) + + try: + with app.app_context(): + current_head = alembic.script_directory.get_current_head() + current_rev = alembic.migration_context.get_current_revision() + app.logger.debug('Current revision: {}'.format(current_rev)) + app.logger.debug('Current head: {}'.format(current_head)) + if current_head is None: + raise OrloStartupError('No alembic revisions, this is a bug') + elif current_rev is None: + app.logger.info('Database not configured, calling alembic ' + 'upgrade') + try: + alembic.upgrade() + except sqlalchemy.exc.ProgrammingError as e: + # if this occurs on upgrade from None, it's probably because + # tables already existed + app.logger.error("Database migration failed: {}".format( + e.message)) + if e.message.endswith('relation "platform" already exists\n'): + app.logger.warning( + "This error is expected when installing an " + "alembic-enabled build for the first time, " + "continuing") + stamp_initial_revision() + else: + raise + elif current_head != current_rev: + app.logger.info('New database revision available, calling ' + 'alembic upgrade') + alembic.upgrade() + except sqlalchemy.exc.OperationalError: + if 'sqlite' in app.config['SQLALCHEMY_DATABASE_URI']: + app.logger.warning('Database is not configured, creating tables') + db.create_all() + stamp_initial_revision() + except alembic_exc.CommandError: + app.logger.error( + 'Alembic raised an exception, please check the state of the ' + 'database, and that there aren\'t any extra files in ' + '/opt/venvs/orlo/local/lib/python2.7/site-packages/orlo/' + 'migrations. Exception was:\n{}'.format( + traceback.format_exc())) + raise SystemExit(1) + finally: + alembic.migration_context.connection.close() + + app.logger.info('Database is configured') + + +def main(): + """ + Entry point for setuptools to call + """ + try: + script_manager.run() + except StandardError: + # Should print here as there is no guarantee logging is working + tb = traceback.format_exc() + print('Exception on execution:\n{}'.format(tb)) + + +if __name__ == "__main__": + main() diff --git a/orlo/app.py b/orlo/app.py new file mode 100644 index 0000000..a389931 --- /dev/null +++ b/orlo/app.py @@ -0,0 +1,112 @@ +from __future__ import print_function + +import os +import logging +import gunicorn.app.base +from gunicorn.six import iteritems + +from logging import Formatter +from logging.handlers import RotatingFileHandler + +from flask import Flask +from flask_alembic import Alembic + +from orlo.config import config +from orlo.exceptions import OrloStartupError + + +__author__ = 'alforbes' + +app = Flask(__name__) + +alembic = Alembic() +alembic.init_app(app) + +app.config['SQLALCHEMY_DATABASE_URI'] = config.get('db', 'uri') +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +if not app.config['SQLALCHEMY_DATABASE_URI'].startswith('sqlite'): + # SQLite doesn't support these + app.config['SQLALCHEMY_POOL_SIZE'] = config.getint('db', 'pool_size') + app.config['SQLALCHEMY_POOL_RECYCLE'] = 2 + app.config['SQLALCHEMY_MAX_OVERFLOW'] = 10 + +if config.getboolean('flask', 'propagate_exceptions'): + app.config['PROPAGATE_EXCEPTIONS'] = True + +if config.getboolean('db', 'echo_queries'): + app.config['SQLALCHEMY_ECHO'] = True + +if not config.getboolean('main', 'strict_slashes'): + app.url_map.strict_slashes = False + +# Debug mode ignores all custom logging and should only be used in +# local testing... +if config.getboolean('flask', 'debug'): + app.debug = True + +if not app.debug: + # If you are calling orlo.app directly rather than with the embedded + # gunicorn, you may want to add a streamHandler + try: + _level = config.get('logging', 'level') + log_level = getattr(logging, _level.upper()) + except AttributeError: + app.logger.error( + 'Failed to set log level to {}, see ' + 'https://docs.python.org/3.6/library/logging.html#logging-levels ' + 'for valid levels.'.format(_level)) + log_level = logging.INFO + app.logger.setLevel(log_level) + + log_format = config.get('logging', 'format') + formatter = Formatter(log_format) + + for h in app.logger.handlers: + h.setFormatter(formatter) + + log_dir = config.get('logging', 'directory') + logfile = os.path.join(log_dir, 'orlo.log') + if log_dir != 'disabled': + file_handler = RotatingFileHandler( + logfile, + maxBytes=1048576, + backupCount=1, + ) + + file_handler.setFormatter(formatter) + file_handler.setLevel(logging.DEBUG) + + app.logger.addHandler(file_handler) + + +class OrloApplication(gunicorn.app.base.BaseApplication): + """ Gunicorn application """ + def __init__(self, application, options=None): + self.options = options or {} + self.application = application + super(OrloApplication, self).__init__() + + def load_config(self): + _config = dict([(key, value) for key, value in iteritems(self.options) + if key in self.cfg.settings and value is not None]) + for key, value in iteritems(_config): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + +if config.getboolean('security', 'enabled') and \ + config.get('security', 'secret_key') == 'change_me': + raise OrloStartupError( + "Security is enabled, please configure security:secret_key in orlo.ini") + +app.logger.debug("Log level: {}".format(config.get('logging', 'level'))) + + +# Must be imported last + +import orlo.error_handlers +import orlo.routes +import orlo.user_auth diff --git a/orlo/cli.py b/orlo/cli.py index 723be70..7d1b07c 100644 --- a/orlo/cli.py +++ b/orlo/cli.py @@ -1,12 +1,7 @@ from __future__ import print_function import argparse -try: - from orlo import __version__ -except ImportError: - # _version.py doesn't exist - __version__ = "TEST_BUILD" - +from orlo import __version__ __author__ = 'alforbes' @@ -55,7 +50,7 @@ def parse_args(): def write_config(args): - from orlo import config + from orlo.config import config config_file = open(args.file_path, 'w') config.write(config_file) @@ -75,7 +70,7 @@ def run_server(args): print("Warning: this is a development server and not suitable " "for production, we recommend running under gunicorn.") - from orlo import app + from orlo.app import app app.config['DEBUG'] = True app.config['TRAP_HTTP_EXCEPTIONS'] = True app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False diff --git a/orlo/config.py b/orlo/config.py index bfac44e..ad2ea4a 100644 --- a/orlo/config.py +++ b/orlo/config.py @@ -4,21 +4,30 @@ __author__ = 'alforbes' -try: - CONFIG_FILE = os.environ['ORLO_CONFIG'] -except KeyError: - CONFIG_FILE = '/etc/orlo/orlo.ini' + +# Defaults that can be overridden by environment variables +defaults = { + 'ORLO_CONFIG': '/etc/orlo/orlo.ini', + 'ORLO_LOGDIR': '/var/log/orlo', +} + +for var, default in defaults.items(): + try: + defaults[var] = os.environ[var] + except KeyError: + pass config = RawConfigParser() config.add_section('main') -config.set('main', 'debug_mode', 'false') -config.set('main', 'propagate_exceptions', 'true') config.set('main', 'time_format', '%Y-%m-%dT%H:%M:%SZ') config.set('main', 'time_zone', 'UTC') config.set('main', 'strict_slashes', 'false') config.set('main', 'base_url', 'http://localhost:8080') +config.add_section('gunicorn') +config.set('gunicorn', 'workers', '4') + config.add_section('security') config.set('security', 'enabled', 'false') config.set('security', 'passwd_file', 'none') @@ -35,20 +44,16 @@ config.set('db', 'echo_queries', 'false') config.set('db', 'pool_size', '50') +config.add_section('flask') +config.set('flask', 'propagate_exceptions', 'true') +config.set('flask', 'debug', 'false') + config.add_section('logging') config.set('logging', 'level', 'info') -config.set('logging', 'file', 'disabled') config.set('logging', 'format', '%(asctime)s [%(name)s] %(levelname)s %(' 'module)s:%(funcName)s:%(lineno)d - %(' 'message)s') +config.set('logging', 'directory', defaults['ORLO_LOGDIR']) # "disabled" for no + # log files -config.add_section('deploy') -config.set('deploy', 'timeout', - '3600') # How long to timeout external deployer calls - -config.add_section('deploy_shell') -config.set('deploy_shell', 'command_path', - os.path.dirname(os.path.abspath(__file__)) + - '/../deployer.py') - -config.read(CONFIG_FILE) +config.read(defaults['ORLO_CONFIG']) diff --git a/orlo/deploy.py b/orlo/deploy.py deleted file mode 100644 index 166f9a0..0000000 --- a/orlo/deploy.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import print_function -import json -import subprocess -from threading import Timer -from orlo.config import config -from orlo.exceptions import OrloDeployError -from orlo import app - -__author__ = 'alforbes' - - -class BaseDeploy(object): - """ - A Deploy task - - Deploy tasks are simpler than releases, as they consist of just packages - and versions. The backend deployer is responsible for creating Orlo Releases - via the REST API. - - Integrations can either sub-class Deploy and override the start method and - (optionally) the kill method, or use the pre-built shell and http - integrations. - """ - - def __init__(self, release): - """ - Perform a release - """ - self.release = release - self.server_url = config.get('main', 'base_url') - - def start(self): - """ - Start the deployment - """ - raise NotImplementedError("Please override the start method") - - def kill(self): - """ - Kill a deployment in progress - """ - raise NotImplementedError("Please override the kill method") - - -class HttpDeploy(BaseDeploy): - """ - A http-based Deployment - """ - - def __init__(self, release): - super(HttpDeploy, self).__init__(release) - - def start(self): - pass - - def kill(self): - raise NotImplementedError("No kill method for HTTP deploys") - - -class ShellDeploy(BaseDeploy): - """ - Deployment by shell command - - Data is passed to the shell command given in 3 ways: - - * ORLO_URL, ORLO_RELEASE (the ID), and other Release attributes - are set as environment variables (all prefixed by ORLO_) - * The package and version sets are passed as arguments, e.g. - package-name=1.0.0 - * The metadata dictionary is passed to stdin - """ - - def __init__(self, release): - super(ShellDeploy, self).__init__(release) - self.pid = None - - def start(self): - """ - Start the deploy - """ - args = [config.get('deploy_shell', 'command_path')] - for p in self.release.packages: - args.append("{}={}".format(p.name, p.version)) - app.logger.debug("Args: {}".format(str(args))) - - env = { - 'ORLO_URL': self.server_url, - 'ORLO_RELEASE': str(self.release.id) - } - for key, value in self.release.to_dict().items(): - my_key = "ORLO_" + key.upper() - env[my_key] = json.dumps(value) - - app.logger.debug("Env: {}".format(json.dumps(env))) - - metadata = {} - for m in self.release.metadata: - metadata.update(m.to_dict()) - in_data = json.dumps(metadata) - - return self.run_command( - args, env, in_data, timeout_sec=config.getint('deploy', 'timeout')) - - def kill(self): - """ - Kill a deploy in progress - """ - raise NotImplementedError - - @staticmethod - def run_command(args, env, in_data, timeout_sec=3600): - """ - Run a command in a separate thread - - Adapted from http://stackoverflow.com/questions/1191374 - - :param env: Dict of environment variables - :param in_data: String to pass to stdin - :param list args: Arguments (i.e. full list including command, - as you would pass to subprocess) - :param timeout_sec: Timeout in seconds, 1 hour by default - :return: - """ - try: - proc = subprocess.Popen( - args, - env=env, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - except OSError as e: - raise OrloDeployError( - message="OSError starting process: {}".format( - e.strerror), - payload={'arguments': args} - ) - - timer = Timer(timeout_sec, proc.kill) - out = err = " " - try: - timer.start() - out, err = proc.communicate(in_data.encode('utf-8')) - finally: - timer.cancel() - app.logger.debug("Out:\n{}".format(out)) - app.logger.debug("Err:\n{}".format(err)) - - if proc.returncode is not 0: - raise OrloDeployError( - message="Subprocess exited with code {}".format( - proc.returncode), - payload={ - 'stdout': out, - 'stderr': err, - }, - status_code=500) - else: - app.logger.info( - "Deploy completed successfully. Output:\n{}".format(out)) - app.logger.debug("end run") - - return { - 'stdout': out, - 'stderr': err, - } diff --git a/orlo/error_handlers.py b/orlo/error_handlers.py index 00d8947..45d3c9d 100644 --- a/orlo/error_handlers.py +++ b/orlo/error_handlers.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import jsonify, request -from orlo import app +from orlo.app import app from orlo.exceptions import InvalidUsage, OrloError __author__ = 'alforbes' diff --git a/orlo/migrations/e60a77e44da8_initial_db_revision.py b/orlo/migrations/e60a77e44da8_initial_db_revision.py new file mode 100644 index 0000000..5e9ce6f --- /dev/null +++ b/orlo/migrations/e60a77e44da8_initial_db_revision.py @@ -0,0 +1,133 @@ +"""Initial DB revision + +Revision ID: e60a77e44da8 +Revises: +Create Date: 2016-11-24 17:03:25.249133 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy_utils.types.uuid import UUIDType +from sqlalchemy_utils.types.arrow import ArrowType +from orlo.config import config + + +# revision identifiers, used by Alembic. +revision = 'e60a77e44da8' +down_revision = None +branch_labels = ('default',) +depends_on = None + + +class HackyUUIDType(UUIDType): + """ Horrible hack for SQLAlchemy-utils' UUID, which doesn't support + Alembic yet + + For Postgres, we return the UUID dialect, for others CHAR(36) + See https://github.com/kvesteri/sqlalchemy-utils/issues/129 + """ + def __repr__(self): + """ + :return: + """ + uri = config.get('db', 'uri') + if uri.startswith('postgresql'): + return "sa.dialects.postgresql.UUID()" + elif uri.startswith('mysql'): + return "sa.dialects.mysql.CHAR(32)" + else: + return "sa.types.CHAR(32)" + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'platform', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('name', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + sa.UniqueConstraint('name'), + ) + op.create_table( + 'release', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('references', sa.String(), nullable=True), + sa.Column('stime', ArrowType(), nullable=True), + sa.Column('ftime', ArrowType(), nullable=True), + sa.Column('duration', sa.Interval(), nullable=True), + sa.Column('user', sa.String(), nullable=False), + sa.Column('team', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + ) + op.create_index(op.f('ix_release_stime'), 'release', ['stime'], unique=False) + op.create_table( + 'package', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('stime', ArrowType(), nullable=True), + sa.Column('ftime', ArrowType(), nullable=True), + sa.Column('duration', sa.Interval(), nullable=True), + sa.Column('status', sa.Enum('NOT_STARTED', 'IN_PROGRESS', 'SUCCESSFUL', 'FAILED', name='status_types'), nullable=True), + sa.Column('version', sa.String(length=32), nullable=False), + sa.Column('diff_url', sa.String(), nullable=True), + sa.Column('rollback', sa.Boolean(), nullable=True), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + ) + + op.create_index(op.f('ix_package_stime'), 'package', ['stime'], unique=False) + op.create_table( + 'release_metadata', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.Column('key', sa.Text(), nullable=False), + sa.Column('value', sa.Text(), nullable=False), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + ) + op.create_table( + 'release_note', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + ) + op.create_table( + 'release_platform', + sa.Column('release_id', HackyUUIDType(), nullable=True), + sa.Column('platform_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['platform_id'], ['platform.id'], ), + sa.ForeignKeyConstraint(['release_id'], ['release.id'], ), + ) + + op.create_table( + 'package_result', + sa.Column('id', HackyUUIDType(), nullable=False), + sa.Column('content', sa.Text(), nullable=True), + sa.Column('package_id', HackyUUIDType(), nullable=True), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('id'), + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('package_result') + op.drop_table('release_platform') + op.drop_table('release_note') + op.drop_table('release_metadata') + op.drop_index(op.f('ix_package_stime'), table_name='package') + op.drop_table('package') + op.drop_index(op.f('ix_release_stime'), table_name='release') + op.drop_table('release') + op.drop_table('platform') + ### end Alembic commands ### diff --git a/orlo/migrations/script.py.mako b/orlo/migrations/script.py.mako new file mode 100644 index 0000000..8f45a34 --- /dev/null +++ b/orlo/migrations/script.py.mako @@ -0,0 +1,23 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/orlo/orm.py b/orlo/orm.py index c63bfa2..210fe70 100644 --- a/orlo/orm.py +++ b/orlo/orm.py @@ -2,7 +2,8 @@ from sqlalchemy_utils.types.uuid import UUIDType from sqlalchemy_utils.types.arrow import ArrowType -from orlo import app, config +from orlo.app import app +from orlo.config import config from orlo.exceptions import OrloWorkflowError import pytz import uuid @@ -177,12 +178,13 @@ def to_dict(self): 'id': str(self.id), 'name': self.name, 'version': self.version, - 'stime': self.stime.strftime(config.get('main', 'time_format')) if self.stime else None, - 'ftime': self.ftime.strftime(config.get('main', 'time_format')) if self.ftime else None, + 'stime': self.stime.strftime(time_format) if self.stime else None, + 'ftime': self.ftime.strftime(time_format) if self.ftime else None, 'duration': self.duration.seconds if self.duration else None, 'rollback': self.rollback, 'status': self.status, 'diff_url': self.diff_url, + 'release_id': self.release_id, } diff --git a/orlo/queries.py b/orlo/queries.py index df886f5..77bf923 100644 --- a/orlo/queries.py +++ b/orlo/queries.py @@ -1,7 +1,7 @@ from __future__ import print_function import datetime import arrow -from orlo import app +from orlo.app import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage from sqlalchemy import and_, exc @@ -246,13 +246,13 @@ def apply_package_filters(query, args): return query -def releases(limit=None, offset=None, desc=None, **kwargs): +def releases(limit=None, offset=None, asc=None, **kwargs): """ Return whole releases, based on filters :param limit: Max number of results to return :param offset: Offset results. Provides pagination when combined with limit. - :param desc: Sort descending instead of the default ascending order + :param asc: Sort ascending instead of the default descending order :param kwargs: Request arguments :return: """ @@ -277,10 +277,10 @@ def releases(limit=None, offset=None, desc=None, **kwargs): raise InvalidUsage( "An invalid field was specified: {}".format(e.args[0])) - if desc: - stime_field = Release.stime.desc - else: + if asc: stime_field = Release.stime.asc + else: + stime_field = Release.stime.desc query = query.order_by(stime_field()) diff --git a/orlo/routes/__init__.py b/orlo/routes/__init__.py new file mode 100644 index 0000000..d75cdfd --- /dev/null +++ b/orlo/routes/__init__.py @@ -0,0 +1,13 @@ +from __future__ import print_function + +import orlo.routes.base +import orlo.routes.import_ +import orlo.routes.info +import orlo.routes.internal +import orlo.routes.packages +import orlo.routes.releases +import orlo.routes.stats + + +__author__ = 'alforbes' + diff --git a/orlo/route_base.py b/orlo/routes/base.py similarity index 97% rename from orlo/route_base.py rename to orlo/routes/base.py index 22fa32a..ae3cfb4 100644 --- a/orlo/route_base.py +++ b/orlo/routes/base.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import jsonify, request -from orlo import app +from orlo.app import app from orlo import __version__ __author__ = 'alforbes' diff --git a/orlo/route_import.py b/orlo/routes/import_.py similarity index 99% rename from orlo/route_import.py rename to orlo/routes/import_.py index 360fee4..0ff0445 100644 --- a/orlo/route_import.py +++ b/orlo/routes/import_.py @@ -1,7 +1,7 @@ import arrow import json from flask import jsonify, request -from orlo import app +from orlo.app import app from orlo.orm import db, Package, Release, PackageResult, ReleaseNote, Platform from orlo.util import validate_request_json from orlo.user_auth import token_auth diff --git a/orlo/route_info.py b/orlo/routes/info.py similarity index 99% rename from orlo/route_info.py rename to orlo/routes/info.py index 8086e39..30470bd 100644 --- a/orlo/route_info.py +++ b/orlo/routes/info.py @@ -1,6 +1,6 @@ from __future__ import print_function from flask import request, jsonify, url_for -from orlo import app +from orlo.app import app import orlo.queries as queries __author__ = 'alforbes' diff --git a/orlo/routes/internal.py b/orlo/routes/internal.py new file mode 100644 index 0000000..504547f --- /dev/null +++ b/orlo/routes/internal.py @@ -0,0 +1,15 @@ +from __future__ import print_function +from flask import jsonify +from orlo.app import app +from orlo import __version__ + +__author__ = 'alforbes' + + +@app.route('/internal/version', methods=['GET']) +def internal_version(): + """ + Get the running version of Orlo + :return: + """ + return jsonify({'version': __version__}) diff --git a/orlo/route_packages.py b/orlo/routes/packages.py similarity index 96% rename from orlo/route_packages.py rename to orlo/routes/packages.py index 927c88d..8af718d 100644 --- a/orlo/route_packages.py +++ b/orlo/routes/packages.py @@ -1,6 +1,7 @@ from __future__ import print_function from flask import jsonify, request, Response, json, g -from orlo import app, queries, config +from orlo.app import app +from orlo import queries from orlo.exceptions import InvalidUsage from orlo.user_auth import token_auth from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, \ @@ -8,7 +9,6 @@ from orlo.util import validate_request_json, create_release, \ validate_release_input, validate_package_input, fetch_release, \ create_package, fetch_package, stream_json_list, str_to_bool, is_uuid -from orlo.deploy import ShellDeploy from orlo.user_auth import conditional_auth __author__ = 'alforbes' diff --git a/orlo/route_releases.py b/orlo/routes/releases.py similarity index 90% rename from orlo/route_releases.py rename to orlo/routes/releases.py index 879afa6..2d307b0 100644 --- a/orlo/route_releases.py +++ b/orlo/routes/releases.py @@ -1,5 +1,7 @@ from flask import jsonify, request, Response, json, g -from orlo import app, queries, config +from orlo.app import app +from orlo import queries +from orlo.config import config from orlo.exceptions import InvalidUsage from orlo.user_auth import token_auth from orlo.orm import db, Release, Package, PackageResult, ReleaseNote, \ @@ -7,7 +9,6 @@ from orlo.util import validate_request_json, create_release, \ validate_release_input, validate_package_input, fetch_release, \ create_package, fetch_package, stream_json_list, str_to_bool, is_uuid -from orlo.deploy import ShellDeploy from orlo.user_auth import conditional_auth security_enabled = config.getboolean('security', 'enabled') @@ -122,34 +123,6 @@ def post_results(release_id, package_id): return '', 204 -@app.route('/releases//deploy', methods=['POST']) -@conditional_auth(token_auth.token_required) -def post_releases_deploy(release_id): - """ - Deploy a Release - - :param string release_id: Release UUID - - **Example curl**: - - .. sourcecode:: shell - - curl -H "Content-Type: application/json" \\ - -X POST http://127.0.0.1/releases/${RELEASE_ID}/deploy - """ - release = fetch_release(release_id) - app.logger.info("Release start, release {}".format(release_id)) - - release.start() - db.session.add(release) - db.session.commit() - - # TODO call deploy Class start Method, i.e. pure python rather than shell - deploy = ShellDeploy(release) - output = deploy.start() - return jsonify(output), 200 - - @app.route('/releases//start', methods=['POST']) @conditional_auth(token_auth.token_required) def post_releases_start(release_id): @@ -312,10 +285,10 @@ def get_releases(release_id=None): :param string release_id: Optionally specify a single release UUID to fetch. This does not disable filters. - :query bool desc: Normally results are returned ordered by stime - ascending, setting desc to true will reverse this and sort by stime - descending - :query int limit: Limit the results by int + :query bool asc: Normally results are returned ordered by stime + descending, setting asc to true will reverse this and sort by stime + ascending + :query int limit: Limit the results by int (default 100) :query int offset: Offset the results by int :query string package_name: Filter releases by package name :query string user: Filter releases by user the that performed the release @@ -355,7 +328,6 @@ def get_releases(release_id=None): value. If any one package has the value "IN_PROGRESS" or "FAILED", that status applies to the whole release, with "FAILED" overriding "IN_PROGRESS". - """ booleans = ('rollback', 'package_rollback',) @@ -364,14 +336,13 @@ def get_releases(release_id=None): if not is_uuid(release_id): raise InvalidUsage("Release ID given is not a valid UUID") query = queries.get_release(release_id) - elif len([x for x in request.args.keys()]) == 0: - raise InvalidUsage("Please specify a filter. See " - "http://orlo.readthedocs.org/en/latest/rest.html" - "#get--releases for more info") else: # Bit more complex + # Defaults + args = { + 'limit': 100 + } # Flatten args, as the ImmutableDict puts some values in a list when # expanded - args = {} for k in request.args.keys(): if k in booleans: args[k] = str_to_bool(request.args.get(k)) diff --git a/orlo/route_stats.py b/orlo/routes/stats.py similarity index 99% rename from orlo/route_stats.py rename to orlo/routes/stats.py index 5153911..cf96e4b 100644 --- a/orlo/route_stats.py +++ b/orlo/routes/stats.py @@ -2,7 +2,7 @@ from flask import request, jsonify from orlo import stats -from orlo import app +from orlo.app import app from orlo.exceptions import InvalidUsage import orlo.queries as queries diff --git a/orlo/stats.py b/orlo/stats.py index 43c0450..eb1ff2b 100644 --- a/orlo/stats.py +++ b/orlo/stats.py @@ -1,6 +1,6 @@ from __future__ import print_function from orlo.queries import apply_filters, filter_release_rollback, filter_release_status -from orlo import app +from orlo.app import app from orlo.orm import db, Release, Platform, Package, release_platform from orlo.exceptions import OrloError, InvalidUsage from collections import OrderedDict diff --git a/orlo/user_auth.py b/orlo/user_auth.py index 7892a3b..74ff51a 100644 --- a/orlo/user_auth.py +++ b/orlo/user_auth.py @@ -1,4 +1,4 @@ -from orlo import app +from orlo.app import app from orlo.config import config from orlo.exceptions import OrloAuthError from flask_httpauth import HTTPBasicAuth diff --git a/orlo/util.py b/orlo/util.py index 4f5f9b7..a633616 100644 --- a/orlo/util.py +++ b/orlo/util.py @@ -1,6 +1,6 @@ from __future__ import print_function, unicode_literals from flask import json -from orlo import app +from orlo.app import app from orlo.orm import db, Release, Package, Platform from orlo.exceptions import InvalidUsage from sqlalchemy.orm import exc diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 591387a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -gunicorn -arrow>=0.7.0 -Flask>=0.10.1 -Flask-SQLAlchemy>=2.0 -Flask-Migrate>=1.5.1 -Flask-HTTPAuth>=2.7.1 -Flask-TokenAuth>=0.0.2 -requests>=2.4.2 -SQLAlchemy>=1.0.8 -sqlalchemy-utils>=0.31.0 -psycopg2>=2.6.1 -pyldap>=2.4.25 -pytz -six>=1.10.0 diff --git a/requirements_testing.txt b/requirements_testing.txt deleted file mode 100644 index 0382455..0000000 --- a/requirements_testing.txt +++ /dev/null @@ -1,20 +0,0 @@ -Flask-HTTPAuth>=2.7.1 -Flask-Migrate>=1.5.1 -Flask-SQLAlchemy>=2.0 -Flask-TokenAuth>=0.0.2 -Flask>=0.10.1 -SQLAlchemy>=1.0.8 -arrow>=0.7.0 -gunicorn -orloclient>=0.1.1 -passlib>=1.6.1 -psycopg2>=2.6.1 -pytest -pytz -requests>=2.4.2 -sqlalchemy-utils>=0.31.0 -mock==1.3.0 -mockldap>=0.2.6 -pyldap>=2.4.25 -six>=1.10.0 -git+https://github.com/al4/flask-testing.git diff --git a/runserver.py b/runserver.py deleted file mode 100755 index 69cb434..0000000 --- a/runserver.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -from orlo import app - -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///orlo.db' - -app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index d9a9a1c..ec6ed65 100755 --- a/setup.py +++ b/setup.py @@ -1,19 +1,27 @@ #!/usr/bin/env python # from distutils.core import setup -from setuptools import setup +from setuptools import setup, find_packages import multiprocessing # nopep8 import os __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -VERSION = '0.3.1' +VERSION = '0.4.0' version_file = open(os.path.join(__location__, 'orlo', '_version.py'), 'w') version_file.write("__version__ = '{}'".format(VERSION)) version_file.close() +tests_require = [ + 'Flask-Testing', + 'orloclient>=0.2.0', + 'mockldap', + 'pytest', + 'tox', +] + setup( name='orlo', @@ -24,31 +32,34 @@ license='GPL', long_description=open(os.path.join(__location__, 'README.md')).read(), url='https://github.com/eBayClassifiedsGroup/orlo', - packages=[ - 'orlo', - ], + packages=find_packages( + exclude=['tests', 'debian', 'docs', 'etc'] + ), include_package_data=True, install_requires=[ 'Flask', + 'Flask-Alembic >= 2.0.1', + 'Flask-HTTPAuth', 'Flask-Migrate', 'Flask-SQLAlchemy', - 'Flask-HTTPAuth', + 'Flask-Script >= 2.0.5', 'Flask-TokenAuth', 'arrow', 'gunicorn', + 'orloclient>=0.2.0', 'psycopg2', 'pyldap', 'pytz', 'sphinxcontrib-httpdomain', 'sqlalchemy-utils', ], - tests_require=[ - 'Flask-Testing', - 'orloclient>=0.1.1', - ], + extras_require={ + 'test': tests_require, + }, + tests_require=tests_require, test_suite='tests', # Creates a script in /usr/local/bin entry_points={ - 'console_scripts': ['orlo=orlo.cli:main'] + 'console_scripts': ['orlo=orlo.__main__:main'] } ) diff --git a/tests/test_deploy.py b/tests/test_deploy.py deleted file mode 100644 index e13d854..0000000 --- a/tests/test_deploy.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import print_function -from test_base import OrloLiveTest, OrloTest, ConfigChange, ReleaseDbUtil -from orlo.deploy import BaseDeploy, HttpDeploy, ShellDeploy -from orlo.orm import db, Release, Package -from test_base import OrloLiveTest -from test_base import ReleaseDbUtil -import unittest - -__author__ = 'alforbes' - - -class DeployTest(OrloLiveTest, ReleaseDbUtil): - """ - Test the Deploy class - """ - CLASS = BaseDeploy - - def setUp(self): - super(DeployTest, self).setUp() - rid = self._create_release() - pid = self._create_package(rid) - self.release = db.session.query(Release).first() - - def test_init(self): - """ - Test that we can instantiate the class - """ - o = self.CLASS(self.release) - self.assertIsInstance(o, BaseDeploy) - - -class TestBaseDeploy(DeployTest): - def test_not_implemented(self): - """ - Base Deploy class should raise NotImplementedError on start - """ - o = self.CLASS(self.release) - with self.assertRaises(NotImplementedError): - o.start() - - -class TestHttpDeploy(DeployTest): - CLASS = HttpDeploy - - @unittest.skip("Not implemented") - def test_start(self): - """ - Test that start emits an http call - """ - pass - - @unittest.skip("Not implemented") - def test_kill(self): - """ - Test that kill emits an http call - """ - pass - - -class TestShellDeploy(DeployTest): - CLASS = ShellDeploy - - def test_start(self): - """ - Test that start emits a shell command - """ - with ConfigChange('deploy', 'timeout', '3'), \ - ConfigChange('deploy_shell', 'command_path', '/bin/true'): - deploy = ShellDeploy(self.release) - - # Override server_url, normally it is set by config: - deploy.server_url = self.get_server_url() - - deploy.start() - - @unittest.skip("Not implemented") - def test_kill(self): - """ - Test that kill emits a shell command - """ - pass - - @unittest.skip("Doesn't work on travis") - def test_start_example_deployer(self): - """ - Test the example deployer completes - - DOESN'T WORK ON TRAVIS, as /bin/env python gives the system python - """ - with ConfigChange('deploy', 'timeout', '3'): - deploy = ShellDeploy(self.release) - - # Override server_url, normally it is set by config: - deploy.server_url = self.get_server_url() - - deploy.start() - - def test_output(self): - """ - Test that we return the output of the deploy - - Not a good test, as it relies on the test-package being an argument, - and simply echoing it back. This is the "spec", but this test could - break if the arguments change. - """ - with ConfigChange('deploy', 'timeout', '3'), \ - ConfigChange('deploy_shell', 'command_path', '/bin/echo'): - deploy = ShellDeploy(self.release) - - # Override server_url, normally it is set by config: - deploy.server_url = self.get_server_url() - - output = deploy.start() - self.assertEqual(output['stdout'], b'test-package=1.2.3\n') - diff --git a/tests/test_deployer.py b/tests/test_deployer.py deleted file mode 100644 index 89ca0b4..0000000 --- a/tests/test_deployer.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import print_function -import unittest - -""" -Test the example deployer -""" - - -class TestDeployer(unittest.TestCase): - ENV = {} - ARGS = {} - METADATA = {} - - def test_exit_0(self): - """ - Test the deployer exits 0 - """ \ No newline at end of file diff --git a/tests/test_orm.py b/tests/test_orm.py index b5e3912..66b8bb8 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -3,7 +3,7 @@ from random import randrange from orlo.orm import db from orlo.orm import Release, Package, PackageResult, Platform -from orlo import app +from orlo.app import app from sqlalchemy.orm import exc import arrow import datetime diff --git a/tests/test_route_internal.py b/tests/test_route_internal.py new file mode 100644 index 0000000..c2c67ca --- /dev/null +++ b/tests/test_route_internal.py @@ -0,0 +1,14 @@ +from __future__ import print_function +from test_route_base import OrloHttpTest + +__author__ = 'alforbes' + + +class TestInternalRoute(OrloHttpTest): + def test_version(self): + """ + Test /version returns 200 + """ + response = self.client.get('/internal/version') + self.assert200(response) + self.assertIn('version', response.json) diff --git a/tests/test_route_releases.py b/tests/test_route_releases.py index d6d0c8b..6f82675 100644 --- a/tests/test_route_releases.py +++ b/tests/test_route_releases.py @@ -478,16 +478,18 @@ def test_get_release_offset(self): """ Test that offset=1 skips the first release """ - rid = None + rids = [] for _ in range(0, 2): - rid = self._create_release() + rids.append(self._create_release()) sleep(0.1) r = self._get_releases(filters=['offset=1']) self.assertEqual(len(r['releases']), 1) - self.assertEqual(r['releases'][0]['id'], rid) + # Default order is now descending, so the skip means we're skipping the + # last release, so the release given should be the first created. + self.assertEqual(r['releases'][0]['id'], rids[0]) - def test_get_release_desc(self): + def test_get_release_asc(self): """ Should return in reverse order """ @@ -497,9 +499,9 @@ def test_get_release_desc(self): rid = self._create_release() sleep(0.1) - r = self._get_releases(filters=['desc=true']) - # First in list should be last to be created - self.assertEqual(r['releases'][0]['id'], rid) + r = self._get_releases(filters=['asc=true']) + # Last in list should be last to be created + self.assertEqual(r['releases'][2]['id'], rid) def test_get_release_package_name(self): """ @@ -680,11 +682,3 @@ def test_get_release_with_bad_status(self): result = self._get_releases(filters=['status=garbage_boz'], expected_status=400) self.assertIn('message', result) - - def test_get_releases_with_no_filters(self): - """ - Test get /releases without filters returns 400 - """ - self._get_releases(expected_status=400) - - diff --git a/tox.ini b/tox.ini index beadf0b..e5c2cda 100644 --- a/tox.ini +++ b/tox.ini @@ -9,8 +9,5 @@ envlist = py27, py34 [testenv] commands = py.test {posargs} passenv = TRAVIS -deps = - pytest - Flask-Testing - orloclient - mockldap +deps = .[test] +setenv = ORLO_LOGDIR = disabled