Merge pull request #33 from al4/master

Bugfix and pre-0.2 release

Vagrantfile

@@ -71,34 +71,41 @@ Vagrant.configure(2) do |config|
   # SHELL
   config.vm.provision "shell", inline: <<-SHELL
     # sudo sed -i 's/archive.ubuntu.com/nl.archive.ubuntu.com/g' /etc/apt/sources.list
-    sudo apt-get update
-    sudo apt-get -y install python-pip python-dev postgresql postgresql-server-dev-all
+    apt-get update
+    apt-get -y install python-pip python-dev postgresql postgresql-server-dev-all
     echo "CREATE USER orlo WITH PASSWORD 'password'; CREATE DATABASE orlo OWNER orlo; " \
         | sudo -u postgres -i psql
 
     # python-ldap dependencies
-    sudo apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev
+    apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev
 
     # Build tools
-    sudo apt-get -y install build-essential git-buildpackage debhelper python-dev dh-systemd python-virtualenv
+    apt-get -y install build-essential git-buildpackage debhelper python-dev dh-systemd python-virtualenv
     wget -P /tmp/ \
         'https://launchpad.net/ubuntu/+archive/primary/+files/dh-virtualenv_0.11-1_all.deb'
     dpkg -i /tmp/dh-virtualenv_0.11-1_all.deb
+    apt-get -f install -y
 
-    sudo pip install --upgrade pip
-    sudo pip install sphinx sphinxcontrib-httpdomain
-    sudo pip install git+https://github.com/jarus/flask-testing.git
+    pip install --upgrade pip
+    pip install virtualenv
 
-    cd /vagrant/
-    # why do we need to do this twice?
-    sudo python /vagrant/setup.py install
-    sudo python /vagrant/setup.py install
+    # Virtualenv is to avoid conflict with Debian's python-six
+    virtualenv /home/vagrant/virtualenv/orlo
+    source /home/vagrant/virtualenv/orlo/bin/activate
+    echo "source ~/virtualenv/orlo/bin/activate" >> /home/vagrant/.profile
 
-    python /vagrant/create_db.py
-    sudo pip install -r /vagrant/requirements.txt
-    sudo pip install pytest Flask-Testing
+    pip install -r /vagrant/requirements.txt
+    pip install -r /vagrant/requirements_testing.txt
+    pip install -r /vagrant/docs/requirements.txt
+
+    # Flask-Testing hasn't been released to pip in ages :(
+    pip install --upgrade git+https://github.com/jarus/flask-testing.git
 
-    # For deployer.rb
-    sudo apt-get install ruby-rest-client ruby-json
+    sudo chown -R vagrant:vagrant /home/vagrant/virtualenv
+
+    # Create the database
+    python /vagrant/create_db.py
+    mkdir /etc/orlo
+    chown vagrant:root /etc/orlo
   SHELL
 end

bin/orlo

@@ -1,5 +1,9 @@
 #!/bin/bash
 
-/usr/share/python/orlo/bin/python /usr/share/python/orlo/lib/python2.7/site-packages/orlo/cli.py $@
+# This script is for DEBIAN packaging only.
+# The bin/orlo script referenced below is generated by setuptools (see entry_points in setup.py).
 
+# For the actual logic that is run by the script below, see orlo/cli.py
+
+/usr/share/python/orlo/bin/orlo $@
 

debian/preinst

@@ -9,5 +9,8 @@ else
 fi
 
 mkdir -p /var/{lib,log}/orlo
-chown orlo:orlo /var/{lib,log}/orlo
+touch /var/log/orlo/app.log
+
+chown -R orlo:orlo /var/{lib,log}/orlo
 chmod 755 /var/{lib,log}/orlo
+chmod 664 /var/log/orlo/app.log

deployer.py

@@ -23,19 +23,16 @@ class DeployerError(Exception):
     def __init__(self, message):
         self.message = message
         error(message)
-        raise SystemExit(1)
 
 
 def get_params():
     parser = argparse.ArgumentParser(description="Orlo Test Deployer")
     parser.add_argument('packages', choices=None, const=None, nargs='*',
-                        help="Packages to deploy, in format package_name=version, "
-                             "e.g test-package=1.0.0")
+                        help="Packages to deploy, in format "
+                             "package_name=version, e.g test-package=1.0.0")
     args, unknown = parser.parse_known_args()
 
     _packages = OrderedDict()
-    if not args.packages:
-        raise DeployerError("No packages to deploy, see help")
 
     for pkg in args.packages:
         package, version = pkg.split('=')
@@ -62,13 +59,14 @@ def deploy(package, meta=None):
     :param dict meta: Dictionary of metadata (unused in this dummy function)
     :return:
     """
+    info("Package start - {}:{}".format(package.name, package.version))
     orlo_client.package_start(package)
 
-    # Do the deploy...
-
+    # Do stuff
     # Determining success status is up to you
     success = True
 
+    info("Package stop - {}:{}".format(package.name, package.version))
     orlo_client.package_stop(package, success=success)
 
     return success
@@ -79,33 +77,45 @@ def deploy(package, meta=None):
         orlo_url = os.environ['ORLO_URL']
     else:
         # This deployer is only every supposed to accept releases from Orlo
-        # Other deployers could use this to detect whether they are being invoked by Orlo
+        # Other deployers could use this to detect whether they are being
+        # invoked by Orlo
         raise DeployerError("Could not detect ORLO_URL from environment")
 
     if os.getenv('ORLO_RELEASE'):
-        orlo_release = os.environ['ORLO_RELEASE']
+        orlo_release_id = os.environ['ORLO_RELEASE']
     else:
         raise DeployerError("Could not detect ORLO_RELEASE in environment")
 
+    logging.info(
+        "Environment: \n" +
+        json.dumps(os.environ, indent=2, default=lambda o: o.__dict__))
+
+    # Fetch packages and metadata. Packages is not used, it is just to
+    # demonstrate they are passed as arguments
     packages, metadata = get_params()
 
+    logging.info("Stdin: \n" + str(metadata))
+
+    # Create an instance of the Orlo client
     orlo_client = OrloClient(uri=orlo_url)
-    # The release is created in Orlo before being handed to the deployer
-    # So fetch it here
-    release = orlo_client.get_release(orlo_release)
 
-    # TODO - using package info from arguments makes no sense when we could fetch from Orlo
-    orlo_packages = []
-    for p, v in packages.items():
-        info("Creating Package {}:{}".format(p, v))
-        pkg = orlo_client.create_package(release, p, v)
-        orlo_packages.append(pkg)
+    # The release is created in Orlo before the deployer is invoked, so fetch
+    # it here. If you prefer, you can to do the release creation within your
+    # deployer and use Orlo only for receiving data
+    release = orlo_client.get_release(orlo_release_id)
+
+    # While we fetch Packages using the Orlo client, they are passed on the
+    # CLI as well, which is useful for non-python deployers
+    info("Fetching packages from Orlo")
+    if not release.packages:
+        raise DeployerError("No packages to deploy")
 
     info("Starting Release")
-    for pkg in orlo_packages:
+    for pkg in release.packages:
         info("Deploying {}".format(pkg.name))
         deploy(pkg, meta=metadata)
-    info("Finishing Release")
 
+    info("Finishing Release")
     orlo_client.release_stop(release)
+
     info("Done.")

orlo/cli.py

@@ -71,6 +71,11 @@ def run_server(args):
     app.run(host=args.host, port=args.port, debug=True, use_reloader=True)
 
 
-if __name__ == '__main__':
+def main():
+    # Referenced by entry_points in setup.py
     args = parse_args()
     args.func(args)
+
+
+if __name__ == '__main__':
+    main()

orlo/config.py

@@ -13,7 +13,6 @@
 config.set('main', 'strict_slashes', 'false')
 config.set('main', 'base_url', 'http://localhost:8080')
 
-
 config.add_section('security')
 config.set('security', 'enabled', 'false')
 config.set('security', 'passwd_file', os.path.dirname(__file__) + '/../etc/passwd')
@@ -33,12 +32,11 @@
 config.set('logging', 'debug', 'false')
 config.set('logging', 'file', 'disabled')
 
-config.read('/etc/orlo/orlo.ini')
-
 config.add_section('deploy')
 config.set('deploy', 'timeout', '3600')  # How long to timeout external deployer calls
 
 config.add_section('deploy_shell')
 config.set('deploy_shell', 'command_path', os.path.dirname(os.path.abspath(__file__)) +
            '/../deployer.py')
 
+config.read('/etc/orlo/orlo.ini')

orlo/deploy.py

@@ -2,49 +2,14 @@
 import json
 import subprocess
 from threading import Timer
+from orlo import app
 from orlo.config import config
 from orlo.exceptions import OrloError
 
 
 __author__ = 'alforbes'
 
 
-# Stolen from http://stackoverflow.com/questions/1191374
-def run(args, env, in_data, timeout_sec=3600):
-    """
-    Run a command in a separate thread
-
-    :param env: Dict of environment variables
-    :param in_data: String to pass to stdin
-    :param args: List of arguments
-    :param timeout_sec: Timeout in seconds, 1 hour by default
-    :return:
-    """
-    proc = subprocess.Popen(
-        args,
-        env=env,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-
-    timer = Timer(timeout_sec, proc.kill)
-    out = err = " "
-    try:
-        timer.start()
-        out, err = proc.communicate(in_data)
-    finally:
-        timer.cancel()
-        print("Out:\n{}".format(out))
-        print("Err:\n{}".format(err))
-
-    if proc.returncode is not 0:
-        raise OrloError("Subprocess exited with code {}".format(
-            proc.returncode
-        ))
-    print("end run")
-
-
 class BaseDeploy(object):
     """
     A Deploy task
@@ -90,16 +55,20 @@ def start(self):
         pass
 
     def kill(self):
-        pass
+        raise NotImplementedError("No kill method for HTTP deploys")
 
 
 class ShellDeploy(BaseDeploy):
     """
     Deployment by shell command
 
-    meta {} => stdin
-    deployer pkg1=1
-    capture stdout,
+    Data is passed to the shell command given in 3 ways:
+
+    * ORLO_URL, ORLO_RELEASE (the ID), and other Release attributes
+      are set as environment variables (all prefixed by ORLO_)
+    * The package and version sets are passed as arguments, e.g.
+      package-name=1.0.0
+    * The metadata dictionary is passed to stdin
     """
 
     def __init__(self, release):
@@ -130,10 +99,48 @@ def start(self):
             metadata.update(m.to_dict())
         in_data = json.dumps(metadata)
 
-        run(args, env, in_data, timeout_sec=config.getint('deploy', 'timeout'))
+        self.run_command(args, env, in_data,
+                         timeout_sec=config.getint('deploy', 'timeout'))
 
     def kill(self):
         """
         Kill a deploy in progress
         """
         raise NotImplementedError
+
+    @staticmethod
+    def run_command(args, env, in_data, timeout_sec=3600):
+        """
+        Run a command in a separate thread
+
+        Adapted from http://stackoverflow.com/questions/1191374
+
+        :param env: Dict of environment variables
+        :param in_data: String to pass to stdin
+        :param args: List of arguments
+        :param timeout_sec: Timeout in seconds, 1 hour by default
+        :return:
+        """
+        proc = subprocess.Popen(
+            args,
+            env=env,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        timer = Timer(timeout_sec, proc.kill)
+        out = err = " "
+        try:
+            timer.start()
+            out, err = proc.communicate(in_data)
+        finally:
+            timer.cancel()
+            print("Out:\n{}".format(out))
+            print("Err:\n{}".format(err))
+
+        if proc.returncode is not 0:
+            raise OrloError("Subprocess exited with code {}".format(
+                proc.returncode), status_code=500)
+        print("end run")
+

setup.py

@@ -3,10 +3,12 @@
 # from distutils.core import setup
 from setuptools import setup
 import multiprocessing  # nopep8
+import os
 
 
-VERSION = '0.1.1-1'
-version_file = open('./orlo/_version.py', 'w')
+VERSION = '0.2.0-pre1'
+my_path = os.path.dirname(os.path.realpath(__file__))
+version_file = open('{}/orlo/_version.py'.format(my_path), 'w')
 version_file.write("__version__ = '{}'".format(VERSION))
 version_file.close()
 
@@ -42,4 +44,8 @@
         'orloclient>=0.1.1',
     ],
     test_suite='tests',
+    # Creates a script in /usr/local/bin
+    entry_points={
+        'console_scripts': ['orlo=orlo.cli:main']
+    }
 )