Merge pull request #79 from deniszh/DZ-Python3

Python 3 / Ubuntu 18.04 support

.gitignore

@@ -1 +1,3 @@
 .vagrant
+.idea
+*.log

README.md

@@ -5,7 +5,7 @@ Synthesize
 
 Installing Graphite doesn't have to be difficult. The `install` script in synthesize is designed to make it brain-dead easy to install Graphite and related services onto a modern Linux distribution.
 
-Synthesize is built to run on Ubuntu 14.04 LTS. It will __not__ run on other Ubuntu releases or Linux distributions. The goal of this project is not to become an automation alternative to modern configuration management utilities (e.g. Chef or Puppet), but rather, to make it as easy as possible for the beginner Graphite user to get started and familiar with the project without having to learn a suite of other automation and/or infrastructure-related projects.
+Synthesize is built to run on Ubuntu 18.04 LTS. It will __not__ run on other Ubuntu releases or Linux distributions. The goal of this project is not to become an automation alternative to modern configuration management utilities (e.g. Chef or Puppet), but rather, to make it as easy as possible for the beginner Graphite user to get started and familiar with the project without having to learn a suite of other automation and/or infrastructure-related projects.
 
 The resulting Graphite web interface __listens only on https port 443__ and has been configured to collect metrics specifically for helping profile the performance of your Graphite and Carbon services. It uses memcached for improved query performance, and Statsite for a fast, C-based implementation of the StatsD collector/aggregator.
 
@@ -15,14 +15,14 @@ Beginning with version 3.0.0 we've also incorporated the Grafana dashboard proje
 
 ## Provides
 
-* Graphite 1.0.0 ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper))
+* Graphite 1.1.x ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper))
 * StatsD ([statsite](https://github.com/armon/statsite))
 * [Collectd](http://collectd.org/)
 * [Grafana](https://grafana.org/)
 
 ## Dependencies
 
-* Vagrant, an Ubuntu 14.04 VM or a non-production server
+* Vagrant, an Ubuntu 18.04 VM or a non-production server
 * Some mechanism for downloading Synthesize
 
 ## Installation

Vagrantfile

@@ -5,13 +5,13 @@
 VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "mayflower/trusty64-puppet3"
+  config.vm.box = "ubuntu/bionic64"
   config.vm.network :forwarded_port, guest: 443, host: 8443
   config.vm.network :forwarded_port, guest: 8125, host: 8125, protocol: 'tcp'
   config.vm.network :forwarded_port, guest: 8125, host: 8125, protocol: 'udp'
   config.vm.network :forwarded_port, guest: 2003, host: 22003
   config.vm.network :forwarded_port, guest: 2004, host: 22004
   config.vm.network :forwarded_port, guest: 3000, host: 3030
-  graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? '1.0.2' : ENV['GRAPHITE_RELEASE']
+  graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? '1.1.7' : ENV['GRAPHITE_RELEASE']
   config.vm.provision "shell", inline: "cd /vagrant; GRAPHITE_RELEASE=#{graphite_version} ./install"
 end

app.yml

@@ -1,5 +1,5 @@
 name: synthesize
-image: ubuntu-14-04-x64
+image: ubuntu-18-04-x64
 min_size: 512mb
 config:
   #cloud-config
@@ -11,4 +11,4 @@ config:
   packages:
     - git
   runcmd:
-    - cd /home/install && apt-get -y remove python-openssl && git clone https://github.com/obfuscurity/synthesize.git && cd synthesize && bash install
+    - cd /home/install && git clone https://github.com/obfuscurity/synthesize.git && cd synthesize && bash install

install

@@ -1,5 +1,5 @@
 #!/bin/bash -ex
-# Graphite installation script for Ubuntu 14.04
+# Graphite installation script for Ubuntu 18.04
 # Jason Dixon <[email protected]>
 
 SYNTHESIZE_HOME=$( cd "$( dirname "$0" )" && pwd )
@@ -9,45 +9,16 @@ GRAPHITE_HOME='/opt/graphite'
 GRAPHITE_CONF="${GRAPHITE_HOME}/conf"
 GRAPHITE_STORAGE="${GRAPHITE_HOME}/storage"
 
-# Check to see if python-openssl is installed first
-set +e
-PYTHON_OPENSSL_INSTALLED=`dpkg -s python-openssl 2>/dev/null | grep '^Status'`
-set -e
-
-# This can break Carbon's python-twisted, exit with explanation
-if [[ ! $PYTHON_OPENSSL_INSTALLED == '' ]]; then
-  set +x
-  clear
-  echo
-  echo
-  echo
-  echo "##########################################################"
-  echo
-  echo "It appears that you have python-openssl installed."
-  echo "There are known issues with this package that can cause"
-  echo "the Carbon service to fail to start. If you don't need"
-  echo "the python-openssl library, it's recommended that you"
-  echo "uninstall this package before continuing."
-  echo
-  echo "  $ sudo apt-get remove python-openssl"
-  echo
-  echo "More information is provided at the link below."
-  echo
-  echo "https://github.com/graphite-project/graphite-web/issues/1721#issuecomment-253279181"
-  echo
-  echo "##########################################################"
-  echo
-  echo
-  echo
-  exit 1
+if [ -z $GRAPHITE_RELEASE ]; then
+    GRAPHITE_RELEASE='1.1.7'
 fi
 
-if [ -z $GRAPHITE_RELEASE ]; then
-    GRAPHITE_RELEASE='1.0.2'
+if [ -z $STATSITE_RELEASE ]; then
+    STATSITE_RELEASE='master'
 fi
 
-if [[ ! $UBUNTU_RELEASE =~ 'Ubuntu 14.04' ]]; then
-  echo "Sorry, this is only supported for Ubuntu Linux 14.04."
+if [[ ! $UBUNTU_RELEASE =~ 'Ubuntu 18.04' ]]; then
+  echo "Sorry, this is only supported for Ubuntu Linux 18.04."
   exit 1
 fi
 if [[ -d $GRAPHITE_HOME ]]; then
@@ -59,7 +30,9 @@ fi
 apt-get update -y
 
 # Install package dependencies from apt
-RUNLEVEL=1 apt-get install -y libcairo2-dev libffi-dev pkg-config python-dev python-pip fontconfig apache2 libapache2-mod-wsgi git-core collectd memcached gcc g++ make libtool automake
+RUNLEVEL=1 apt-get install -y git wget curl netbase collectd memcached \
+           build-essential automake libtool libffi-dev libcairo2-dev libcairo2 libffi6 \
+           python3-dev python3-pip fontconfig apache2 libapache2-mod-wsgi-py3
 
 # Download source repositories for Graphite/Carbon/Whisper and Statsite
 cd /usr/local/src
@@ -69,13 +42,34 @@ git clone https://github.com/graphite-project/whisper.git
 git clone https://github.com/armon/statsite.git
 
 # Build and install Graphite/Carbon/Whisper and Statsite
-cd whisper; git checkout ${GRAPHITE_RELEASE}; python setup.py install
-cd ../carbon; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python setup.py install
-cd ../graphite-web; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python check-dependencies.py; python setup.py install
-cd ../statsite; ./autogen.sh; ./configure; make; cp statsite /usr/local/sbin/; cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py
+mkdir -p /opt/graphite
+
+cd whisper; \
+   git checkout ${GRAPHITE_RELEASE}; \
+   python3 setup.py install
+
+cd ../carbon; \
+   git checkout ${GRAPHITE_RELEASE}; \
+   pip3 install -r requirements.txt; \
+   python3 setup.py install
+
+cd ../graphite-web; \
+   pip3 install django==2.2.9; \
+   git checkout ${GRAPHITE_RELEASE}; \
+   pip3 install -r requirements.txt; \
+   python3 check-dependencies.py; \
+   python3 setup.py install
+
+cd ../statsite; \
+   git checkout ${STATSITE_RELEASE}; \
+   ./autogen.sh; \
+   ./configure; make; \
+   cp statsite /usr/local/sbin/; \
+   cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py
 
 # Update txamqp to support RabbitMQ 2.4+
-pip install txamqp==0.6.2 --upgrade
+# also install service_identity to remove TLS error
+pip3 install txamqp service_identity --upgrade
 
 # Install configuration files for Graphite/Carbon and Apache
 cp ${SYNTHESIZE_HOME}/templates/statsite/statsite.conf /etc/statsite.conf
@@ -84,8 +78,11 @@ mv ${GRAPHITE_CONF}/*.example ${GRAPHITE_CONF}/examples/
 cp ${SYNTHESIZE_HOME}/templates/graphite/conf/* ${GRAPHITE_CONF}/
 cp ${SYNTHESIZE_HOME}/templates/collectd/collectd.conf /etc/collectd/
 cp ${SYNTHESIZE_HOME}/templates/apache/graphite.conf /etc/apache2/sites-available/
-cp ${SYNTHESIZE_HOME}/templates/init/* /etc/init/
-cp ${SYNTHESIZE_HOME}/templates/init.d/* /etc/init.d/
+
+# using systemd units instead of init and upstart files
+cp ${SYNTHESIZE_HOME}/templates/systemd/* /etc/systemd/system/
+# installing scripts for carbon-cache instances control script
+cp ${SYNTHESIZE_HOME}/templates/scripts/* /usr/local/bin/
 
 # Setup the correct Apache site and modules
 a2dissite 000-default
@@ -101,7 +98,7 @@ sed -i -e "s/UNSAFE_DEFAULT/`date | md5sum | cut -d ' ' -f 1`/" local_settings.p
 
 # Setup the Django database
 PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py migrate --noinput --settings=graphite.settings --run-syncdb
-PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py loaddata --settings=graphite.settings initial_data.json
+PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py collectstatic --noinput --settings=graphite.settings
 
 # Add carbon system user and set permissions
 groupadd -g 998 carbon
@@ -110,35 +107,40 @@ chmod 775 ${GRAPHITE_STORAGE}
 chown www-data:carbon ${GRAPHITE_STORAGE}
 chown www-data:www-data ${GRAPHITE_STORAGE}/graphite.db
 chown -R carbon ${GRAPHITE_STORAGE}/whisper
+mkdir -p ${GRAPHITE_STORAGE}/log/carbon-cache/
+chown -R carbon ${GRAPHITE_STORAGE}/log
 mkdir ${GRAPHITE_STORAGE}/log/apache2
 chown -R www-data ${GRAPHITE_STORAGE}/log/webapp
-chmod +x /etc/init.d/carbon-cache
 
-# Setup hourly cron to rebuild Graphite index
-cp ${SYNTHESIZE_HOME}/templates/graphite/cron/build-index /etc/cron.hourly/graphite-build-index
-chmod 755 /etc/cron.hourly/graphite-build-index
-sudo -u www-data /opt/graphite/bin/build-index.sh
+# Setup hourly systemd timer to rebuild Graphite index
+cp ${SYNTHESIZE_HOME}/templates/scripts/graphite-build-index /usr/local/bin/graphite-build-index
+systemctl daemon-reload
+systemctl enable graphite-build-index.timer
+systemctl start graphite-build-index.timer
+systemctl start graphite-build-index
 
 # Install Grafana
-echo 'deb https://packagecloud.io/grafana/stable/debian/ wheezy main' > /etc/apt/sources.list.d/grafana.list
-curl https://packagecloud.io/gpg.key | apt-key add -
-apt-get update -y
-DEBIAN_FRONTEND=noninteractive apt-get install -y grafana
-service grafana-server start
-sleep 5
-curl -X POST -H 'Content-Type: application/json' -u 'admin:admin' \
-  -d '{ "name": "graphite", "type": "graphite", "url": "https://127.0.0.1:443", "access": "proxy", "basicAuth": false }' \
-  "http://127.0.0.1:3000/api/datasources"
-curl -X POST -H 'Content-Type: application/json' -u 'admin:admin' \
-  -d '{ "inputs": [{"name": "*", "pluginId": "graphite", "type": "datasource", "value": "graphite"}],  "overwrite": true, "path": "dashboards/carbon_metrics.json", "pluginId": "graphite" }' \
-  "http://127.0.0.1:3000/api/dashboards/import"
+apt-get install -y apt-transport-https
+apt-get install -y software-properties-common wget
+wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
+add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
+apt-get install grafana
+
+# Bootstrap Grafana datasources and dashboards
+mkdir -p /etc/grafana/provisioning/datasources/
+mkdir -p /etc/grafana/provisioning/dashboards/
+cp ${SYNTHESIZE_HOME}/templates/grafana/provisioning/datasources/* /etc/grafana/provisioning/datasources/
+cp ${SYNTHESIZE_HOME}/templates/grafana/provisioning/dashboards/* /etc/grafana/provisioning/dashboards/
+systemctl restart grafana-server
+
+# Enabling processes start
+/usr/local/bin/carbon-cache enable
+systemctl enable statsite
+systemctl enable grafana-server
 
 # Start our processes
-update-rc.d carbon-cache defaults
-service carbon-cache start
-service memcached start
-service collectd start
-service apache2 start
-service statsite start
+/usr/local/bin/carbon-cache start
+service memcached restart
+service collectd restart
 service apache2 restart
-
+service statsite restart

templates/apache/graphite.conf

@@ -6,20 +6,20 @@
   CustomLog /opt/graphite/storage/log/apache2/access.log combined
   ErrorLog /opt/graphite/storage/log/apache2/error.log
 
-  WSGIDaemonProcess graphiteweb python-path=/opt/graphite:/opt/graphite/lib/python2.7/site-packages processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120
+  WSGIDaemonProcess graphiteweb python-path=/opt/graphite:/usr/local/lib/python3.6/dist-packages processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120
   WSGIProcessGroup graphiteweb
   WSGIApplicationGroup %{GLOBAL}
   WSGIImportScript /opt/graphite/conf/graphite.wsgi process-group=graphiteweb application-group=%{GLOBAL}
   WSGIScriptAlias / /opt/graphite/conf/graphite.wsgi
 
   RedirectMatch ^/admin(.*)admin/([^/]+)/([^/]+)$ /media/$2/$3
-  Alias /media/ "/usr/local/lib/python2.7/dist-packages/django/contrib/admin/static/admin/"
+  Alias /media/ "/opt/graphite/static/admin/"
 
   <Directory /opt/graphite/conf>
     Require all granted
   </Directory>
 
-  <Directory /usr/local/lib/python2.7/dist-packages/django/contrib/admin/static/admin>
+  <Directory /opt/graphite/static/admin>
     Require all granted
   </Directory>
 

templates/collectd/collectd.conf

@@ -976,13 +976,13 @@ LoadPlugin write_graphite
 
 		# data retrieval time
 		<Match>
-			Regex "Retrieval of [^ ]+ took ([0-9\.]+)"
+			Regex "Total request processing time ([0-9\.]+)"
 			DSType "CounterInc"
 			Type "counter"
 			Instance "retrieval_count"
 		</Match>
 		<Match>
-			Regex "Retrieval of [^ ]+ took ([0-9\.]+)"
+			Regex "Total request processing time ([0-9\.]+)"
 			DSType "GaugeMin"
 			Type "response_time"
 			Instance "retrieval_time_min"