From 28e5b9ad86be8d70a9a51fbad7ce91bf1f515723 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Mon, 11 Jul 2016 17:34:30 -0400 Subject: [PATCH 01/19] begin tracking Graphite master, woop woop --- Vagrantfile | 2 +- install | 16 +- templates/graphite/conf/graphite.wsgi | 25 +- templates/graphite/webapp/initial_data.json | 660 +++++++++++++++++++- templates/graphite/webapp/local_settings.py | 101 ++- 5 files changed, 756 insertions(+), 48 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index e837a7e..9a9758c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -11,6 +11,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8125, host: 8125, protocol: 'udp' config.vm.network :forwarded_port, guest: 2003, host: 22003 config.vm.network :forwarded_port, guest: 2004, host: 22004 - graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? '0.9.15' : ENV['GRAPHITE_RELEASE'] + graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? 'master' : ENV['GRAPHITE_RELEASE'] config.vm.provision "shell", inline: "cd /vagrant; GRAPHITE_RELEASE=#{graphite_version} ./install" end diff --git a/install b/install index 9e19766..03c6986 100755 --- a/install +++ b/install @@ -10,7 +10,7 @@ GRAPHITE_CONF="${GRAPHITE_HOME}/conf" GRAPHITE_STORAGE="${GRAPHITE_HOME}/storage" if [ -z $GRAPHITE_RELEASE ]; then - GRAPHITE_RELEASE='0.9.15' + GRAPHITE_RELEASE='master' fi if [[ ! $UBUNTU_RELEASE =~ 'Ubuntu 14.04' ]]; then @@ -26,7 +26,7 @@ fi apt-get update -y # Install package dependencies from apt -RUNLEVEL=1 apt-get install -y libcairo2-dev libffi-dev pkg-config python-dev python-pip fontconfig apache2 libapache2-mod-wsgi git-core collectd memcached gcc g++ make +RUNLEVEL=1 apt-get install -y libcairo2-dev libffi-dev pkg-config python-dev python-pip fontconfig apache2 libapache2-mod-wsgi git-core collectd memcached gcc g++ make libtool automake # Download source repositories for Graphite/Carbon/Whisper and Statsite cd /usr/local/src @@ -39,7 +39,7 @@ git clone https://github.com/armon/statsite.git cd whisper; git checkout ${GRAPHITE_RELEASE}; python setup.py install cd ../carbon; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python setup.py install cd ../graphite-web; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python check-dependencies.py; python setup.py install -cd ../statsite; pip install --egg SCons; make; cp statsite /usr/local/sbin/; cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py +cd ../statsite; ./bootstrap.sh; ./configure; make; cp src/statsite /usr/local/sbin/; cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py # Install configuration files for Graphite/Carbon and Apache cp ${SYNTHESIZE_HOME}/templates/statsite/statsite.conf /etc/statsite.conf @@ -64,23 +64,24 @@ cp ${SYNTHESIZE_HOME}/templates/graphite/webapp/* . sed -i -e "s/UNSAFE_DEFAULT/`date | md5sum | cut -d ' ' -f 1`/" local_settings.py # Setup the Django database -python manage.py syncdb --noinput -chown www-data:www-data ${GRAPHITE_STORAGE}/graphite.db +PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py migrate --noinput --settings=graphite.settings --run-syncdb +PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py loaddata --settings=graphite.settings initial_data.json # Add carbon system user and set permissions groupadd -g 998 carbon useradd -c "carbon user" -g 998 -u 998 -s /bin/false carbon chmod 775 ${GRAPHITE_STORAGE} chown www-data:carbon ${GRAPHITE_STORAGE} +chown www-data:www-data ${GRAPHITE_STORAGE}/graphite.db chown -R carbon ${GRAPHITE_STORAGE}/whisper mkdir ${GRAPHITE_STORAGE}/log/apache2 -chown www-data ${GRAPHITE_STORAGE}/log/webapp +chown -R www-data ${GRAPHITE_STORAGE}/log/webapp chmod +x /etc/init.d/carbon-cache # Setup hourly cron to rebuild Graphite index cp ${SYNTHESIZE_HOME}/templates/graphite/cron/build-index /etc/cron.hourly/graphite-build-index chmod 755 /etc/cron.hourly/graphite-build-index -sudo -u www-data /opt/graphite/bin/build-index.sh +#sudo -u www-data /opt/graphite/bin/build-index.sh # Start our processes update-rc.d carbon-cache defaults @@ -89,4 +90,5 @@ service memcached start service collectd start service apache2 start service statsite start +service apache2 restart diff --git a/templates/graphite/conf/graphite.wsgi b/templates/graphite/conf/graphite.wsgi index fd9a686..84ac302 100755 --- a/templates/graphite/conf/graphite.wsgi +++ b/templates/graphite/conf/graphite.wsgi @@ -1,25 +1,4 @@ -import os, sys +import sys sys.path.append('/opt/graphite/webapp') -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'graphite.settings') -import django - -if django.VERSION < (1, 4): - from django.core.handlers.wsgi import WSGIHandler - application = WSGIHandler() -else: - # From 1.4 wsgi support was improved and since 1.7 old style WSGI script - # causes AppRegistryNotReady exception - # https://docs.djangoproject.com/en/dev/releases/1.7/#wsgi-scripts - from django.core.wsgi import get_wsgi_application - application = get_wsgi_application() - - -# READ THIS -# Initializing the search index can be very expensive, please include -# the WSGIImportScript directive pointing to this script in your vhost -# config to ensure the index is preloaded before any requests are handed -# to the process. -from graphite.logger import log -log.info("graphite.wsgi - pid %d - reloading search index" % os.getpid()) -import graphite.metrics.search +from graphite.wsgi import application diff --git a/templates/graphite/webapp/initial_data.json b/templates/graphite/webapp/initial_data.json index 71b0588..79853ed 100644 --- a/templates/graphite/webapp/initial_data.json +++ b/templates/graphite/webapp/initial_data.json @@ -1 +1,659 @@ -[{"pk": 14, "model": "contenttypes.contenttype", "fields": {"model": "contenttype", "name": "content type", "app_label": "contenttypes"}}, {"pk": 6, "model": "contenttypes.contenttype", "fields": {"model": "dashboard", "name": "dashboard", "app_label": "dashboard"}}, {"pk": 7, "model": "contenttypes.contenttype", "fields": {"model": "event", "name": "event", "app_label": "events"}}, {"pk": 10, "model": "contenttypes.contenttype", "fields": {"model": "group", "name": "group", "app_label": "auth"}}, {"pk": 8, "model": "contenttypes.contenttype", "fields": {"model": "link", "name": "link", "app_label": "url_shortener"}}, {"pk": 13, "model": "contenttypes.contenttype", "fields": {"model": "logentry", "name": "log entry", "app_label": "admin"}}, {"pk": 5, "model": "contenttypes.contenttype", "fields": {"model": "mygraph", "name": "my graph", "app_label": "account"}}, {"pk": 9, "model": "contenttypes.contenttype", "fields": {"model": "permission", "name": "permission", "app_label": "auth"}}, {"pk": 1, "model": "contenttypes.contenttype", "fields": {"model": "profile", "name": "profile", "app_label": "account"}}, {"pk": 12, "model": "contenttypes.contenttype", "fields": {"model": "session", "name": "session", "app_label": "sessions"}}, {"pk": 15, "model": "contenttypes.contenttype", "fields": {"model": "tag", "name": "tag", "app_label": "tagging"}}, {"pk": 16, "model": "contenttypes.contenttype", "fields": {"model": "taggeditem", "name": "tagged item", "app_label": "tagging"}}, {"pk": 11, "model": "contenttypes.contenttype", "fields": {"model": "user", "name": "user", "app_label": "auth"}}, {"pk": 2, "model": "contenttypes.contenttype", "fields": {"model": "variable", "name": "variable", "app_label": "account"}}, {"pk": 3, "model": "contenttypes.contenttype", "fields": {"model": "view", "name": "view", "app_label": "account"}}, {"pk": 4, "model": "contenttypes.contenttype", "fields": {"model": "window", "name": "window", "app_label": "account"}}, {"pk": 1, "model": "auth.permission", "fields": {"codename": "add_mygraph", "name": "Can add my graph", "content_type": 5}}, {"pk": 6, "model": "auth.permission", "fields": {"codename": "change_mygraph", "name": "Can change my graph", "content_type": 5}}, {"pk": 11, "model": "auth.permission", "fields": {"codename": "delete_mygraph", "name": "Can delete my graph", "content_type": 5}}, {"pk": 2, "model": "auth.permission", "fields": {"codename": "add_profile", "name": "Can add profile", "content_type": 1}}, {"pk": 7, "model": "auth.permission", "fields": {"codename": "change_profile", "name": "Can change profile", "content_type": 1}}, {"pk": 12, "model": "auth.permission", "fields": {"codename": "delete_profile", "name": "Can delete profile", "content_type": 1}}, {"pk": 3, "model": "auth.permission", "fields": {"codename": "add_variable", "name": "Can add variable", "content_type": 2}}, {"pk": 8, "model": "auth.permission", "fields": {"codename": "change_variable", "name": "Can change variable", "content_type": 2}}, {"pk": 13, "model": "auth.permission", "fields": {"codename": "delete_variable", "name": "Can delete variable", "content_type": 2}}, {"pk": 4, "model": "auth.permission", "fields": {"codename": "add_view", "name": "Can add view", "content_type": 3}}, {"pk": 9, "model": "auth.permission", "fields": {"codename": "change_view", "name": "Can change view", "content_type": 3}}, {"pk": 14, "model": "auth.permission", "fields": {"codename": "delete_view", "name": "Can delete view", "content_type": 3}}, {"pk": 5, "model": "auth.permission", "fields": {"codename": "add_window", "name": "Can add window", "content_type": 4}}, {"pk": 10, "model": "auth.permission", "fields": {"codename": "change_window", "name": "Can change window", "content_type": 4}}, {"pk": 15, "model": "auth.permission", "fields": {"codename": "delete_window", "name": "Can delete window", "content_type": 4}}, {"pk": 37, "model": "auth.permission", "fields": {"codename": "add_logentry", "name": "Can add log entry", "content_type": 13}}, {"pk": 38, "model": "auth.permission", "fields": {"codename": "change_logentry", "name": "Can change log entry", "content_type": 13}}, {"pk": 39, "model": "auth.permission", "fields": {"codename": "delete_logentry", "name": "Can delete log entry", "content_type": 13}}, {"pk": 25, "model": "auth.permission", "fields": {"codename": "add_group", "name": "Can add group", "content_type": 10}}, {"pk": 28, "model": "auth.permission", "fields": {"codename": "change_group", "name": "Can change group", "content_type": 10}}, {"pk": 31, "model": "auth.permission", "fields": {"codename": "delete_group", "name": "Can delete group", "content_type": 10}}, {"pk": 26, "model": "auth.permission", "fields": {"codename": "add_permission", "name": "Can add permission", "content_type": 9}}, {"pk": 29, "model": "auth.permission", "fields": {"codename": "change_permission", "name": "Can change permission", "content_type": 9}}, {"pk": 32, "model": "auth.permission", "fields": {"codename": "delete_permission", "name": "Can delete permission", "content_type": 9}}, {"pk": 27, "model": "auth.permission", "fields": {"codename": "add_user", "name": "Can add user", "content_type": 11}}, {"pk": 30, "model": "auth.permission", "fields": {"codename": "change_user", "name": "Can change user", "content_type": 11}}, {"pk": 33, "model": "auth.permission", "fields": {"codename": "delete_user", "name": "Can delete user", "content_type": 11}}, {"pk": 40, "model": "auth.permission", "fields": {"codename": "add_contenttype", "name": "Can add content type", "content_type": 14}}, {"pk": 41, "model": "auth.permission", "fields": {"codename": "change_contenttype", "name": "Can change content type", "content_type": 14}}, {"pk": 42, "model": "auth.permission", "fields": {"codename": "delete_contenttype", "name": "Can delete content type", "content_type": 14}}, {"pk": 16, "model": "auth.permission", "fields": {"codename": "add_dashboard", "name": "Can add dashboard", "content_type": 6}}, {"pk": 17, "model": "auth.permission", "fields": {"codename": "change_dashboard", "name": "Can change dashboard", "content_type": 6}}, {"pk": 18, "model": "auth.permission", "fields": {"codename": "delete_dashboard", "name": "Can delete dashboard", "content_type": 6}}, {"pk": 19, "model": "auth.permission", "fields": {"codename": "add_event", "name": "Can add event", "content_type": 7}}, {"pk": 20, "model": "auth.permission", "fields": {"codename": "change_event", "name": "Can change event", "content_type": 7}}, {"pk": 21, "model": "auth.permission", "fields": {"codename": "delete_event", "name": "Can delete event", "content_type": 7}}, {"pk": 34, "model": "auth.permission", "fields": {"codename": "add_session", "name": "Can add session", "content_type": 12}}, {"pk": 35, "model": "auth.permission", "fields": {"codename": "change_session", "name": "Can change session", "content_type": 12}}, {"pk": 36, "model": "auth.permission", "fields": {"codename": "delete_session", "name": "Can delete session", "content_type": 12}}, {"pk": 43, "model": "auth.permission", "fields": {"codename": "add_tag", "name": "Can add tag", "content_type": 15}}, {"pk": 45, "model": "auth.permission", "fields": {"codename": "change_tag", "name": "Can change tag", "content_type": 15}}, {"pk": 47, "model": "auth.permission", "fields": {"codename": "delete_tag", "name": "Can delete tag", "content_type": 15}}, {"pk": 44, "model": "auth.permission", "fields": {"codename": "add_taggeditem", "name": "Can add tagged item", "content_type": 16}}, {"pk": 46, "model": "auth.permission", "fields": {"codename": "change_taggeditem", "name": "Can change tagged item", "content_type": 16}}, {"pk": 48, "model": "auth.permission", "fields": {"codename": "delete_taggeditem", "name": "Can delete tagged item", "content_type": 16}}, {"pk": 22, "model": "auth.permission", "fields": {"codename": "add_link", "name": "Can add link", "content_type": 8}}, {"pk": 23, "model": "auth.permission", "fields": {"codename": "change_link", "name": "Can change link", "content_type": 8}}, {"pk": 24, "model": "auth.permission", "fields": {"codename": "delete_link", "name": "Can delete link", "content_type": 8}}, {"pk": 1, "model": "auth.user", "fields": {"username": "admin", "first_name": "", "last_name": "", "is_active": true, "is_superuser": true, "is_staff": true, "last_login": "2015-04-19T03:59:46.970Z", "groups": [], "user_permissions": [], "password": "pbkdf2_sha256$10000$TGanizFI0Cja$y88pHH9soEQKI/8gAwvdXtcfX4SQRIHKbwcpDsNbUbY=", "email": "jason@dixongroup.net", "date_joined": "2015-04-19T03:59:46.970Z"}}] \ No newline at end of file +[ +{ + "model": "sessions.session", + "pk": "0wf2kay4ic3yjkixtsn5indg6qpzhbmq", + "fields": { + "session_data": "N2E4ZWVkYjg0YjFkNGU3MTgyYTc5Nzg1MTE0MWM0NzI1YWZkZjFhNTp7Il9hdXRoX3VzZXJfaGFzaCI6IjJhZWU3ZDgzMTAwNDQzZjg0MTNjYmI0NDhlMmViNzlhYzA5YzdjMWUiLCJfYXV0aF91c2VyX2JhY2tlbmQiOiJkamFuZ28uY29udHJpYi5hdXRoLmJhY2tlbmRzLk1vZGVsQmFja2VuZCIsIl9hdXRoX3VzZXJfaWQiOiIyIn0=", + "expire_date": "2016-07-25T21:06:28.060Z" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 1, + "fields": { + "app_label": "account", + "model": "profile" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 2, + "fields": { + "app_label": "account", + "model": "variable" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 3, + "fields": { + "app_label": "account", + "model": "view" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 4, + "fields": { + "app_label": "account", + "model": "window" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 5, + "fields": { + "app_label": "account", + "model": "mygraph" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 6, + "fields": { + "app_label": "dashboard", + "model": "dashboard" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 7, + "fields": { + "app_label": "dashboard", + "model": "template" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 8, + "fields": { + "app_label": "events", + "model": "event" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 9, + "fields": { + "app_label": "url_shortener", + "model": "link" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 10, + "fields": { + "app_label": "auth", + "model": "permission" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 11, + "fields": { + "app_label": "auth", + "model": "group" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 12, + "fields": { + "app_label": "auth", + "model": "user" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 13, + "fields": { + "app_label": "sessions", + "model": "session" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 14, + "fields": { + "app_label": "admin", + "model": "logentry" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 15, + "fields": { + "app_label": "contenttypes", + "model": "contenttype" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 16, + "fields": { + "app_label": "tagging", + "model": "tag" + } +}, +{ + "model": "contenttypes.contenttype", + "pk": 17, + "fields": { + "app_label": "tagging", + "model": "taggeditem" + } +}, +{ + "model": "auth.permission", + "pk": 1, + "fields": { + "name": "Can add profile", + "content_type": 1, + "codename": "add_profile" + } +}, +{ + "model": "auth.permission", + "pk": 2, + "fields": { + "name": "Can change profile", + "content_type": 1, + "codename": "change_profile" + } +}, +{ + "model": "auth.permission", + "pk": 3, + "fields": { + "name": "Can delete profile", + "content_type": 1, + "codename": "delete_profile" + } +}, +{ + "model": "auth.permission", + "pk": 4, + "fields": { + "name": "Can add variable", + "content_type": 2, + "codename": "add_variable" + } +}, +{ + "model": "auth.permission", + "pk": 5, + "fields": { + "name": "Can change variable", + "content_type": 2, + "codename": "change_variable" + } +}, +{ + "model": "auth.permission", + "pk": 6, + "fields": { + "name": "Can delete variable", + "content_type": 2, + "codename": "delete_variable" + } +}, +{ + "model": "auth.permission", + "pk": 7, + "fields": { + "name": "Can add view", + "content_type": 3, + "codename": "add_view" + } +}, +{ + "model": "auth.permission", + "pk": 8, + "fields": { + "name": "Can change view", + "content_type": 3, + "codename": "change_view" + } +}, +{ + "model": "auth.permission", + "pk": 9, + "fields": { + "name": "Can delete view", + "content_type": 3, + "codename": "delete_view" + } +}, +{ + "model": "auth.permission", + "pk": 10, + "fields": { + "name": "Can add window", + "content_type": 4, + "codename": "add_window" + } +}, +{ + "model": "auth.permission", + "pk": 11, + "fields": { + "name": "Can change window", + "content_type": 4, + "codename": "change_window" + } +}, +{ + "model": "auth.permission", + "pk": 12, + "fields": { + "name": "Can delete window", + "content_type": 4, + "codename": "delete_window" + } +}, +{ + "model": "auth.permission", + "pk": 13, + "fields": { + "name": "Can add my graph", + "content_type": 5, + "codename": "add_mygraph" + } +}, +{ + "model": "auth.permission", + "pk": 14, + "fields": { + "name": "Can change my graph", + "content_type": 5, + "codename": "change_mygraph" + } +}, +{ + "model": "auth.permission", + "pk": 15, + "fields": { + "name": "Can delete my graph", + "content_type": 5, + "codename": "delete_mygraph" + } +}, +{ + "model": "auth.permission", + "pk": 16, + "fields": { + "name": "Can add dashboard", + "content_type": 6, + "codename": "add_dashboard" + } +}, +{ + "model": "auth.permission", + "pk": 17, + "fields": { + "name": "Can change dashboard", + "content_type": 6, + "codename": "change_dashboard" + } +}, +{ + "model": "auth.permission", + "pk": 18, + "fields": { + "name": "Can delete dashboard", + "content_type": 6, + "codename": "delete_dashboard" + } +}, +{ + "model": "auth.permission", + "pk": 19, + "fields": { + "name": "Can add template", + "content_type": 7, + "codename": "add_template" + } +}, +{ + "model": "auth.permission", + "pk": 20, + "fields": { + "name": "Can change template", + "content_type": 7, + "codename": "change_template" + } +}, +{ + "model": "auth.permission", + "pk": 21, + "fields": { + "name": "Can delete template", + "content_type": 7, + "codename": "delete_template" + } +}, +{ + "model": "auth.permission", + "pk": 22, + "fields": { + "name": "Can add event", + "content_type": 8, + "codename": "add_event" + } +}, +{ + "model": "auth.permission", + "pk": 23, + "fields": { + "name": "Can change event", + "content_type": 8, + "codename": "change_event" + } +}, +{ + "model": "auth.permission", + "pk": 24, + "fields": { + "name": "Can delete event", + "content_type": 8, + "codename": "delete_event" + } +}, +{ + "model": "auth.permission", + "pk": 25, + "fields": { + "name": "Can add link", + "content_type": 9, + "codename": "add_link" + } +}, +{ + "model": "auth.permission", + "pk": 26, + "fields": { + "name": "Can change link", + "content_type": 9, + "codename": "change_link" + } +}, +{ + "model": "auth.permission", + "pk": 27, + "fields": { + "name": "Can delete link", + "content_type": 9, + "codename": "delete_link" + } +}, +{ + "model": "auth.permission", + "pk": 28, + "fields": { + "name": "Can add permission", + "content_type": 10, + "codename": "add_permission" + } +}, +{ + "model": "auth.permission", + "pk": 29, + "fields": { + "name": "Can change permission", + "content_type": 10, + "codename": "change_permission" + } +}, +{ + "model": "auth.permission", + "pk": 30, + "fields": { + "name": "Can delete permission", + "content_type": 10, + "codename": "delete_permission" + } +}, +{ + "model": "auth.permission", + "pk": 31, + "fields": { + "name": "Can add group", + "content_type": 11, + "codename": "add_group" + } +}, +{ + "model": "auth.permission", + "pk": 32, + "fields": { + "name": "Can change group", + "content_type": 11, + "codename": "change_group" + } +}, +{ + "model": "auth.permission", + "pk": 33, + "fields": { + "name": "Can delete group", + "content_type": 11, + "codename": "delete_group" + } +}, +{ + "model": "auth.permission", + "pk": 34, + "fields": { + "name": "Can add user", + "content_type": 12, + "codename": "add_user" + } +}, +{ + "model": "auth.permission", + "pk": 35, + "fields": { + "name": "Can change user", + "content_type": 12, + "codename": "change_user" + } +}, +{ + "model": "auth.permission", + "pk": 36, + "fields": { + "name": "Can delete user", + "content_type": 12, + "codename": "delete_user" + } +}, +{ + "model": "auth.permission", + "pk": 37, + "fields": { + "name": "Can add session", + "content_type": 13, + "codename": "add_session" + } +}, +{ + "model": "auth.permission", + "pk": 38, + "fields": { + "name": "Can change session", + "content_type": 13, + "codename": "change_session" + } +}, +{ + "model": "auth.permission", + "pk": 39, + "fields": { + "name": "Can delete session", + "content_type": 13, + "codename": "delete_session" + } +}, +{ + "model": "auth.permission", + "pk": 40, + "fields": { + "name": "Can add log entry", + "content_type": 14, + "codename": "add_logentry" + } +}, +{ + "model": "auth.permission", + "pk": 41, + "fields": { + "name": "Can change log entry", + "content_type": 14, + "codename": "change_logentry" + } +}, +{ + "model": "auth.permission", + "pk": 42, + "fields": { + "name": "Can delete log entry", + "content_type": 14, + "codename": "delete_logentry" + } +}, +{ + "model": "auth.permission", + "pk": 43, + "fields": { + "name": "Can add content type", + "content_type": 15, + "codename": "add_contenttype" + } +}, +{ + "model": "auth.permission", + "pk": 44, + "fields": { + "name": "Can change content type", + "content_type": 15, + "codename": "change_contenttype" + } +}, +{ + "model": "auth.permission", + "pk": 45, + "fields": { + "name": "Can delete content type", + "content_type": 15, + "codename": "delete_contenttype" + } +}, +{ + "model": "auth.permission", + "pk": 46, + "fields": { + "name": "Can add tag", + "content_type": 16, + "codename": "add_tag" + } +}, +{ + "model": "auth.permission", + "pk": 47, + "fields": { + "name": "Can change tag", + "content_type": 16, + "codename": "change_tag" + } +}, +{ + "model": "auth.permission", + "pk": 48, + "fields": { + "name": "Can delete tag", + "content_type": 16, + "codename": "delete_tag" + } +}, +{ + "model": "auth.permission", + "pk": 49, + "fields": { + "name": "Can add tagged item", + "content_type": 17, + "codename": "add_taggeditem" + } +}, +{ + "model": "auth.permission", + "pk": 50, + "fields": { + "name": "Can change tagged item", + "content_type": 17, + "codename": "change_taggeditem" + } +}, +{ + "model": "auth.permission", + "pk": 51, + "fields": { + "name": "Can delete tagged item", + "content_type": 17, + "codename": "delete_taggeditem" + } +}, +{ + "model": "auth.user", + "pk": 1, + "fields": { + "password": "!", + "last_login": null, + "is_superuser": false, + "username": "default", + "first_name": "", + "last_name": "", + "email": "default@localhost.localdomain", + "is_staff": false, + "is_active": true, + "date_joined": "2016-07-11T21:04:28.438Z", + "groups": [], + "user_permissions": [] + } +}, +{ + "model": "auth.user", + "pk": 2, + "fields": { + "password": "pbkdf2_sha256$24000$M2J4isqxmZ63$H5z1xNfqRE0c4iY7gk+4kv0hUvhgwU9uWJAGWDlAJrw=", + "last_login": "2016-07-11T21:06:28.049Z", + "is_superuser": true, + "username": "admin", + "first_name": "", + "last_name": "", + "email": "root@localhost", + "is_staff": true, + "is_active": true, + "date_joined": "2016-07-11T21:06:23.230Z", + "groups": [], + "user_permissions": [] + } +}, +{ + "model": "account.profile", + "pk": 1, + "fields": { + "user": 1, + "history": "", + "advancedUI": false + } +}, +{ + "model": "account.profile", + "pk": 2, + "fields": { + "user": 2, + "history": "", + "advancedUI": false + } +} +] diff --git a/templates/graphite/webapp/local_settings.py b/templates/graphite/webapp/local_settings.py index a64016d..c118a93 100644 --- a/templates/graphite/webapp/local_settings.py +++ b/templates/graphite/webapp/local_settings.py @@ -23,12 +23,16 @@ #TIME_ZONE = 'America/Los_Angeles' # Override this to provide documentation specific to your Graphite deployment -#DOCUMENTATION_URL = "http://graphite.readthedocs.org/" +#DOCUMENTATION_URL = "http://graphite.readthedocs.io/" + +# Metric data and graphs are cached for one minute by default +#DEFAULT_CACHE_DURATION = 60 # Logging +LOG_ROTATION = True +LOG_ROTATION_COUNT = 5 LOG_RENDERING_PERFORMANCE = True LOG_CACHE_PERFORMANCE = True -#LOG_METRIC_ACCESS = True # Enable full debug page display on exceptions (Internal Server Error pages) #DEBUG = True @@ -46,9 +50,21 @@ # as every webapp in the cluster should use the exact same values to prevent # unneeded cache misses. Set to [] to disable caching of images and fetched data #MEMCACHE_HOSTS = ['10.10.10.10:11211', '10.10.10.11:11211', '10.10.10.12:11211'] -#DEFAULT_CACHE_DURATION = 60 # Cache images and data for 1 minute MEMCACHE_HOSTS = ['127.0.0.1:11211'] +# Metric data and graphs are cached for one minute by default. If defined, +# DEFAULT_CACHE_POLICY is a list of tuples of minimum query time ranges mapped +# to the cache duration for the results. This allows for larger queries to be +# cached for longer periods of times. All times are in seconds. If the policy is +# empty or undefined, all results will be cached for DEFAULT_CACHE_DURATION. +#DEFAULT_CACHE_DURATION = 60 # Cache images and data for 1 minute +#DEFAULT_CACHE_POLICY = [(0, 60), # default is 60 seconds +# (7200, 120), # >= 2 hour queries are cached 2 minutes +# (21600, 180)] # >= 6 hour queries are cached 3 minutes +#MEMCACHE_KEY_PREFIX = 'graphite' + +# Set URL_PREFIX when deploying graphite-web to a non-root location +#URL_PREFIX = '/graphite' ##################################### # Filesystem Paths # @@ -62,7 +78,7 @@ # of these is relative to GRAPHITE_ROOT #CONF_DIR = '/opt/graphite/conf' #STORAGE_DIR = '/opt/graphite/storage' -#CONTENT_DIR = '/opt/graphite/webapp/content' +#STATIC_ROOT = '/opt/graphite/static' # To further or fully customize the paths, modify the following. Note that the # default settings for each of these are relative to CONF_DIR and STORAGE_DIR @@ -72,10 +88,12 @@ #GRAPHTEMPLATES_CONF = '/opt/graphite/conf/graphTemplates.conf' ## Data directories -# NOTE: If any directory is unreadable in DATA_DIRS it will break metric browsing +# NOTE: If any directory is unreadable in STANDARD_DIRS it will break metric browsing +#CERES_DIR = '/opt/graphite/storage/ceres' #WHISPER_DIR = '/opt/graphite/storage/whisper' #RRD_DIR = '/opt/graphite/storage/rrd' -#DATA_DIRS = [WHISPER_DIR, RRD_DIR] # Default: set from the above variables +# Data directories using the "Standard" finder (i.e. not Ceres) +#STANDARD_DIRS = [WHISPER_DIR, RRD_DIR] # Default: set from the above variables #LOG_DIR = '/opt/graphite/storage/log/webapp' #INDEX_FILE = '/opt/graphite/storage/index' # Search index file @@ -102,22 +120,30 @@ #USE_LDAP_AUTH = True #LDAP_SERVER = "ldap.mycompany.com" #LDAP_PORT = 389 -# OR +#LDAP_USE_TLS = False +# OR #LDAP_URI = "ldaps://ldap.mycompany.com:636" #LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com" #LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com" #LDAP_BASE_PASS = "readonly_account_password" #LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)" # +# User DN template to use for binding (and authentication) against +# the LDAP server. %(username) is replaced with the username supplied at +# graphite login. +#LDAP_USER_DN_TEMPLATE = "CN=%(username)s,OU=users,DC=mycompany,DC=com" +# # If you want to further customize the ldap connection options you should # directly use ldap.set_option to set the ldap module's global options. # For example: # #import ldap -#ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW) +#ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW) # Use ldap.OPT_X_TLS_DEMAND to force TLS +#ldap.set_option(ldap.OPT_REFERRALS, 0) # Enable for Active Directory #ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, "/etc/ssl/ca") #ldap.set_option(ldap.OPT_X_TLS_CERTFILE, "/etc/ssl/mycert.pem") #ldap.set_option(ldap.OPT_X_TLS_KEYFILE, "/etc/ssl/mykey.pem") +#ldap.set_option(ldap.OPT_DEBUG_LEVEL, 65535) # To enable verbose debugging # See http://www.python-ldap.org/ for further details on these options. ## REMOTE_USER authentication. See: https://docs.djangoproject.com/en/dev/howto/auth-remote-user/ @@ -127,6 +153,27 @@ #LOGIN_URL = '/account/login' +############################### +# Authorization for Dashboard # +############################### +# By default, there is no security on dashboards - any user can add, change or delete them. +# This section provides 3 different authorization models, of varying strictness. + +# If set to True, users must be logged in to save or delete dashboards. Defaults to False +#DASHBOARD_REQUIRE_AUTHENTICATION = True + +# If set to the name of a user group, dashboards can be saved and deleted by any user in this +# group. Groups can be set in the Django Admin app, or in LDAP. Defaults to None. +# NOTE: Ignored if DASHBOARD_REQUIRE_AUTHENTICATION is not set +#DASHBOARD_REQUIRE_EDIT_GROUP = 'dashboard-editors-group' + +# If set to True, dashboards can be saved or deleted by any user having the appropriate +# (change or delete) permission (as set in the Django Admin app). Defaults to False +# NOTE: Ignored if DASHBOARD_REQUIRE_AUTHENTICATION is not set +#DASHBOARD_REQUIRE_PERMISSIONS = True + + + ########################## # Database Configuration # ########################## @@ -136,10 +183,10 @@ # Django models such as saved graphs, dashboards, user preferences, etc. # Metric data is not stored here. # -# DO NOT FORGET TO RUN 'manage.py syncdb' AFTER SETTING UP A NEW DATABASE +# DO NOT FORGET TO RUN MIGRATIONS AFTER SETTING UP A NEW DATABASE +# http://graphite.readthedocs.io/en/latest/config-database-setup.html # # The following built-in database engines are available: -# django.db.backends.postgresql # Removed in Django 1.4 # django.db.backends.postgresql_psycopg2 # django.db.backends.mysql # django.db.backends.sqlite3 @@ -160,7 +207,6 @@ #} # - ######################### # Cluster Configuration # ######################### @@ -172,11 +218,19 @@ # used. #CLUSTER_SERVERS = ["10.0.2.2:80", "10.0.2.3:80"] +# This settings control wether https is used to communicate between cluster members +#INTRACLUSTER_HTTPS = False ## These are timeout values (in seconds) for requests to remote webapps -#REMOTE_STORE_FETCH_TIMEOUT = 6 # Timeout to fetch series data -#REMOTE_STORE_FIND_TIMEOUT = 2.5 # Timeout for metric find requests -#REMOTE_STORE_RETRY_DELAY = 60 # Time before retrying a failed remote webapp -#REMOTE_FIND_CACHE_DURATION = 300 # Time to cache remote metric find results +#REMOTE_FIND_TIMEOUT = 3.0 # Timeout for metric find requests +#REMOTE_FETCH_TIMEOUT = 6.0 # Timeout to fetch series data +#REMOTE_RETRY_DELAY = 60.0 # Time before retrying a failed remote webapp +#REMOTE_EXCLUDE_LOCAL = False # Try to detect when a cluster server is localhost and don't forward queries +#FIND_CACHE_DURATION = 300 # Time to cache remote metric find results +# If the query doesn't fall entirely within the FIND_TOLERANCE window +# we disregard the window. This prevents unnecessary remote fetches +# caused when carbon's cache skews node.intervals, giving the appearance +# remote systems have data we don't have locally, which we probably do. +#FIND_TOLERANCE = 2 * FIND_CACHE_DURATION ## Remote rendering settings # Set to True to enable rendering of Graphs on a remote webapp @@ -195,6 +249,22 @@ # You *should* use 127.0.0.1 here in most cases #CARBONLINK_HOSTS = ["127.0.0.1:7002:a", "127.0.0.1:7102:b", "127.0.0.1:7202:c"] #CARBONLINK_TIMEOUT = 1.0 +#CARBONLINK_RETRY_DELAY = 15 # Seconds to blacklist a failed remote server + +# A "keyfunc" is a user-defined python function that is given a metric name +# and returns a string that should be used when hashing the metric name. +# This is important when your hashing has to respect certain metric groupings. +#CARBONLINK_HASHING_KEYFUNC = "/opt/graphite/bin/keyfuncs.py:my_keyfunc" + +# Prefix set in carbon for the carbon specific metrics. Default in carbon is 'carbon' +#CARBON_METRIC_PREFIX='carbon' + +# The replication factor to use with consistent hashing +# This should usually match the value configured in Carbon +#REPLICATION_FACTOR = 1 + +# How often should render.datalib.fetch() retry to get remote data +# MAX_FETCH_RETRIES = 2 ##################################### # Additional Django Settings # @@ -202,4 +272,3 @@ # Uncomment the following line for direct access to Django settings such as # MIDDLEWARE_CLASSES or APPS #from graphite.app_settings import * - From 9d7d45f99cdf4553b26901ce3057fc36a57a780b Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Mon, 11 Jul 2016 17:46:05 -0400 Subject: [PATCH 02/19] update OS release version --- uninstall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uninstall b/uninstall index 84503c9..e091ea8 100755 --- a/uninstall +++ b/uninstall @@ -1,5 +1,5 @@ #!/bin/bash -# Graphite uninstall script for Ubuntu 13.10 x64 +# Graphite uninstall script for Ubuntu 14.04 x64 # Jason Dixon service carbon-cache stop From 0d46b9136d99f52bd52564e94db41e98d755ad8c Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Mon, 11 Jul 2016 17:46:37 -0400 Subject: [PATCH 03/19] initial stab at updating upgrade script, untested --- upgrade | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/upgrade b/upgrade index 6c64e1b..0fd6947 100755 --- a/upgrade +++ b/upgrade @@ -1,5 +1,5 @@ #!/bin/bash -# Synthesize upgrade script for Graphite 0.9.15 +# Synthesize upgrade script for Graphite master # Jason Dixon SYNTHESIZE_HOME=$( cd "$( dirname "$0" )" && pwd ) @@ -10,7 +10,7 @@ GRAPHITE_CONF="${GRAPHITE_HOME}/conf" GRAPHITE_STORAGE="${GRAPHITE_HOME}/storage" if [ -z $GRAPHITE_RELEASE ]; then - GRAPHITE_RELEASE='0.9.15' + GRAPHITE_RELEASE='master' fi if [[ ! $UBUNTU_RELEASE =~ 'Ubuntu 14.04' ]]; then @@ -49,11 +49,14 @@ cp -b ${SYNTHESIZE_HOME}/templates/graphite/webapp/* . sed -i -e "s/UNSAFE_DEFAULT/`date | md5sum | cut -d ' ' -f 1`/" local_settings.py # Upgrade Django and migrate the webapp database -pip install Django==1.7 -cp ${GRAPHITE_STORAGE}/graphite.db ${GRAPHITE_STORAGE}/graphite.db.backup-`date +%Y%m%d_%H%M%S` -PYTHONPATH=/opt/graphite/webapp django-admin.py syncdb --noinput --no-initial-data --settings=graphite.settings +service apache2 stop +BACKUP=`date +%Y%m%d_%H%M%S` +mv ${GRAPHITE_STORAGE}/graphite.db ${GRAPHITE_STORAGE}/graphite.db.backup-${BACKUP} +PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py dumpdata --indent 4 --settings=graphite.settings > dump_${BACKUP}_data.json +PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py migrate --noinput --settings=graphite.settings --run-syncdb +PYTHONPATH=${GRAPHITE_HOME}/webapp django-admin.py loaddata --settings=graphite.settings dump_${BACKUP}_data.json # Restart our processes service carbon-cache restart service memcached restart -service apache2 restart +service apache2 start From fe259b5525c393f7f866c032ee3a10ef6500cdf1 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Mon, 11 Jul 2016 17:53:15 -0400 Subject: [PATCH 04/19] caveat emptor --- upgrade | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/upgrade b/upgrade index 0fd6947..20e8b45 100755 --- a/upgrade +++ b/upgrade @@ -9,6 +9,15 @@ GRAPHITE_HOME='/opt/graphite' GRAPHITE_CONF="${GRAPHITE_HOME}/conf" GRAPHITE_STORAGE="${GRAPHITE_HOME}/storage" +if [[ ! $DANGER_ZONE =~ 'TRUE' ]]; then + echo "This upgrade script has not been thoroughly tested. There is a non-zero" + echo "chance that running this script will break your Synthesize-Graphite" + echo "installation. If you accept this warning and wish to continue anyways," + echo "run this script again with the DANGER_ZONE environment variable set to TRUE." + echo + exit 1 +fi + if [ -z $GRAPHITE_RELEASE ]; then GRAPHITE_RELEASE='master' fi From f129c6ca06fd9765a6bda95ca4eb4d6af492d945 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Mon, 11 Jul 2016 22:54:58 -0400 Subject: [PATCH 05/19] attempt to get DO one-click installer working with experimental branch --- app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.yml b/app.yml index 36cfb88..434e8b4 100644 --- a/app.yml +++ b/app.yml @@ -11,4 +11,4 @@ config: packages: - git runcmd: - - cd /home/install && git clone https://github.com/obfuscurity/synthesize.git && cd synthesize && bash install + - cd /home/install && git clone https://github.com/obfuscurity/synthesize.git && cd synthesize && git checkout feature/graphite-master && bash install From 72f27b1c82c437a6468d0a1c97d378b72065439d Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Mon, 11 Jul 2016 23:12:20 -0400 Subject: [PATCH 06/19] Revert "attempt to get DO one-click installer working with experimental branch" This reverts commit f129c6ca06fd9765a6bda95ca4eb4d6af492d945. --- app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.yml b/app.yml index 434e8b4..36cfb88 100644 --- a/app.yml +++ b/app.yml @@ -11,4 +11,4 @@ config: packages: - git runcmd: - - cd /home/install && git clone https://github.com/obfuscurity/synthesize.git && cd synthesize && git checkout feature/graphite-master && bash install + - cd /home/install && git clone https://github.com/obfuscurity/synthesize.git && cd synthesize && bash install From 4a36b1e0c2bf7333cd977536ee68e8d280c5f718 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Tue, 12 Jul 2016 12:54:26 -0400 Subject: [PATCH 07/19] add Grafana and update README --- README.md | 22 +++++++++++++++++----- Vagrantfile | 1 + install | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ca94095..0dc6734 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,18 @@ Installing Graphite doesn't have to be difficult. The `install` script in synthe Synthesize is built to run on Ubuntu 14.04 LTS. It will __not__ run on other Ubuntu releases or Linux distributions. The goal of this project is not to become an automation alternative to modern configuration management utilities (e.g. Chef or Puppet), but rather, to make it as easy as possible for the beginner Graphite user to get started and familiar with the project without having to learn a suite of other automation and/or infrastructure-related projects. -The resulting Graphite server __listens only on https port 443__ and has been configured to collect metrics specifically for helping profile the performance of your Graphite and Carbon services. It uses memcached for improved query performance, and Statsite for a fast, C-based implementation of the StatsD collector/aggregator. +The resulting Graphite web interface __listens only on https port 443__ and has been configured to collect metrics specifically for helping profile the performance of your Graphite and Carbon services. It uses memcached for improved query performance, and Statsite for a fast, C-based implementation of the StatsD collector/aggregator. + +Beginning with version 3.0 we've also incorporated the Grafana dashboard project, a modern and full-featured alternative to Graphite's built-in Composer and Dashboard interfaces. It also includes a default dashboard for monitoring Carbon's internal statistics. :warning: **WARNING:** You should not install Synthesize directly on your personal development system. It's strongly suggested that you use a VM or other temporary VPS instance for sandboxing Synthesize. ## Provides -* Graphite 0.9.15 ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper)) +* Graphite master (0.10.0-alpha or pre-1.0) ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper)) * StatsD ([statsite](https://github.com/armon/statsite)) -* collectd agent +* (Collectd)[http://collectd.org/] +* (Grafana)[https://grafana.org/] ## Dependencies @@ -39,9 +42,11 @@ Synthesize configures the following host ports to forward to the private vagrant ``` config.vm.network :forwarded_port, guest: 443, host: 8443 -config.vm.network :forwarded_port, guest: 8125, host: 8125 +config.vm.network :forwarded_port, guest: 8125, host: 8125, protocol: 'tcp' +config.vm.network :forwarded_port, guest: 8125, host: 8125, protocol: 'udp' config.vm.network :forwarded_port, guest: 2003, host: 22003 config.vm.network :forwarded_port, guest: 2004, host: 22004 +config.vm.network :forwarded_port, guest: 3000, host: 3030 ``` ``` @@ -64,9 +69,16 @@ $ cd /opt/graphite/webapp/graphite $ sudo python manage.py changepassword admin ``` +Grafana includes a default user to start: + +* username `admin` +* password `admin` + ## Upgrade -It's now possible to upgrade an existing Synthesize (e.g. Graphite 0.9.12) to the newest Graphite 0.9.15. You will need to checkout at least version 2.3.0 of Synthesize for this feature. Besides upgrading the Graphite components, it will also migrate the webapp database (`graphite.db`) to the newest fixtures version. +:warning: **WARNING:** The following information is outdated for this experimental branch. If you attempt to run the upgrade script it will display a warning with further instructions to acknowledge the current experimental status and override the warning. + +It's now possible to upgrade an existing Synthesize (e.g. Graphite 0.9.15) to the newest Graphite `HEAD`. Besides upgrading the Graphite components, it will also migrate the webapp database (`graphite.db`) to the newest fixtures version. ``` $ cd synthesize diff --git a/Vagrantfile b/Vagrantfile index 9a9758c..6f2533e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -11,6 +11,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8125, host: 8125, protocol: 'udp' config.vm.network :forwarded_port, guest: 2003, host: 22003 config.vm.network :forwarded_port, guest: 2004, host: 22004 + config.vm.network :forwarded_port, guest: 3000, host: 3030 graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? 'master' : ENV['GRAPHITE_RELEASE'] config.vm.provision "shell", inline: "cd /vagrant; GRAPHITE_RELEASE=#{graphite_version} ./install" end diff --git a/install b/install index 03c6986..56c4f03 100755 --- a/install +++ b/install @@ -83,6 +83,20 @@ cp ${SYNTHESIZE_HOME}/templates/graphite/cron/build-index /etc/cron.hourly/graph chmod 755 /etc/cron.hourly/graphite-build-index #sudo -u www-data /opt/graphite/bin/build-index.sh +# Install Grafana +echo 'deb https://packagecloud.io/grafana/stable/debian/ wheezy main' > /etc/apt/sources.list.d/grafana.list +curl https://packagecloud.io/gpg.key | apt-key add - +apt-get update -y +apt-get install -y grafana +service grafana-server start +sleep 5 +curl -X POST -H 'Content-Type: application/json' -u 'admin:admin' \ + -d '{ "name": "graphite", "type": "graphite", "url": "https://127.0.0.1:443", "access": "proxy", "basicAuth": false }' \ + "http://127.0.0.1:3000/api/datasources" +curl -X POST -H 'Content-Type: application/json' -u 'admin:admin' \ + -d '{ "inputs": [{"name": "*", "pluginId": "graphite", "type": "datasource", "value": "graphite"}], "overwrite": true, "path": "dashboards/carbon_metrics.json", "pluginId": "graphite" }' \ + "http://127.0.0.1:3000/api/dashboards/import" + # Start our processes update-rc.d carbon-cache defaults service carbon-cache start From 65ad6a06266d8e2e18d739fdcf115f40564cb838 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Tue, 12 Jul 2016 13:00:44 -0400 Subject: [PATCH 08/19] fix markdown --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0dc6734..84b2f75 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ Beginning with version 3.0 we've also incorporated the Grafana dashboard project * Graphite master (0.10.0-alpha or pre-1.0) ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper)) * StatsD ([statsite](https://github.com/armon/statsite)) -* (Collectd)[http://collectd.org/] -* (Grafana)[https://grafana.org/] +* [Collectd](http://collectd.org/) +* [Grafana](https://grafana.org/) ## Dependencies From 437a4fb8aa642883f11e24557ec22962df6e1a60 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Fri, 16 Sep 2016 15:27:50 -0400 Subject: [PATCH 09/19] re-enable build-index since this was fixed in graphite master --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 56c4f03..e88c947 100755 --- a/install +++ b/install @@ -81,7 +81,7 @@ chmod +x /etc/init.d/carbon-cache # Setup hourly cron to rebuild Graphite index cp ${SYNTHESIZE_HOME}/templates/graphite/cron/build-index /etc/cron.hourly/graphite-build-index chmod 755 /etc/cron.hourly/graphite-build-index -#sudo -u www-data /opt/graphite/bin/build-index.sh +sudo -u www-data /opt/graphite/bin/build-index.sh # Install Grafana echo 'deb https://packagecloud.io/grafana/stable/debian/ wheezy main' > /etc/apt/sources.list.d/grafana.list From fdfa107bb6c3cd5778b04a8e517b5f77348a99f6 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Fri, 16 Sep 2016 15:28:10 -0400 Subject: [PATCH 10/19] lower default cache duration --- templates/graphite/webapp/local_settings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/templates/graphite/webapp/local_settings.py b/templates/graphite/webapp/local_settings.py index c118a93..38a0040 100644 --- a/templates/graphite/webapp/local_settings.py +++ b/templates/graphite/webapp/local_settings.py @@ -25,9 +25,6 @@ # Override this to provide documentation specific to your Graphite deployment #DOCUMENTATION_URL = "http://graphite.readthedocs.io/" -# Metric data and graphs are cached for one minute by default -#DEFAULT_CACHE_DURATION = 60 - # Logging LOG_ROTATION = True LOG_ROTATION_COUNT = 5 @@ -58,6 +55,7 @@ # cached for longer periods of times. All times are in seconds. If the policy is # empty or undefined, all results will be cached for DEFAULT_CACHE_DURATION. #DEFAULT_CACHE_DURATION = 60 # Cache images and data for 1 minute +DEFAULT_CACHE_DURATION = 10 #DEFAULT_CACHE_POLICY = [(0, 60), # default is 60 seconds # (7200, 120), # >= 2 hour queries are cached 2 minutes # (21600, 180)] # >= 6 hour queries are cached 3 minutes From 756366d699d6d161f7e802679d314fd875b256d9 Mon Sep 17 00:00:00 2001 From: Greg Burek Date: Thu, 15 Sep 2016 17:06:25 -0700 Subject: [PATCH 11/19] Match WSGIDaemonProcess with graphite-web example The included apache configuration of WSGIDaemonProcess is very limited. This commit adopts the example settings from graphite-web[1]. This serves as a bread crumb for tuning this configuration for deployments with significant read traffic. [1] https://github.com/graphite-project/graphite-web/blob/20d443b7158b3e601b0f77279247804ca7b4fc3a/examples/example-graphite-vhost.conf#L31 --- templates/apache/graphite.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/apache/graphite.conf b/templates/apache/graphite.conf index 6f1400b..8f0895b 100644 --- a/templates/apache/graphite.conf +++ b/templates/apache/graphite.conf @@ -6,7 +6,7 @@ CustomLog /opt/graphite/storage/log/apache2/access.log combined ErrorLog /opt/graphite/storage/log/apache2/error.log - WSGIDaemonProcess graphiteweb python-path=/opt/graphite:/opt/graphite/lib/python2.7/site-packages + WSGIDaemonProcess graphiteweb python-path=/opt/graphite:/opt/graphite/lib/python2.7/site-packages processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 WSGIProcessGroup graphiteweb WSGIApplicationGroup %{GLOBAL} WSGIImportScript /opt/graphite/conf/graphite.wsgi process-group=graphiteweb application-group=%{GLOBAL} From e602fe669507723083a3298644caf1fc8af491e1 Mon Sep 17 00:00:00 2001 From: Greg Burek Date: Thu, 15 Sep 2016 17:22:01 -0700 Subject: [PATCH 12/19] Adds vim syntax hint to apache config --- templates/apache/graphite.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/apache/graphite.conf b/templates/apache/graphite.conf index 8f0895b..b053d42 100644 --- a/templates/apache/graphite.conf +++ b/templates/apache/graphite.conf @@ -42,3 +42,5 @@ RewriteCond %{HTTPS} off RewriteRule ^/?(.*) https://%{SERVER_NAME}/$1 [R,L] + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet From f37a3662f0ec034f681050e01f4988404da2c43d Mon Sep 17 00:00:00 2001 From: Colin Ingarfield Date: Tue, 1 Nov 2016 10:01:33 -0500 Subject: [PATCH 13/19] fix statsite build to call its autogen instead of bootstrap --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index e88c947..1a91498 100755 --- a/install +++ b/install @@ -39,7 +39,7 @@ git clone https://github.com/armon/statsite.git cd whisper; git checkout ${GRAPHITE_RELEASE}; python setup.py install cd ../carbon; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python setup.py install cd ../graphite-web; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python check-dependencies.py; python setup.py install -cd ../statsite; ./bootstrap.sh; ./configure; make; cp src/statsite /usr/local/sbin/; cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py +cd ../statsite; ./autogen.sh; ./configure; make; cp statsite /usr/local/sbin/; cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py # Install configuration files for Graphite/Carbon and Apache cp ${SYNTHESIZE_HOME}/templates/statsite/statsite.conf /etc/statsite.conf From a5afb5a5e644574804a5358278023d13dc85c531 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Fri, 2 Dec 2016 10:22:07 -0500 Subject: [PATCH 14/19] force unix line-endings, refs #21 --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf From 9ef3844f24f71df1e7196bbb0fd4586d294cfae8 Mon Sep 17 00:00:00 2001 From: Chris Mihalcik Date: Fri, 6 Jan 2017 15:28:44 -0600 Subject: [PATCH 15/19] support RabbitMQ 3.2.4+, fixes obfuscurity/synthesize#54 --- install | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install b/install index 1a91498..8af90d5 100755 --- a/install +++ b/install @@ -41,6 +41,9 @@ cd ../carbon; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; cd ../graphite-web; git checkout ${GRAPHITE_RELEASE}; pip install -r requirements.txt; python check-dependencies.py; python setup.py install cd ../statsite; ./autogen.sh; ./configure; make; cp statsite /usr/local/sbin/; cp sinks/graphite.py /usr/local/sbin/statsite-sink-graphite.py +# Update txamqp to support RabbitMQ 2.4+ +pip install txamqp==0.6.2 --upgrade + # Install configuration files for Graphite/Carbon and Apache cp ${SYNTHESIZE_HOME}/templates/statsite/statsite.conf /etc/statsite.conf mkdir ${GRAPHITE_CONF}/examples From 4f574416676788d75bb0d2a75208c026e5c721c6 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Sat, 25 Feb 2017 21:38:28 -0500 Subject: [PATCH 16/19] check for python-openssl and exit if installed --- install | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/install b/install index 8af90d5..274e21d 100755 --- a/install +++ b/install @@ -9,6 +9,39 @@ GRAPHITE_HOME='/opt/graphite' GRAPHITE_CONF="${GRAPHITE_HOME}/conf" GRAPHITE_STORAGE="${GRAPHITE_HOME}/storage" +# Check to see if python-openssl is installed first +set +e +PYTHON_OPENSSL_INSTALLED=`dpkg -s python-openssl 2>/dev/null | grep '^Status'` +set -e + +# This can break Carbon's python-twisted, exit with explanation +if [[ ! $PYTHON_OPENSSL_INSTALLED == '' ]]; then + set +x + clear + echo + echo + echo + echo "##########################################################" + echo + echo "It appears that you have the python-openssl installed." + echo "There are known issues with this package that can cause" + echo "the Carbon service to fail to start. If you don't need" + echo "the python-openssl library, it's recommended that you" + echo "uninstall this package before continuing." + echo + echo " $ sudo apt-get remove python-openssl" + echo + echo "More information is provided at the link below." + echo + echo "https://github.com/graphite-project/graphite-web/issues/1721#issuecomment-253279181" + echo + echo "##########################################################" + echo + echo + echo + exit 1 +fi + if [ -z $GRAPHITE_RELEASE ]; then GRAPHITE_RELEASE='master' fi From 2fb29e9e18038deffbf42238380594641be469b8 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Tue, 11 Apr 2017 19:30:25 -0400 Subject: [PATCH 17/19] update scripts for 1.0.0 --- Vagrantfile | 2 +- install | 2 +- upgrade | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 6f2533e..e99d418 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,6 +12,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 2003, host: 22003 config.vm.network :forwarded_port, guest: 2004, host: 22004 config.vm.network :forwarded_port, guest: 3000, host: 3030 - graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? 'master' : ENV['GRAPHITE_RELEASE'] + graphite_version = ENV['GRAPHITE_RELEASE'].nil? ? '1.0.0' : ENV['GRAPHITE_RELEASE'] config.vm.provision "shell", inline: "cd /vagrant; GRAPHITE_RELEASE=#{graphite_version} ./install" end diff --git a/install b/install index 274e21d..0bf2671 100755 --- a/install +++ b/install @@ -43,7 +43,7 @@ if [[ ! $PYTHON_OPENSSL_INSTALLED == '' ]]; then fi if [ -z $GRAPHITE_RELEASE ]; then - GRAPHITE_RELEASE='master' + GRAPHITE_RELEASE='1.0.0' fi if [[ ! $UBUNTU_RELEASE =~ 'Ubuntu 14.04' ]]; then diff --git a/upgrade b/upgrade index 20e8b45..e173abf 100755 --- a/upgrade +++ b/upgrade @@ -1,5 +1,5 @@ #!/bin/bash -# Synthesize upgrade script for Graphite master +# Synthesize upgrade script for Graphite 1.0.0 # Jason Dixon SYNTHESIZE_HOME=$( cd "$( dirname "$0" )" && pwd ) @@ -19,7 +19,7 @@ if [[ ! $DANGER_ZONE =~ 'TRUE' ]]; then fi if [ -z $GRAPHITE_RELEASE ]; then - GRAPHITE_RELEASE='master' + GRAPHITE_RELEASE='1.0.0' fi if [[ ! $UBUNTU_RELEASE =~ 'Ubuntu 14.04' ]]; then From 05f7ee6832d6ad6a3a0b3b43ecfded074a7f1ad1 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Tue, 11 Apr 2017 19:43:30 -0400 Subject: [PATCH 18/19] silence interactive dpkg noise --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 0bf2671..1352e00 100755 --- a/install +++ b/install @@ -123,7 +123,7 @@ sudo -u www-data /opt/graphite/bin/build-index.sh echo 'deb https://packagecloud.io/grafana/stable/debian/ wheezy main' > /etc/apt/sources.list.d/grafana.list curl https://packagecloud.io/gpg.key | apt-key add - apt-get update -y -apt-get install -y grafana +DEBIAN_FRONTEND=noninteractive apt-get install -y grafana service grafana-server start sleep 5 curl -X POST -H 'Content-Type: application/json' -u 'admin:admin' \ From 66effd5d5bd81914a5960b578b34e873576749a8 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Tue, 11 Apr 2017 19:48:30 -0400 Subject: [PATCH 19/19] minor updates to README --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 84b2f75..aae7395 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ Synthesize is built to run on Ubuntu 14.04 LTS. It will __not__ run on other Ubu The resulting Graphite web interface __listens only on https port 443__ and has been configured to collect metrics specifically for helping profile the performance of your Graphite and Carbon services. It uses memcached for improved query performance, and Statsite for a fast, C-based implementation of the StatsD collector/aggregator. -Beginning with version 3.0 we've also incorporated the Grafana dashboard project, a modern and full-featured alternative to Graphite's built-in Composer and Dashboard interfaces. It also includes a default dashboard for monitoring Carbon's internal statistics. +Beginning with version 3.0.0 we've also incorporated the Grafana dashboard project, a modern and full-featured alternative to Graphite's built-in Composer and Dashboard interfaces. It also includes a default dashboard for monitoring Carbon's internal statistics. :warning: **WARNING:** You should not install Synthesize directly on your personal development system. It's strongly suggested that you use a VM or other temporary VPS instance for sandboxing Synthesize. ## Provides -* Graphite master (0.10.0-alpha or pre-1.0) ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper)) +* Graphite 1.0.0 ([graphite-web](https://github.com/graphite-project/graphite-web), [carbon](https://github.com/graphite-project/carbon), [whisper](https://github.com/graphite-project/whisper)) * StatsD ([statsite](https://github.com/armon/statsite)) * [Collectd](http://collectd.org/) * [Grafana](https://grafana.org/) @@ -57,6 +57,8 @@ $ vagrant up ## Administration +### Graphite-Web + There is a superuser (Django) account that grants access to the administrative features in the backend Django database. The default credentials are: * username `admin` @@ -69,6 +71,8 @@ $ cd /opt/graphite/webapp/graphite $ sudo python manage.py changepassword admin ``` +### Grafana + Grafana includes a default user to start: * username `admin`