From 84047c3dd9b969accfa0f7f0efb98047277f3193 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 9 Apr 2018 13:25:13 -0400 Subject: [PATCH 01/36] - docker info on gunicorn --- docs/source/installation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 4fb72f12..98a41eda 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -166,6 +166,8 @@ Read the `Gunicorn Docs `_ fo .. note:: For the *Metrics* section to broadcast via websocket, you must have gunicorn set to 1 worker. +.. note:: The Docker container available on DockerHub is pre-configured to run with gunicorn. + Troubleshooting --------------- From 38d19942f306f670f26d5ed8c4f119172f981f85 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 9 Apr 2018 14:13:50 -0400 Subject: [PATCH 02/36] -- --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 561b0231..d80feeb6 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ ElasticHQ Simplified Monitoring and Management for ElasticSearch clusters. -[![gitHub stars](https://img.shields.io/github/stars/ElasticHQ/elasticsearch-HQ.svg)](https://github.com/ElasticHQ/elasticsearch-HQ) -[![gitHub issues](https://img.shields.io/github/issues/ElasticHQ/elasticsearch-HQ.svg)](https://github.com/ElasticHQ/elasticsearch-HQ) +[![gitHub stars](https://img.shields.io/github/stars/ElasticHQ/elasticsearch-HQ.svg)](https://github.com/ElasticHQ/elasticsearch-HQ) +[![docker pulls](https://img.shields.io/docker/pulls/elastichq/elasticsearch-hq.svg)](https://hub.docker.com/r/elastichq/elasticsearch-hq 'DockerHub') [![latest](https://img.shields.io/github/release/ElasticHQ/elasticsearch-HQ.svg)](https://github.com/ElasticHQ/elasticsearch-HQ) -[![docker pulls](https://img.shields.io/docker/pulls/elastichq/elasticsearch-hq.svg)](https://hub.docker.com/r/elastichq/elasticsearch-hq 'DockerHub') +[![gitHub issues](https://img.shields.io/github/issues/ElasticHQ/elasticsearch-HQ.svg)](https://github.com/ElasticHQ/elasticsearch-HQ) ![python](https://img.shields.io/badge/python-v3.4%20%2F%20v3.6-blue.svg) [![license](https://img.shields.io/badge/license-ASL-blue.svg)](https://opensource.org/licenses/ASL) From 64831c6e4487c7eed0d4773aa73a444faff9ed19 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Wed, 11 Apr 2018 12:48:29 -0400 Subject: [PATCH 03/36] -- --- elastichq/api/endpoints.py | 2 ++ elastichq/api/hq.py | 52 +++++++++++++++++++++++++++ elastichq/common/utils.py | 12 +++++++ elastichq/config/settings.py | 26 +++++++++++++- elastichq/globals.py | 18 ++++++++-- elastichq/service/HQService.py | 59 +++++++++++++++++++++++++++++++ elastichq/service/QueryService.py | 8 +++++ elastichq/service/__init__.py | 2 +- requirements.txt | 5 ++- 9 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 elastichq/api/hq.py diff --git a/elastichq/api/endpoints.py b/elastichq/api/endpoints.py index a1e92318..08643764 100644 --- a/elastichq/api/endpoints.py +++ b/elastichq/api/endpoints.py @@ -18,5 +18,7 @@ from . import snapshot # noinspection PyUnresolvedReferences from . import query +# noinspection PyUnresolvedReferences +from . import hq diff --git a/elastichq/api/hq.py b/elastichq/api/hq.py new file mode 100644 index 00000000..58c76475 --- /dev/null +++ b/elastichq/api/hq.py @@ -0,0 +1,52 @@ +""" +.. module:: hq + +.. moduleauthor:: Roy Russo +""" + +from flask import request +from flask_restful import Resource + +from . import api +from ..common.api_response import APIResponse +from ..common.exceptions import request_wrapper +from ..common.status_codes import HTTP_Status +from ..service import HQService + + +class HQClusterSettings(Resource): + + @request_wrapper + def get(self, cluster_name): + """ + Gets or Creates the HQ settings for this cluster. + :param cluster_name: + :return: + """ + response = HQService().get_settings(cluster_name) + return APIResponse(response, HTTP_Status.OK, None) + + @request_wrapper + def put(self, cluster_name): + """ + Accepts partial settings doc, ie: + { + "websocket_interval": 15, + "historic_days_to_store" : 10 + } + + :param cluster_name: + :return: + """ + json_data = request.get_json(force=True) + response = HQService().update_settings(cluster_name, json_data) + return APIResponse(response, HTTP_Status.OK, None) + + @request_wrapper + def delete(self, cluster_name): + response = HQService().delete_settings(cluster_name) + return APIResponse(response, HTTP_Status.OK, None) + + +api.add_resource(HQClusterSettings, '/hq//_settings', endpoint='hq_cluster_settings', + methods=['GET', 'PUT', 'DELETE']) diff --git a/elastichq/common/utils.py b/elastichq/common/utils.py index 6f4c09d5..801c7ccb 100644 --- a/elastichq/common/utils.py +++ b/elastichq/common/utils.py @@ -21,3 +21,15 @@ def string_to_bool(value): def get_key_from_dict(key, dict): return jmespath.search(key, dict) + + +def merge_two_dicts(x, y): + """ + https://stackoverflow.com/a/26853961/831697 + :param x: + :param y: y-values will replace x-values. + :return: + """ + z = x.copy() # start with x's keys and values + z.update(y) # modifies z with y's keys and values & returns None + return z diff --git a/elastichq/config/settings.py b/elastichq/config/settings.py index c4e8953e..77bd18d4 100644 --- a/elastichq/config/settings.py +++ b/elastichq/config/settings.py @@ -1,7 +1,7 @@ import json import os from functools import lru_cache -from datetime import datetime + from apscheduler.jobstores.memory import MemoryJobStore __author__ = 'wmcginnis' @@ -53,6 +53,18 @@ class TestSettings(BaseSettings): _scheduler_api_enabled = False _sqlalchemy_track_modifications = False + # cluster settings + HQ_CLUSTER_SETTINGS = { + 'doc_id': 'hqsettings', + 'index_name': '.elastichq', + 'version': 1, + 'doc_type': 'data', + 'store_metrics': True, + 'websocket_interval': 5, + 'historic_poll_interval': 60, + 'historic_days_to_store': 7 + } + # static HQ_SITE_URL = 'http://elastichq.org' HQ_GH_URL = 'https://github.com/ElasticHQ/elasticsearch-HQ' @@ -119,6 +131,18 @@ class ProdSettings(BaseSettings): API_VERSION = '3.3.0' SERVER_NAME = None + # cluster settings: specific settings for each cluster and how HQ should handle it. + HQ_CLUSTER_SETTINGS = { + 'doc_id': 'hqsettings', + 'index_name': '.elastichq', + 'version': 1, + 'doc_type': 'data', + 'store_metrics': True, # whether to store metrics for this cluster + 'websocket_interval': 5, # seconds + 'historic_poll_interval': 60 * 5, # seconds + 'historic_days_to_store': 7 # num days to keep historical metrics data + } + SCHEDULER_EXECUTORS = { 'default': {'type': 'threadpool', 'max_workers': 20} } diff --git a/elastichq/globals.py b/elastichq/globals.py index 51d9d18c..1dccd34c 100644 --- a/elastichq/globals.py +++ b/elastichq/globals.py @@ -8,6 +8,7 @@ from flask_migrate import Migrate from flask_socketio import SocketIO from flask_sqlalchemy import SQLAlchemy + from elastichq.common.TaskPool import TaskPool from .config import settings from .vendor.elasticsearch.connections import Connections @@ -22,6 +23,7 @@ socketio = SocketIO() taskPool = TaskPool() + def init_marshmallow(app): ma.init_app(app) @@ -70,9 +72,21 @@ def init_socketio(app): return socketio + def init_task_pool(socketio): taskPool.init_app(socketio) + +# +# def init_cache(app): +# """ +# https://pythonhosted.org/Flask-Caching/#configuring-flask-caching +# :param app: +# :return: +# """ +# cache.init_app(app, config={'CACHE_TYPE': 'simple'}) + + LOG = logging.getLogger('elastichq') # Global configurations loaded from setting file @@ -84,5 +98,5 @@ def init_task_pool(socketio): # TODO: This has to be persisted and made configurable REQUEST_TIMEOUT = 30 - - +# Cache +# cache = Cache(config={'CACHE_TYPE': 'simple'}) diff --git a/elastichq/service/HQService.py b/elastichq/service/HQService.py index 7c4f86ef..ee473fec 100644 --- a/elastichq/service/HQService.py +++ b/elastichq/service/HQService.py @@ -6,6 +6,8 @@ from elastichq.model import ClusterDTO from elastichq.service import ClusterService +from elastichq.vendor.elasticsearch.exceptions import NotFoundError +from .ConnectionService import ConnectionService from ..globals import LOG @@ -37,3 +39,60 @@ def get_status(self): "default_url": os.environ.get('HQ_DEFAULT_URL', current_app.config.get('DEFAULT_URL')) } return status + + def get_settings(self, cluster_name): + + try: + connection = ConnectionService().get_connection(cluster_name) + settings_doc = connection.get_source(index=current_app.config.get('HQ_CLUSTER_SETTINGS')[ + 'index_name'], + id=current_app.config.get('HQ_CLUSTER_SETTINGS')[ + 'doc_id'], + doc_type=current_app.config.get('HQ_CLUSTER_SETTINGS')[ + 'doc_type']) + + return settings_doc + except NotFoundError as nfe: + if current_app.config.get('HQ_CLUSTER_SETTINGS')['store_metrics']: + self.save_settings(cluster_name) + + return current_app.config.get('HQ_CLUSTER_SETTINGS') + + def save_settings(self, cluster_name, body=None): + if body is None: + body = current_app.config.get('HQ_CLUSTER_SETTINGS') + connection = ConnectionService().get_connection(cluster_name) + connection.index(index=current_app.config.get('HQ_CLUSTER_SETTINGS')['index_name'], + doc_type=current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_type'], + id=current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_id'], + body=body, refresh=True) + + def update_settings(self, cluster_name, body=None): + if body is None: + body = current_app.config.get('HQ_CLUSTER_SETTINGS') + + current_settings = self.get_settings(cluster_name) + new_settings = { + 'doc_id': current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_id'], + 'index_name': current_app.config.get('HQ_CLUSTER_SETTINGS')['index_name'], + 'version': 1, + 'doc_type': current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_type'], + 'store_metrics': body.get('store_metrics', current_settings.get('store_metrics')), + 'websocket_interval': body.get('websocket_interval', + current_settings.get('websocket_interval')), + 'historic_poll_interval': body.get('historic_poll_interval', + current_settings.get('historic_poll_interval')), + 'historic_days_to_store': body.get('historic_days_to_store', + current_settings.get('historic_days_to_store')) + } + + connection = ConnectionService().get_connection(cluster_name) + connection.update(index=current_app.config.get('HQ_CLUSTER_SETTINGS')['index_name'], + doc_type=current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_type'], + id=current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_id'], + body={"doc": new_settings}, refresh=True) + return new_settings + + def delete_settings(self, cluster_name): + connection = ConnectionService().get_connection(cluster_name) + return connection.indices.delete(index=current_app.config.get('HQ_CLUSTER_SETTINGS')['index_name']) diff --git a/elastichq/service/QueryService.py b/elastichq/service/QueryService.py index 643c6cb9..e4d1d4ea 100644 --- a/elastichq/service/QueryService.py +++ b/elastichq/service/QueryService.py @@ -35,3 +35,11 @@ def run_query(self, cluster_name, index_name, query_json): es_results = search.execute() return es_results.to_dict() + + def get_by_id(self, cluster_name, index_name, doc_id, doc_type='_all'): + connection = ConnectionService().get_connection(cluster_name) + return connection.get(index_name, doc_type=doc_type, id=doc_id) + + def get_source_by_id(self, cluster_name, index_name, doc_id, doc_type='_all'): + connection = ConnectionService().get_connection(cluster_name) + return connection.get_source(index_name, doc_type=doc_type, id=doc_id) diff --git a/elastichq/service/__init__.py b/elastichq/service/__init__.py index 9150e92b..cdfa13c8 100644 --- a/elastichq/service/__init__.py +++ b/elastichq/service/__init__.py @@ -5,6 +5,6 @@ from elastichq.service.IndicesService import * from elastichq.service.NodeService import * from elastichq.service.DiagnosticsService import * +from elastichq.service.QueryService import * from elastichq.service.HQService import * from elastichq.service.SnapshotService import * -from elastichq.service.QueryService import * diff --git a/requirements.txt b/requirements.txt index aa7c3871..2ac6fd36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,13 @@ flask-marshmallow==0.8.0 marshmallow-sqlalchemy==0.13.2 marshmallow==3.0.0b5 - +# requests urllib3<1.23,>=1.21.1 requests>=2.0.0, <3.0.0 +# Cache +#Flask-Caching==1.3.3 + # Database SQLAlchemy==1.2.0 Flask-SQLAlchemy==2.3.2 From 32144b0d1b4b7198214142cdcb854fb27c118eaf Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Thu, 12 Apr 2018 14:10:42 -0400 Subject: [PATCH 04/36] Fixes #367 --- elastichq/__init__.py | 2 +- elastichq/config/logger.json | 4 ++-- elastichq/config/settings.py | 16 ++++++++++++++ elastichq/globals.py | 38 +++++++++++++++++++++++++--------- elastichq/model/Task.py | 5 +++-- elastichq/service/HQService.py | 6 +++++- requirements.txt | 2 +- 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/elastichq/__init__.py b/elastichq/__init__.py index 888166ae..439b7b37 100644 --- a/elastichq/__init__.py +++ b/elastichq/__init__.py @@ -3,7 +3,7 @@ # noinspection PyUnresolvedReferences from elastichq.api import api_blueprint, endpoints, public_blueprint, ws_blueprint from elastichq.config.settings import ProdSettings, TestSettings -from elastichq.globals import init_database, init_log, init_marshmallow, init_socketio, init_task_pool +from elastichq.globals import init_cache, init_database, init_log, init_marshmallow, init_socketio, init_task_pool __author__ = 'royrusso' diff --git a/elastichq/config/logger.json b/elastichq/config/logger.json index 88ab1e28..9f7bacba 100644 --- a/elastichq/config/logger.json +++ b/elastichq/config/logger.json @@ -56,13 +56,13 @@ "elasticsearch": { "level": "DEBUG", "handlers": [], - "qualname": "requests", + "qualname": "elasticsearch", "propagate": "no" }, "elasticsearch.trace": { "level": "INFO", "handlers": [], - "qualname": "requests", + "qualname": "elasticsearch", "propagate": "no" }, "urllib3.connectionpool": { diff --git a/elastichq/config/settings.py b/elastichq/config/settings.py index 77bd18d4..d1721074 100644 --- a/elastichq/config/settings.py +++ b/elastichq/config/settings.py @@ -143,6 +143,22 @@ class ProdSettings(BaseSettings): 'historic_days_to_store': 7 # num days to keep historical metrics data } + DEFAULT_CACHE_BACKEND = "dogpile.cache.memory" + DEFAULT_CACHE_EXPIRE_TIME = 60 * 60 * 2 + DEFAULT_CACHE_ARGUMENTS = { + 'distributed_lock': True + } + + REDIS_CACHE_CONFIG = { + "cache.local.backend": "dogpile.cache.redis", + "cache.local.expiration_time": 3600, + "cache.local.arguments.host": 'localhost', + "cache.local.arguments.port": 6379, + "cache.local.arguments.db": 0, + "cache.local.arguments.redis_expiration_time": 3600, + "cache.local.arguments.distributed_lock": True + } + SCHEDULER_EXECUTORS = { 'default': {'type': 'threadpool', 'max_workers': 20} } diff --git a/elastichq/globals.py b/elastichq/globals.py index 1dccd34c..9e6c907a 100644 --- a/elastichq/globals.py +++ b/elastichq/globals.py @@ -3,6 +3,8 @@ import logging.config import os +from dogpile.cache import make_region +from dogpile.cache.proxy import ProxyBackend from flask_apscheduler import APScheduler from flask_marshmallow import Marshmallow from flask_migrate import Migrate @@ -77,14 +79,31 @@ def init_task_pool(socketio): taskPool.init_app(socketio) -# -# def init_cache(app): -# """ -# https://pythonhosted.org/Flask-Caching/#configuring-flask-caching -# :param app: -# :return: -# """ -# cache.init_app(app, config={'CACHE_TYPE': 'simple'}) +class CacheLoggingProxy(ProxyBackend): + def set(self, key, value): + LOG.info('Setting Cache Key: %s' % key) + self.proxied.set(key, value) + + def get(self, key): + LOG.info('Getting Cache Key: %s' % key) + return self.proxied.get(key) + + def delete(self, key): + LOG.info('Deleting Cache Key: %s' % key) + return self.proxied.delete(key) + + +def init_cache(): + # TODO: make env configurable, for testing. Will likely require us to set an ENV when running tests. + # default + CACHE_REGION = make_region().configure( + CONFIG.ProdSettings.DEFAULT_CACHE_BACKEND, + expiration_time=CONFIG.ProdSettings.DEFAULT_CACHE_EXPIRE_TIME, + arguments={ + 'distributed_lock': CONFIG.ProdSettings.DEFAULT_CACHE_ARGUMENTS['distributed_lock'] + }, wrap=[CacheLoggingProxy] + ) + return CACHE_REGION LOG = logging.getLogger('elastichq') @@ -98,5 +117,4 @@ def init_task_pool(socketio): # TODO: This has to be persisted and made configurable REQUEST_TIMEOUT = 30 -# Cache -# cache = Cache(config={'CACHE_TYPE': 'simple'}) +CACHE_REGION = init_cache() diff --git a/elastichq/model/Task.py b/elastichq/model/Task.py index 02856c7b..6538f2ec 100644 --- a/elastichq/model/Task.py +++ b/elastichq/model/Task.py @@ -4,7 +4,7 @@ import eventlet import jmespath -from elastichq.service import NodeService +from elastichq.service import NodeService, HQService from ..globals import LOG, socketio eventlet.monkey_patch() @@ -30,6 +30,7 @@ def __init__(self, room_name, cluster_name, metric): self.room_name = room_name self.cluster_name = cluster_name self.metric = metric + self.loop_delay = HQService().get_settings(self.cluster_name).get('websocket_interval', 5) def remove_session(self, session_id): self.sessions.remove(session_id) @@ -59,7 +60,7 @@ def run(self): self.stop() if loop_count > 5: - eventlet.sleep(5) + eventlet.sleep(self.loop_delay) LOG.debug('-----------------------------------------') LOG.debug(' Doing background task') diff --git a/elastichq/service/HQService.py b/elastichq/service/HQService.py index ee473fec..88635a72 100644 --- a/elastichq/service/HQService.py +++ b/elastichq/service/HQService.py @@ -8,7 +8,7 @@ from elastichq.service import ClusterService from elastichq.vendor.elasticsearch.exceptions import NotFoundError from .ConnectionService import ConnectionService -from ..globals import LOG +from ..globals import CACHE_REGION, LOG class HQService: @@ -40,6 +40,7 @@ def get_status(self): } return status + @CACHE_REGION.cache_on_arguments() def get_settings(self, cluster_name): try: @@ -91,6 +92,9 @@ def update_settings(self, cluster_name, body=None): doc_type=current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_type'], id=current_app.config.get('HQ_CLUSTER_SETTINGS')['doc_id'], body={"doc": new_settings}, refresh=True) + + self.get_settings.invalidate(self, cluster_name) # alter cache + return new_settings def delete_settings(self, cluster_name): diff --git a/requirements.txt b/requirements.txt index 2ac6fd36..69f9d0c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ urllib3<1.23,>=1.21.1 requests>=2.0.0, <3.0.0 # Cache -#Flask-Caching==1.3.3 +dogpile.cache == 0.6.5 # Database SQLAlchemy==1.2.0 From c8552d8f3b06257a1d9b3bb8a6f2f5d7fc6ef31c Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 16 Apr 2018 10:55:58 -0400 Subject: [PATCH 05/36] Fixes #372 --- elastichq/model/Task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastichq/model/Task.py b/elastichq/model/Task.py index 6538f2ec..4c654175 100644 --- a/elastichq/model/Task.py +++ b/elastichq/model/Task.py @@ -76,8 +76,8 @@ def run(self): for node_id in node_ids: node_dict = node_stats['nodes'][node_id] - available_in_bytes = jmespath.search("fs.data[0].available_in_bytes", node_dict) - total_in_bytes = jmespath.search("fs.data[0].total_in_bytes", node_dict) + available_in_bytes = jmespath.search("fs.data[0].available_in_bytes", node_dict) or 0 + total_in_bytes = jmespath.search("fs.data[0].total_in_bytes", node_dict) or 0 node = \ { From 7e963129b1b7ff95f5e38826ff9155cc5003831b Mon Sep 17 00:00:00 2001 From: Peter Casanova Date: Mon, 16 Apr 2018 13:31:23 -0400 Subject: [PATCH 06/36] UI to edit Cluster Settings for Elastic HQ --- .../clusters/clusters.controller.js | 47 +++++++++++++- .../components/clusters/clusters.style.scss | 14 ++++ .../clusters/clusters.template.html | 6 +- .../clusters/edit-cluster-modal.html | 64 +++++++++++++++++++ ui/src/services/common-services.js | 2 + ui/src/services/hq/hq.service.js | 27 ++++++++ ui/src/services/hq/hq.spec.js | 7 ++ 7 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 ui/src/components/clusters/edit-cluster-modal.html create mode 100644 ui/src/services/hq/hq.service.js create mode 100644 ui/src/services/hq/hq.spec.js diff --git a/ui/src/components/clusters/clusters.controller.js b/ui/src/components/clusters/clusters.controller.js index 295017dd..3162f544 100644 --- a/ui/src/components/clusters/clusters.controller.js +++ b/ui/src/components/clusters/clusters.controller.js @@ -1,15 +1,17 @@ import './clusters.style.scss'; import addClusterModal from './add-cluster-modal.html'; +import editClusterModal from './edit-cluster-modal.html'; import _ from 'lodash'; import numeral from 'numeral'; class clustersController { - constructor(ClusterConnection, Notification, $state, $sce, $filter, $rootScope, $uibModal) { + constructor(ClusterConnection, Hq, Notification, $state, $sce, $filter, $rootScope, $uibModal) { 'ngInject'; this.service = ClusterConnection; + this.hq = Hq; this.Notification = Notification; this.$state = $state; @@ -148,6 +150,49 @@ class clustersController { }); } + editCluster(cluster) { + console.log('--- cluster to edit: ', cluster) + this.hq.settings(cluster.cluster_name).then((resp) => { + console.log('---- cluster settings; ', resp.data) + this.editModal(cluster, resp.data.data[0]) + }) + // editClusterModal + } + + editModal(cluster, settings) { + const modalInstance = this.$uibModal.open({ + template: editClusterModal, + size: 'lg', + controller: ($scope, $uibModalInstance, settings) => { + 'ngInject'; + + // After you pass in the resolver, below, attache it for reference + $uibModalInstance = $uibModalInstance; + + + $scope.formData = settings; + $scope.cancel = () => { + $uibModalInstance.dismiss('close'); + }; + + $scope.save = () => { + this.hq.updateSettings(cluster.cluster_name, $scope.formData).then((resp) => { + this.Notification.success({message: 'Cluster settings with Elastic HQ successfully updated', delay: 3000}); + $uibModalInstance.close('Closed'); + }, (err) => { + this.Notification.error({message: 'Error saving settings'}); + console.log(err) + }) + } + + }, + resolve: { + settings: () => {return settings;} + } + + }); + } + parseURI(connectionUri) { let uri = new URL(connectionUri); let tmp = { diff --git a/ui/src/components/clusters/clusters.style.scss b/ui/src/components/clusters/clusters.style.scss index e93dbdbf..5cb900c2 100644 --- a/ui/src/components/clusters/clusters.style.scss +++ b/ui/src/components/clusters/clusters.style.scss @@ -29,4 +29,18 @@ } } } +} + +.edit-cluster-settings-modal { + .checkbox { + margin-top: 0; + label { + display: flex; + align-items: center; + } + } + label, .checkbox label { + font-size: 14px; + font-weight: bold; + } } \ No newline at end of file diff --git a/ui/src/components/clusters/clusters.template.html b/ui/src/components/clusters/clusters.template.html index 93558a50..0e9ae2a2 100644 --- a/ui/src/components/clusters/clusters.template.html +++ b/ui/src/components/clusters/clusters.template.html @@ -28,7 +28,11 @@ - Edit + diff --git a/ui/src/components/clusters/edit-cluster-modal.html b/ui/src/components/clusters/edit-cluster-modal.html new file mode 100644 index 00000000..bbdb24f9 --- /dev/null +++ b/ui/src/components/clusters/edit-cluster-modal.html @@ -0,0 +1,64 @@ +
+
+ +
+
+
+
+
+
+
+
+ +
+
+
+
+
+ + + How many days to store before data is discarded. +
+
+
+
+ + + How often to poll in seconds. i.e. For every 5 min, then enter 300. +
+
+
+
+
+
+ + + How often to poll in seconds when getting live stream of data. +
+
+
+ +
+
+
+
+ + +
+
\ No newline at end of file diff --git a/ui/src/services/common-services.js b/ui/src/services/common-services.js index f153da1a..76091add 100644 --- a/ui/src/services/common-services.js +++ b/ui/src/services/common-services.js @@ -3,6 +3,7 @@ import ClusterIndices from './cluster-indices/cluster-indices.service'; import ClusterNodes from './cluster-nodes/cluster-nodes.service'; import ClusterAliases from './cluster-aliases/cluster-aliases.service'; import ClusterRepositories from './cluster-repositories/cluster-repositories.service'; +import Hq from './hq/hq.service'; const CommonServices = angular.module('common', []) .service('ClusterConnection', ClusterConnection) @@ -10,6 +11,7 @@ const CommonServices = angular.module('common', []) .service('ClusterAliases', ClusterAliases) .service('ClusterNodes', ClusterNodes) .service('ClusterRepositories', ClusterRepositories) + .service('Hq', Hq) .name; export default CommonServices; \ No newline at end of file diff --git a/ui/src/services/hq/hq.service.js b/ui/src/services/hq/hq.service.js new file mode 100644 index 00000000..f818a9bb --- /dev/null +++ b/ui/src/services/hq/hq.service.js @@ -0,0 +1,27 @@ +class HqService { + + // Imports go here + constructor(QueuedFactory) { + 'ngInject'; + + this.que = QueuedFactory; + } + + + settings(cluster_name) { + return this.que.add({ + url: ('api/hq/' + cluster_name + '/_settings'), + method: 'GET' + }); + } + + updateSettings(cluster_name, data) { + return this.que.add({ + url: ('api/hq/' + cluster_name + '/_settings'), + method: 'PUT', + data: data + }); + } +} + +export default HqService; diff --git a/ui/src/services/hq/hq.spec.js b/ui/src/services/hq/hq.spec.js new file mode 100644 index 00000000..319f155b --- /dev/null +++ b/ui/src/services/hq/hq.spec.js @@ -0,0 +1,7 @@ + +// Stubbed test. +describe('hq Service', () => { + it('base test', () => { + expect(1).toEqual(1); + }); +}); From 66a8c0eb99a073f2ef67de1fdee3a9aa113ccf30 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 16 Apr 2018 13:36:42 -0400 Subject: [PATCH 07/36] -- --- application.py | 7 +++- elastichq/__init__.py | 8 +++-- elastichq/config/settings.py | 35 ++++++++++++------ elastichq/factory.py | 50 -------------------------- elastichq/globals.py | 56 ++++++++++++++++++++++++++--- elastichq/model/ClusterModel.py | 10 ++---- elastichq/service/ClusterService.py | 2 ++ elastichq/service/HQService.py | 2 +- 8 files changed, 93 insertions(+), 77 deletions(-) delete mode 100644 elastichq/factory.py diff --git a/application.py b/application.py index fe321778..f05738c2 100644 --- a/application.py +++ b/application.py @@ -9,6 +9,7 @@ default_port = 5000 default_debug = False default_url = 'http://localhost:9200' +is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "") application = create_app() @@ -33,4 +34,8 @@ # set default url, override with env for docker application.config['DEFAULT_URL'] = os.environ.get('HQ_DEFAULT_URL', options.url) - socketio.run(application, host=options.host, port=options.port, debug=options.debug) + if is_gunicorn: + # we set reloader False so gunicorn doesn't call two instances of all the Flask init functions. + socketio.run(application, host=options.host, port=options.port, debug=options.debug, use_reloader=False) + else: + socketio.run(application, host=options.host, port=options.port, debug=options.debug) diff --git a/elastichq/__init__.py b/elastichq/__init__.py index 439b7b37..633386aa 100644 --- a/elastichq/__init__.py +++ b/elastichq/__init__.py @@ -1,9 +1,10 @@ from flask import Flask +import os # noinspection PyUnresolvedReferences from elastichq.api import api_blueprint, endpoints, public_blueprint, ws_blueprint from elastichq.config.settings import ProdSettings, TestSettings -from elastichq.globals import init_cache, init_database, init_log, init_marshmallow, init_socketio, init_task_pool +from elastichq.globals import init_cache, init_database, init_log, init_marshmallow, init_socketio, init_task_pool, init_scheduler, init_connections __author__ = 'royrusso' @@ -34,10 +35,13 @@ def create_app(env='PROD'): init_marshmallow(app) - # init_scheduler(app) + # TODO: For now as assume always in debug mode, so it doesn't execute the scheduler twice. + init_scheduler(app, True) socketio = init_socketio(app) init_task_pool(socketio) + init_connections(True) + return app diff --git a/elastichq/config/settings.py b/elastichq/config/settings.py index d1721074..512995c1 100644 --- a/elastichq/config/settings.py +++ b/elastichq/config/settings.py @@ -53,6 +53,13 @@ class TestSettings(BaseSettings): _scheduler_api_enabled = False _sqlalchemy_track_modifications = False + # CACHE + DEFAULT_CACHE_BACKEND = "dogpile.cache.memory" + DEFAULT_CACHE_EXPIRE_TIME = 60 * 60 * 2 + DEFAULT_CACHE_ARGUMENTS = { + 'distributed_lock': True + } + # cluster settings HQ_CLUSTER_SETTINGS = { 'doc_id': 'hqsettings', @@ -68,7 +75,7 @@ class TestSettings(BaseSettings): # static HQ_SITE_URL = 'http://elastichq.org' HQ_GH_URL = 'https://github.com/ElasticHQ/elasticsearch-HQ' - API_VERSION = 'v3.3.0' + API_VERSION = 'v3.5.0' ES_V2_HOST = '127.0.0.1' ES_V2_PORT = '9200' ES_V5_HOST = '127.0.0.1' @@ -128,7 +135,7 @@ class ProdSettings(BaseSettings): # static HQ_SITE_URL = 'http://elastichq.org' HQ_GH_URL = 'https://github.com/ElasticHQ/elasticsearch-HQ' - API_VERSION = '3.3.0' + API_VERSION = '3.5.0' SERVER_NAME = None # cluster settings: specific settings for each cluster and how HQ should handle it. @@ -143,6 +150,7 @@ class ProdSettings(BaseSettings): 'historic_days_to_store': 7 # num days to keep historical metrics data } + # CACHE DEFAULT_CACHE_BACKEND = "dogpile.cache.memory" DEFAULT_CACHE_EXPIRE_TIME = 60 * 60 * 2 DEFAULT_CACHE_ARGUMENTS = { @@ -159,16 +167,23 @@ class ProdSettings(BaseSettings): "cache.local.arguments.distributed_lock": True } - SCHEDULER_EXECUTORS = { - 'default': {'type': 'threadpool', 'max_workers': 20} + # METRICS + METRICS_INDEX_NAME = '.elastichq_metrics' + + SCHEDULER_JOBSTORES = { + 'default': MemoryJobStore() } + # SCHEDULER_JOBSTORES = { + # 'default': SQLAlchemyJobStore(url='sqlite:///flask_context.db') + # } - SCHEDULER_JOB_DEFAULTS = { - 'coalesce': False, - 'max_instances': 3 + SCHEDULER_EXECUTORS = { + 'default': {'type': 'threadpool', 'max_workers': 10} } + # SCHEDULER_JOB_DEFAULTS = { + # 'coalesce': False, + # 'max_instances': 3 + # } + SCHEDULER_API_ENABLED = True - SCHEDULER_JOBSTORES = { - 'default': MemoryJobStore() - } diff --git a/elastichq/factory.py b/elastichq/factory.py deleted file mode 100644 index 472ba710..00000000 --- a/elastichq/factory.py +++ /dev/null @@ -1,50 +0,0 @@ -# import os -# from flask import Flask -# from elastichq.api import api_blueprint, public_blueprint, ws_blueprint -# from elastichq.globals import init_log, init_database, init_marshmallow, init_socketio, init_task_pool -# from elastichq.config.settings import ProdSettings, TestSettings -# -# # noinspection PyUnresolvedReferences -# from elastichq.api import endpoints -# -# -# __author__ = 'royrusso' -# -# -# def create_app(env='PROD', port=5000, host='0.0.0.0', debug=True): -# app = Flask(__name__) -# -# if env.lower() == 'prod': -# app.config.from_object(ProdSettings()) -# elif env.lower() == 'test': -# app.config.from_object(TestSettings()) -# else: -# raise ValueError('Unknown environment: %s' % (env, )) -# -# init_log() -# -# app.register_blueprint(api_blueprint) -# app.register_blueprint(public_blueprint) -# app.register_blueprint(ws_blueprint) -# -# app.jinja_env.auto_reload = True -# app.config['TEMPLATES_AUTO_RELOAD'] = True -# -# # Stop the app from initializing twice in debug mode. -# # if not app.debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true": -# # The app is not in debug mode or we are in the reloaded process -# init_database(app, tests=env.lower() == 'test') -# -# init_marshmallow(app) -# -# #init_scheduler(app) -# -# socketio = init_socketio(app) -# -# init_task_pool(socketio) -# -# socketio.run(app, port=port, host=host, debug=debug) -# -# #socketio.run(app, port=port, host=host) -# -# return app diff --git a/elastichq/globals.py b/elastichq/globals.py index 9e6c907a..ddcf6f36 100644 --- a/elastichq/globals.py +++ b/elastichq/globals.py @@ -3,6 +3,7 @@ import logging.config import os +from apscheduler.schedulers.background import BackgroundScheduler from dogpile.cache import make_region from dogpile.cache.proxy import ProxyBackend from flask_apscheduler import APScheduler @@ -20,7 +21,7 @@ db = SQLAlchemy() ma = Marshmallow() migrate = Migrate() -scheduler = APScheduler() +scheduler = APScheduler(BackgroundScheduler()) socketio = SocketIO() taskPool = TaskPool() @@ -58,10 +59,55 @@ def migrate_db(app): pass -# -# def init_scheduler(app): -# scheduler.init_app(app) -# scheduler.start() +def init_connections(debug=True): + """ + Inits connections to all of the configured clusters. + :return: + """ + from elastichq.service import ClusterService + is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "") + if is_gunicorn: + ClusterService().get_clusters() + else: + if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': + ClusterService().get_clusters() + + +def init_scheduler(app, debug=True): + """ + Two criteria here... 1/ with gunicorn we explicitly add a job, as this function will only be called once because use_reloader=False. + With wsgi, we have to filter out the second call, so we don't create the same job twice. + :param app: + :param debug: assume true so we don't start the same job twice. + :return: + """ + is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "") + if is_gunicorn: + scheduler.init_app(app) + scheduler.start() + JOB = { + 'trigger': 'interval', + 'seconds': 10, + 'args': (app, 'in') + } + scheduler.add_job('job1', the_job, **JOB) + else: + if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': + scheduler.init_app(app) + scheduler.start() + JOB = { + 'trigger': 'interval', + 'seconds': 10, + 'args': (app, 'in') + } + scheduler.add_job('job1', the_job, **JOB) + + +def the_job(app, foo): + with app.app_context(): + # HQService().get_status() + LOG.info('a') + def init_socketio(app): # Set this variable to "threading", "eventlet" or "gevent" to test the diff --git a/elastichq/model/ClusterModel.py b/elastichq/model/ClusterModel.py index f36c37a4..c8fc25d8 100644 --- a/elastichq/model/ClusterModel.py +++ b/elastichq/model/ClusterModel.py @@ -22,6 +22,7 @@ class ClusterModel(db.Model): cluster_password = db.Column(db.String, nullable=True) cluster_connected = False cluster_health = None + cluster_settings = None def __init__(self, cluster_name, cluster_ip, cluster_port='9200', cluster_scheme='http', username=None, password=None): @@ -61,16 +62,9 @@ class ClusterDTO(ma.ModelSchema): Generic data transfer object for a cluster. """ - _links = ma.Hyperlinks({ - 'summary': ma.AbsoluteURLFor('.clusters_summary', cluster_name='', _external=True), - 'stats': ma.AbsoluteURLFor('.clusters_stats', cluster_name='', _external=True), - 'health': ma.AbsoluteURLFor('.clusters_health', cluster_name='', _external=True), - 'collection': ma.AbsoluteURLFor('.clusters_list') - }) - class Meta: ordered = True model = ClusterModel fields = ( 'cluster_name', 'cluster_ip', 'cluster_port', 'cluster_scheme', 'cluster_connected', 'cluster_host', - 'cluster_version', 'cluster_health', '_links') + 'cluster_version', 'cluster_health', 'cluster_settings') diff --git a/elastichq/service/ClusterService.py b/elastichq/service/ClusterService.py index 8e65562b..8fa3226d 100644 --- a/elastichq/service/ClusterService.py +++ b/elastichq/service/ClusterService.py @@ -8,6 +8,7 @@ from elastichq.globals import REQUEST_TIMEOUT from .ConnectionService import ConnectionService +from .HQService import HQService from .NodeService import NodeService @@ -49,6 +50,7 @@ def get_clusters(self, create_if_missing=True): for cluster in clusters: if cluster.cluster_connected is True: cluster.cluster_health = self.get_cluster_health(cluster_name=cluster.cluster_name) + cluster.cluster_settings = HQService().get_settings(cluster.cluster_name) return clusters def get_cluster_summary(self, cluster_name): diff --git a/elastichq/service/HQService.py b/elastichq/service/HQService.py index 88635a72..432388d3 100644 --- a/elastichq/service/HQService.py +++ b/elastichq/service/HQService.py @@ -5,7 +5,6 @@ from flask import current_app from elastichq.model import ClusterDTO -from elastichq.service import ClusterService from elastichq.vendor.elasticsearch.exceptions import NotFoundError from .ConnectionService import ConnectionService from ..globals import CACHE_REGION, LOG @@ -26,6 +25,7 @@ def get_status(self): stable_version = (json.loads(version_str)).get("version", None) + from elastichq.service import ClusterService clusters = ClusterService().get_clusters(create_if_missing=False) schema = ClusterDTO(many=True) result = schema.dump(clusters) From eb2d84d48b873e8b94002c939faa0cef1d98e46a Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 16 Apr 2018 14:04:57 -0400 Subject: [PATCH 08/36] - delete settings - expire cache --- elastichq/service/HQService.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastichq/service/HQService.py b/elastichq/service/HQService.py index 432388d3..3dcf1c52 100644 --- a/elastichq/service/HQService.py +++ b/elastichq/service/HQService.py @@ -99,4 +99,5 @@ def update_settings(self, cluster_name, body=None): def delete_settings(self, cluster_name): connection = ConnectionService().get_connection(cluster_name) + self.get_settings.invalidate(self, cluster_name) # alter cache return connection.indices.delete(index=current_app.config.get('HQ_CLUSTER_SETTINGS')['index_name']) From 27c43ab801fa620b921dd00e327a857de8e8535a Mon Sep 17 00:00:00 2001 From: Peter Casanova Date: Mon, 16 Apr 2018 14:05:08 -0400 Subject: [PATCH 09/36] Add Reset Cluster Settings for Elastic HQ --- ui/src/components/clusters/clusters.controller.js | 13 +++++++++++-- ui/src/components/clusters/clusters.template.html | 8 ++++++-- ui/src/services/hq/hq.service.js | 7 +++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ui/src/components/clusters/clusters.controller.js b/ui/src/components/clusters/clusters.controller.js index 3162f544..7358e2b7 100644 --- a/ui/src/components/clusters/clusters.controller.js +++ b/ui/src/components/clusters/clusters.controller.js @@ -150,10 +150,19 @@ class clustersController { }); } + resetCluster(cluster) { + this.hq.resetSettings(cluster.cluster_name).then((resp) => { + let msg = `Cluster settings for "${cluster.cluster_name}" has been reset.` + this.Notification.success({message: msg, delay: 3000}); + this.editModal(cluster, resp.data.data[0]) + }, (err) => { + this.Notification.error({message: 'Error restting settings'}); + console.log(err) + }) + } + editCluster(cluster) { - console.log('--- cluster to edit: ', cluster) this.hq.settings(cluster.cluster_name).then((resp) => { - console.log('---- cluster settings; ', resp.data) this.editModal(cluster, resp.data.data[0]) }) // editClusterModal diff --git a/ui/src/components/clusters/clusters.template.html b/ui/src/components/clusters/clusters.template.html index 0e9ae2a2..0c019fd1 100644 --- a/ui/src/components/clusters/clusters.template.html +++ b/ui/src/components/clusters/clusters.template.html @@ -28,11 +28,15 @@ - Edit - Reset + diff --git a/ui/src/services/hq/hq.service.js b/ui/src/services/hq/hq.service.js index f818a9bb..eed2ab3f 100644 --- a/ui/src/services/hq/hq.service.js +++ b/ui/src/services/hq/hq.service.js @@ -15,6 +15,13 @@ class HqService { }); } + resetSettings(cluster_name) { + return this.que.add({ + url: ('api/hq/' + cluster_name + '/_settings'), + method: 'DELETE' + }); + } + updateSettings(cluster_name, data) { return this.que.add({ url: ('api/hq/' + cluster_name + '/_settings'), From 35f3fde5a28c48f314604d34434fb83e9f93aece Mon Sep 17 00:00:00 2001 From: Peter Casanova Date: Mon, 16 Apr 2018 14:08:15 -0400 Subject: [PATCH 10/36] Typo --- ui/src/components/clusters/clusters.controller.js | 1 - ui/src/components/clusters/clusters.template.html | 13 +++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ui/src/components/clusters/clusters.controller.js b/ui/src/components/clusters/clusters.controller.js index 7358e2b7..1222f6e6 100644 --- a/ui/src/components/clusters/clusters.controller.js +++ b/ui/src/components/clusters/clusters.controller.js @@ -154,7 +154,6 @@ class clustersController { this.hq.resetSettings(cluster.cluster_name).then((resp) => { let msg = `Cluster settings for "${cluster.cluster_name}" has been reset.` this.Notification.success({message: msg, delay: 3000}); - this.editModal(cluster, resp.data.data[0]) }, (err) => { this.Notification.error({message: 'Error restting settings'}); console.log(err) diff --git a/ui/src/components/clusters/clusters.template.html b/ui/src/components/clusters/clusters.template.html index 0c019fd1..b0500d0d 100644 --- a/ui/src/components/clusters/clusters.template.html +++ b/ui/src/components/clusters/clusters.template.html @@ -28,18 +28,19 @@ - + + + From 8c142c25a70d02e4cfd0555a77bc68e06c1ae3a3 Mon Sep 17 00:00:00 2001 From: Peter Casanova Date: Mon, 16 Apr 2018 14:14:10 -0400 Subject: [PATCH 11/36] make uibModal stop complaining when modal closes --- ui/src/components/clusters/clusters.controller.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ui/src/components/clusters/clusters.controller.js b/ui/src/components/clusters/clusters.controller.js index 1222f6e6..e4236552 100644 --- a/ui/src/components/clusters/clusters.controller.js +++ b/ui/src/components/clusters/clusters.controller.js @@ -198,6 +198,11 @@ class clustersController { settings: () => {return settings;} } + }).result.then(() => { + + }, (resp) => { + // so uibModal does not complain???? + // https://stackoverflow.com/questions/42416570/how-to-handle-possibly-unhandled-rejection-backdrop-click-in-a-general-way#answer-43797839 }); } From 8dee9d96e4afbc9988999e89c6280f7360e764ad Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 16 Apr 2018 17:09:08 -0400 Subject: [PATCH 12/36] -- --- elastichq/__init__.py | 5 ++++- elastichq/common/JobPool.py | 13 +++++++++++++ elastichq/globals.py | 29 ++++++++++++++++++++--------- 3 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 elastichq/common/JobPool.py diff --git a/elastichq/__init__.py b/elastichq/__init__.py index 633386aa..b6ea1464 100644 --- a/elastichq/__init__.py +++ b/elastichq/__init__.py @@ -35,6 +35,9 @@ def create_app(env='PROD'): init_marshmallow(app) + # Init connections, or all other startup inits that require active connections, will fail. + init_connections(True) + # TODO: For now as assume always in debug mode, so it doesn't execute the scheduler twice. init_scheduler(app, True) @@ -42,6 +45,6 @@ def create_app(env='PROD'): init_task_pool(socketio) - init_connections(True) + return app diff --git a/elastichq/common/JobPool.py b/elastichq/common/JobPool.py new file mode 100644 index 00000000..350be957 --- /dev/null +++ b/elastichq/common/JobPool.py @@ -0,0 +1,13 @@ + + + +class JobPool(): + + def init_app(self, app): + self.app = app + return self + + def blah(self): + from elastichq.service import ClusterService + clusters = ClusterService().get_clusters(create_if_missing=False) + return clusters diff --git a/elastichq/globals.py b/elastichq/globals.py index ddcf6f36..9c69e5b3 100644 --- a/elastichq/globals.py +++ b/elastichq/globals.py @@ -72,6 +72,15 @@ def init_connections(debug=True): if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': ClusterService().get_clusters() +# +# from flask_apscheduler import APScheduler as _BaseAPScheduler +# +# +# class APScheduler(_BaseAPScheduler): +# def run_job(self, id, jobstore=None): +# with self.app.app_context(): +# super().run_job(id=id, jobstore=jobstore) + def init_scheduler(app, debug=True): """ @@ -85,22 +94,27 @@ def init_scheduler(app, debug=True): if is_gunicorn: scheduler.init_app(app) scheduler.start() + JOB = { 'trigger': 'interval', - 'seconds': 10, - 'args': (app, 'in') + 'seconds': 10 # , + # 'args': (app, 'in') } - scheduler.add_job('job1', the_job, **JOB) + from elastichq.common import JobPool + with app.app_context(): + scheduler.add_job('job1', JobPool.blah, **JOB) else: if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': scheduler.init_app(app) scheduler.start() JOB = { 'trigger': 'interval', - 'seconds': 10, - 'args': (app, 'in') + 'seconds': 10 # , } - scheduler.add_job('job1', the_job, **JOB) + # scheduler.add_job('job1', the_job, **JOB) + from elastichq.common import JobPool + with app.app_context(): + scheduler.add_job('job1', JobPool.blah, **JOB) def the_job(app, foo): @@ -113,11 +127,8 @@ def init_socketio(app): # Set this variable to "threading", "eventlet" or "gevent" to test the # different async modes, or leave it set to None for the application to choose # the best option based on installed packages. - async_mode = 'eventlet' - socketio.init_app(app, async_mode=async_mode, logger=True, engineio_logger=True) - return socketio From 7adf61495ed061c681e7e1aef854a10bcb1c5bd1 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Mon, 16 Apr 2018 17:20:32 -0400 Subject: [PATCH 13/36] Fixes #371 --- elastichq/globals.py | 48 ++++++++++++++--------------- elastichq/service/IndicesService.py | 4 +-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/elastichq/globals.py b/elastichq/globals.py index 9c69e5b3..0d5302ec 100644 --- a/elastichq/globals.py +++ b/elastichq/globals.py @@ -91,30 +91,30 @@ def init_scheduler(app, debug=True): :return: """ is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "") - if is_gunicorn: - scheduler.init_app(app) - scheduler.start() - - JOB = { - 'trigger': 'interval', - 'seconds': 10 # , - # 'args': (app, 'in') - } - from elastichq.common import JobPool - with app.app_context(): - scheduler.add_job('job1', JobPool.blah, **JOB) - else: - if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': - scheduler.init_app(app) - scheduler.start() - JOB = { - 'trigger': 'interval', - 'seconds': 10 # , - } - # scheduler.add_job('job1', the_job, **JOB) - from elastichq.common import JobPool - with app.app_context(): - scheduler.add_job('job1', JobPool.blah, **JOB) + # if is_gunicorn: + # scheduler.init_app(app) + # scheduler.start() + # + # JOB = { + # 'trigger': 'interval', + # 'seconds': 10 # , + # # 'args': (app, 'in') + # } + # from elastichq.common import JobPool + # with app.app_context(): + # scheduler.add_job('job1', JobPool.blah, **JOB) + # else: + # if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': + # scheduler.init_app(app) + # scheduler.start() + # JOB = { + # 'trigger': 'interval', + # 'seconds': 10 # , + # } + # # scheduler.add_job('job1', the_job, **JOB) + # from elastichq.common import JobPool + # with app.app_context(): + # scheduler.add_job('job1', JobPool.blah, **JOB) def the_job(app, foo): diff --git a/elastichq/service/IndicesService.py b/elastichq/service/IndicesService.py index f9e875bd..3740688c 100644 --- a/elastichq/service/IndicesService.py +++ b/elastichq/service/IndicesService.py @@ -112,8 +112,8 @@ def get_indices_summary(self, cluster_name, indices_names=None): index_state = state_indices.get(key) index['settings'] = { - 'number_of_shards': int(jmespath.search("settings.index.number_of_shards", index_state)), - "number_of_replicas": int(jmespath.search("settings.index.number_of_replicas", index_state))} + 'number_of_shards': int(jmespath.search("settings.index.number_of_shards", index_state) or 0), + "number_of_replicas": int(jmespath.search("settings.index.number_of_replicas", index_state) or 0)} index['state'] = index_state.get("state", None) indices.append(index) return sorted(indices, key=lambda k: k['index_name']) From 6b68dca6969489e43e6c942a2db5e8dd90e27387 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Tue, 17 Apr 2018 08:12:09 -0400 Subject: [PATCH 14/36] apscheduler - working --- elastichq/common/JobPool.py | 15 ++++++++-- elastichq/globals.py | 57 ++++++++++++------------------------- 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/elastichq/common/JobPool.py b/elastichq/common/JobPool.py index 350be957..a761ad5a 100644 --- a/elastichq/common/JobPool.py +++ b/elastichq/common/JobPool.py @@ -1,13 +1,24 @@ +from elastichq.globals import scheduler - - +# TODO: rename this to Metrics Service and move to service package class JobPool(): + app = None + def init_app(self, app): self.app = app return self def blah(self): + JOB = { + 'trigger': 'interval', + 'seconds': 3 # , + # 'args': (app, 'in') + } + scheduler.add_job('job1', self.do_task, **JOB) + + + def do_task(self): from elastichq.service import ClusterService clusters = ClusterService().get_clusters(create_if_missing=False) return clusters diff --git a/elastichq/globals.py b/elastichq/globals.py index 0d5302ec..e058bd9a 100644 --- a/elastichq/globals.py +++ b/elastichq/globals.py @@ -42,6 +42,9 @@ def init_log(): def init_database(app, tests=False): + # Added assignment, db.app=app, to stop: "RuntimeError: application not registered on db instance and no application bound to current context" + # http://piotr.banaszkiewicz.org/blog/2012/06/29/flask-sqlalchemy-init_app/ + db.app = app db.init_app(app) app.app_context().push() @@ -72,15 +75,6 @@ def init_connections(debug=True): if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': ClusterService().get_clusters() -# -# from flask_apscheduler import APScheduler as _BaseAPScheduler -# -# -# class APScheduler(_BaseAPScheduler): -# def run_job(self, id, jobstore=None): -# with self.app.app_context(): -# super().run_job(id=id, jobstore=jobstore) - def init_scheduler(app, debug=True): """ @@ -91,36 +85,21 @@ def init_scheduler(app, debug=True): :return: """ is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "") - # if is_gunicorn: - # scheduler.init_app(app) - # scheduler.start() - # - # JOB = { - # 'trigger': 'interval', - # 'seconds': 10 # , - # # 'args': (app, 'in') - # } - # from elastichq.common import JobPool - # with app.app_context(): - # scheduler.add_job('job1', JobPool.blah, **JOB) - # else: - # if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': - # scheduler.init_app(app) - # scheduler.start() - # JOB = { - # 'trigger': 'interval', - # 'seconds': 10 # , - # } - # # scheduler.add_job('job1', the_job, **JOB) - # from elastichq.common import JobPool - # with app.app_context(): - # scheduler.add_job('job1', JobPool.blah, **JOB) - - -def the_job(app, foo): - with app.app_context(): - # HQService().get_status() - LOG.info('a') + if is_gunicorn: + scheduler.init_app(app) + scheduler.start() + + from elastichq.common.JobPool import JobPool + job = JobPool().init_app(app=app) + job.blah() + else: + if not debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': + if not scheduler.running: + scheduler.init_app(app) + scheduler.start() + from elastichq.common.JobPool import JobPool + job = JobPool().init_app(app=app) + job.blah() def init_socketio(app): From 699b3982369b495addac1d673a3d85db92f23056 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Tue, 17 Apr 2018 10:58:59 -0400 Subject: [PATCH 15/36] fixed docker image label --- docs/source/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 98a41eda..eb204989 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -36,7 +36,7 @@ Docker Images Docker images are offered on the `ElasticHQ Dockerhub `_. -The ``latest`` tag deploys the latest stable release. Where ``develop`` is the latest unstable working branch. +The ``master`` tag deploys the latest stable release. Where ``develop`` is the latest unstable working branch. When starting with Docker, see :any:`environment variables` for passing startup args. Environment variables are passed to docker using the `-e` flag. @@ -51,7 +51,7 @@ Pre-release versions are made available as branches in the github repository. We Our branching organization is as follows: * ``master``: contains Latest Stable -* ``develop``: contains latest features and fixes. **Not stable.** +* ``develop``: contains latest features. **Not stable.** * ``#.#.#RC-#``: Release candidates are pre-release versions. **Not stable.** Initial Login From b534c1a466a6ae23405d5cb66db960fdadef8621 Mon Sep 17 00:00:00 2001 From: Roy Russo Date: Wed, 18 Apr 2018 13:26:10 -0400 Subject: [PATCH 16/36] - comment out cluster settings that aren't used atm - comment out scheduler code until next milestone --- docs/source/installation.rst | 4 ++-- elastichq/__init__.py | 2 +- ui/src/components/clusters/edit-cluster-modal.html | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index eb204989..09bcf7c5 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -36,7 +36,7 @@ Docker Images Docker images are offered on the `ElasticHQ Dockerhub `_. -The ``master`` tag deploys the latest stable release. Where ``develop`` is the latest unstable working branch. +The ``latest`` tag deploys the latest stable release. Where ``develop`` is the latest unstable working branch. When starting with Docker, see :any:`environment variables` for passing startup args. Environment variables are passed to docker using the `-e` flag. @@ -50,7 +50,7 @@ Pre-release versions are made available as branches in the github repository. We Our branching organization is as follows: -* ``master``: contains Latest Stable +* ``master``: contains Latest Stable release. * ``develop``: contains latest features. **Not stable.** * ``#.#.#RC-#``: Release candidates are pre-release versions. **Not stable.** diff --git a/elastichq/__init__.py b/elastichq/__init__.py index b6ea1464..56064d5a 100644 --- a/elastichq/__init__.py +++ b/elastichq/__init__.py @@ -39,7 +39,7 @@ def create_app(env='PROD'): init_connections(True) # TODO: For now as assume always in debug mode, so it doesn't execute the scheduler twice. - init_scheduler(app, True) + #init_scheduler(app, True) socketio = init_socketio(app) diff --git a/ui/src/components/clusters/edit-cluster-modal.html b/ui/src/components/clusters/edit-cluster-modal.html index bbdb24f9..b62f1ca9 100644 --- a/ui/src/components/clusters/edit-cluster-modal.html +++ b/ui/src/components/clusters/edit-cluster-modal.html @@ -7,7 +7,7 @@