diff --git a/GNUmakefile b/GNUmakefile index cf87cf93..d19ada83 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -30,7 +30,7 @@ egg: build: # Now build all the build dependencies for this zenpack. rm -rf $(TXSSHCLIENT_DIR)/build - cd $(TXSSHCLIENT_DIR); python setup.py build + cd $(TXSSHCLIENT_DIR); python setup.py build --build-lib build/lib mkdir -p $(ZP_DIR)/lib/sshclient cp -r $(TXSSHCLIENT_DIR)/build/lib/sshclient/* $(ZP_DIR)/lib/sshclient/ diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/__init__.py b/ZenPacks/zenoss/OpenStackInfrastructure/__init__.py index 5e1fc2db..bf6e21f1 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/__init__.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/__init__.py @@ -31,7 +31,7 @@ from OFS.CopySupport import CopyError schema = CFG.zenpack_module.schema -from service_migration import install_migrate_zenpython, remove_migrate_zenpython, fix_service_healthcheck_path, force_update_configs +from service_migration import remove_migrate_zenpython, force_update_configs NOVAHOST_PLUGINS = ['zenoss.cmd.linux.openstack.nova', 'zenoss.cmd.linux.openstack.libvirt', @@ -54,18 +54,12 @@ def install(self, app): self._migrate_productversions() self._update_plugins('/Server/SSH/Linux/NovaHost') super(ZenPack, self).install(app) - install_migrate_zenpython() if VERSION5: # by default, services are only installed during initial zenpack # installs, not upgrades. We run it every time instead, but make # it only process service definitions that are missing, by # overriding getServiceDefinitionFiles to be intelligent. self.installServices() - - # Fix zenpack-provided healthcheck file paths (since the zenpack's) - # directory may change during install/upgrade - fix_service_healthcheck_path() - # We ship a shell script as a "config" file- make sure that # the config is updated if the script has changed. force_update_configs(self, "proxy-zenopenstack", ["opt/zenoss/bin/proxy-zenopenstack"]) @@ -83,8 +77,6 @@ def getServiceDefinitionFiles(self): svcs = set([s.name for s in ctx.services]) files = [] - if "RabbitMQ-Ceilometer" not in svcs: - files.append(self.path('service_definition', "RabbitMQ-Ceilometer.json")) if "zenopenstack" not in svcs: files.append(self.path('service_definition', "zenopenstack.json")) if "proxy-zenopenstack" not in svcs: @@ -130,8 +122,6 @@ def remove(self, app, leaveObjects=False): finally: ZenPack.UNINSTALLING = False - remove_migrate_zenpython() - # Patch last to avoid import recursion problems. from ZenPacks.zenoss.OpenStackInfrastructure import patches diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/apiclients/txrabbitadminapi.py b/ZenPacks/zenoss/OpenStackInfrastructure/apiclients/txrabbitadminapi.py deleted file mode 100644 index e1a51a7d..00000000 --- a/ZenPacks/zenoss/OpenStackInfrastructure/apiclients/txrabbitadminapi.py +++ /dev/null @@ -1,332 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2016-2017, all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -import logging -log = logging.getLogger('zen.OpenStack.rabbitmqadmin') - -import Globals -from Products.ZenUtils.Utils import unused -unused(Globals) - -import json -import urllib -from base64 import b64encode - -from twisted.internet.defer import inlineCallbacks, returnValue -from twisted.web.client import getPage -from twisted.web.error import Error as TwistedError - - -class RabbitMqAdminApiClient(object): - """Class to connect, login, and retrieve data - """ - def __init__(self, host, port, username, password, timeout=60): - self._target = 'http://{}:{}'.format(host, port) - self._username = username - self._password = password - self._timeout = int(timeout) - self._auth = 'Basic {}'.format(b64encode('{}:{}'.format(username, password))) - - @inlineCallbacks - def _request(self, path, method='GET', data=None): - """Asynchronously request the URL. - """ - - headers = { - 'User-Agent': 'RabbitMqAdminApiClient', - 'Accept': 'application/json', - 'Authorization': self._auth, - 'Content-Type': 'application/json' - } - - if data is not None: - # Intended to be used with PUT method that requires a body be sent - headers['Content-Length'] = str(len(data)) - # No need to add this header,we send all the time - # headers['Content-Type'] = 'application/json' - - url = '{0}{1}'.format(self._target, path) - log.debug('requesting page: {}'.format(url)) - - try: - result = yield getPage( - url, - method=method, - postdata=data, - headers=headers, - cookies={}, - timeout=self._timeout - ) - except TwistedError as e1: - if e1.status == '204' and e1.message == 'No Content': - # Usually indicates put/delete success - returnValue('{"status":"204","message":"No Content"}') - elif ((e1.status == '401' and e1.message == 'Unauthorized' and e1.response is not None) - or (e1.status == '500' and e1.message == 'Internal Server Error' and e1.response is not None) - or (e1.status == '404' and e1.message == 'Object Not Found' and e1.response is not None) - or (e1.status == '400' and e1.message == 'Bad Request' and e1.response is not None)): - # We can handle this exception using the json in the response - returnValue(e1.response) - else: - log.error('Could not fetch url {}: {}'.format(url, e1)) - raise - except Exception as e2: - log.exception('Failed to retrieve {}: {}'.format(url, e2)) - raise - - returnValue(result) - - @inlineCallbacks - def json_from_request(self, path, method='GET', data=None): - """Method to retrieve the json returned - """ - - result = yield self._request(path, method, data) - - if result is not None: - try: - log.debug(result) - out_dict = json.loads(result) - returnValue(out_dict) - except Exception as e: - # We don't care what went wrong as long as we can't decode the - # json log it and return null - log.exception('Could not decode JSON "{}": {}'.format(str(result), e)) - raise - - # Should not happen - returnValue(None) - - @inlineCallbacks - def does_user_exist(self, username): - found = False - json_result = yield self.json_from_request('/api/users') - - if self._does_response_contain_an_error_message(json_result): - returnValue(False) - - # [{"name":"test","password_hash":"AbcdefghijKlmnOPqrsTuVw/bDM=","tags":"administrator"},{"name":"zenoss","password_hash":"c7885c9c376154362f338f256f4590a7=","tags":"management"}] - for user in json_result: - if username == user.get('name', None): - found = True - - returnValue(found) - - @inlineCallbacks - def add_user(self, username, password, tags='monitoring'): - # url encode username because someone might create a user with a slash (just look at the slash in the vhosts we create) - path = '/api/users/' + urllib.quote(username, '') - - # The tags key is mandatory. tags is a comma-separated list of tags for the user. Currently recognised tags are "administrator", "monitoring" and "management". - json_result = yield self.json_from_request(path, 'PUT', '{"password":"'+password+'","tags":"'+tags+'"}') - - # did it work? - if json_result is not None and '204' == json_result.get('status', None): - returnValue(True) - returnValue(False) - - @inlineCallbacks - def delete_user(self, username): - # url encode username because someone might create a user with a slash (just look at the slash in the /zenoss we create) - path = '/api/users/' + urllib.quote(username, '') - - json_result = yield self.json_from_request(path, 'DELETE') - - # did it work? - if json_result is not None and '204' == json_result.get('status', None): - returnValue(True) - returnValue(False) - - @inlineCallbacks - def does_user_have_permissions_on_vhost(self, username, permissionsdict, vhost): - found = False - path = '/api/users/' + urllib.quote(username, '') + "/permissions" - - json_result = yield self.json_from_request(path) - - if self._does_response_contain_an_error_message(json_result): - returnValue(False) - - # [{"user":"zenoss","vhost":"/zenoss","configure":".*","write":".*","read":".*"}] - for vhostpermission in json_result: - if username == vhostpermission.get('user', None) and vhost == vhostpermission.get('vhost', None): - foundConfigurePerm = True - foundWritePerm = True - foundReadPerm = True - # If the caller did not specify a permission to check for we assume anything is ok - if "configure" in permissionsdict: - foundConfigurePerm = (permissionsdict["configure"] == vhostpermission.get('configure', None)) - if "write" in permissionsdict: - foundWritePerm = (permissionsdict["write"] == vhostpermission.get('write', None)) - if "read" in permissionsdict: - foundReadPerm = (permissionsdict["read"] == vhostpermission.get('read', None)) - - # if we found everything we should be good - found = (foundConfigurePerm and foundWritePerm and foundReadPerm) - - returnValue(found) - - @inlineCallbacks - def add_user_permissions_to_vhost(self, username, permissionsdict, vhost): - # All keys are mandatory. - if 'configure' not in permissionsdict or 'write' not in permissionsdict or 'read' not in permissionsdict: - log.error("Called add_user_permissions_to_vhost without all 3 permission keys defined (configure, write, read), what we got was: %s", str(permissionsdict)) - returnValue(False) - - # /api/parameters/component/vhost/name - path = '/api/permissions/' + urllib.quote(vhost, '') + '/' + urllib.quote(username, '') - - json_result = yield self.json_from_request(path, 'PUT', '{"configure":"' + permissionsdict['configure'] - + '","write":"' + permissionsdict['write'] - + '","read":"' + permissionsdict['read'] + '"}') - # did it work? - if (not self._does_response_contain_an_error_message(json_result) - and json_result is not None - and '204' == json_result.get('status', None)): - returnValue(True) - returnValue(False) - - # delete permissions - @inlineCallbacks - def delete_user_permissions_from_vhost(self, username, vhost): - path = '/api/permissions/' + urllib.quote(vhost, '') + '/' + urllib.quote(username, '') - json_result = yield self.json_from_request(path, 'DELETE') - - # did it work? - if (not self._does_response_contain_an_error_message(json_result) - and json_result is not None - and '204' == json_result.get('status', None)): - returnValue(True) - - returnValue(False) - - @inlineCallbacks - def delete_user_permissions_from_all_vhosts(self, username): - success = True - path = '/api/users/' + urllib.quote(username, '') + "/permissions" - - json_result = yield self.json_from_request(path) - - for vhostpermission in json_result: - if username == vhostpermission.get('user', None): - delete_result = yield self.delete_user_permissions_from_vhost(username, vhostpermission.get('vhost', None)) - if not delete_result: - log.error("Unable to delete permissions from vHost %s", vhostpermission.get('vhost', None)) - success = False - returnValue(success) - - @inlineCallbacks - def does_vhost_exist(self, vhostname): - found = False - json_result = yield self.json_from_request('/api/vhosts') - - if self._does_response_contain_an_error_message(json_result): - returnValue(False) - - # [{'tracing': False, 'name': '/'}, {'name': '/zenoss', 'tracing': False, 'messages_details': {'rate': 0.0}, 'messages': 0, 'message_stats': {'publish_details': {'rate': 0.0}, 'deliver_get': 34636, 'publish': 34636, 'get_no_ack': 34636, 'get_no_ack_details': {'rate': 0.0}, 'deliver_get_details': {'rate': 0.0}}, 'messages_unacknowledged_details': {'rate': 0.0}, 'messages_ready_details': {'rate': 0.0}, 'messages_unacknowledged': 0, 'messages_ready': 0}] - for vhost in json_result: - if vhostname == vhost.get('name', None): - found = True - - returnValue(found) - - @inlineCallbacks - def add_vhost(self, vhostname): - # url encode vhostname because someone might create a vhost with a slash (just look at the slash in the /zenoss we create) - path = '/api/vhosts/' + urllib.quote(vhostname, '') - log.debug(path) - - # The tags key is mandatory. tags is a comma-separated list of tags for the user. Currently recognised tags are "administrator", "monitoring" and "management". - json_result = yield self.json_from_request(path, 'PUT') - - # did it work? - if json_result is not None and '204' == json_result.get('status', None): - returnValue(True) - returnValue(False) - - @inlineCallbacks - def delete_vhost(self, vhostname): - # url encode vhostname because someone might create a vhost with a slash (just look at the slash in the /zenoss we create) - path = '/api/vhosts/' + urllib.quote(vhostname, '') - - # The tags key is mandatory. tags is a comma-separated list of tags for the user. Currently recognised tags are "administrator", "monitoring" and "management". - json_result = yield self.json_from_request(path, 'DELETE') - - # did it work? - if json_result is not None and '204' == json_result.get('status', None): - returnValue(True) - returnValue(False) - - @inlineCallbacks - def does_exchange_exist_on_vhost(self, exchange, vhostname): - found = False - # /api/exchanges/vhost/name - path = '/api/exchanges/' + urllib.quote(vhostname, '') - - json_result = yield self.json_from_request(path) - - # [{u'name': u'', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'message_stats': {u'publish_out': 43276, u'publish_out_details': {u'rate': 0.0}, u'publish_in': 43276, u'publish_in_details': {u'rate': 0.0}}, u'arguments': {}, u'type': u'direct', u'auto_delete': False}, {u'name': u'amq.direct', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'direct', u'auto_delete': False}, {u'name': u'amq.fanout', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'fanout', u'auto_delete': False}, {u'name': u'amq.headers', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'headers', u'auto_delete': False}, {u'name': u'amq.match', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'headers', u'auto_delete': False}, {u'name': u'amq.rabbitmq.trace', u'durable': True, u'vhost': u'/zenoss', u'internal': True, u'arguments': {}, u'type': u'topic', u'auto_delete': False}, {u'name': u'amq.topic', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'topic', u'auto_delete': False}, {u'name': u'zenoss.openstack.ceilometer', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'topic', u'auto_delete': False}, {u'name': u'zenoss.openstack.heartbeats', u'durable': True, u'vhost': u'/zenoss', u'internal': False, u'arguments': {}, u'type': u'topic', u'auto_delete': False}] - for vhost in json_result: - if vhostname == vhost.get('vhost', None) and exchange == vhost.get('name', None): - found = True - returnValue(found) - - @inlineCallbacks - def add_exchange_to_vhost(self, exchange, vhostname): - # /api/exchanges/vhost/name - path = '/api/exchanges/' + urllib.quote(vhostname, '') + '/' + urllib.quote(exchange, '') - - # The type key is mandatory; other keys are optional. - # {"type":"direct","auto_delete":false,"durable":true,"internal":false,"arguments":{}} - json_result = yield self.json_from_request(path, 'PUT', '{"type":"topic"}') - - # did it work? - if (not self._does_response_contain_an_error_message(json_result) - and json_result is not None - and '204' == json_result.get('status', None)): - returnValue(True) - - returnValue(False) - - @inlineCallbacks - def delete_exchange_from_vhost(self, exchange, vhostname): - # /api/exchanges/vhost/name - path = '/api/exchanges/' + urllib.quote(vhostname, '') + '/' + urllib.quote(exchange, '') - - # The type key is mandatory; other keys are optional. - # {"type":"direct","auto_delete":false,"durable":true,"internal":false,"arguments":{}} - json_result = yield self.json_from_request(path, 'DELETE') - - # did it work? - if (not self._does_response_contain_an_error_message(json_result) - and json_result is not None - and '204' == json_result.get('status', None)): - returnValue(True) - - returnValue(False) - - @inlineCallbacks - def verify_access(self): - json_result = yield self.json_from_request('/api/whoami') - - if self._does_response_contain_an_error_message(json_result): - returnValue(False) - - # TODO: Further verification on the data returned? - returnValue(True) - - def _does_response_contain_an_error_message(self, json_result): - # Check for json results that indicate this happened - # {"error":"Unauthorized","reason":"\"Unauthorized\"\n"} - # {"error":"not_authorised","reason":"Not administrator user"} - # {"error":"Internal Server Error","reason":"{error, ....}"} - if type(json_result) not in ['list'] and 'error' in json_result: - log.error("Unsuccessful, the following error response was returned instead: %s", str(json_result)) - return True - return False diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/configure.zcml b/ZenPacks/zenoss/OpenStackInfrastructure/configure.zcml index 23a30819..b8296821 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/configure.zcml +++ b/ZenPacks/zenoss/OpenStackInfrastructure/configure.zcml @@ -38,12 +38,6 @@ factory=".datasources.PerfAMQPDataSource.PerfAMQPDataSourceInfo" /> - - now: - log.debug("[%s/%s] Timestamp (%s) appears to be in the future. Using now instead." % (resourceId, meter, value['data']['timestamp'])) - - if timestamp < now - CACHE_EXPIRE_TIME: - log.debug("[%s/%s] Timestamp (%s) is already %d seconds old- discarding message." % (resourceId, meter, value['data']['timestamp'], now - timestamp)) - else: - cache[device_id].add_perf(resourceId, meter, meter_value, timestamp) - - else: - log.error("Discarding unrecognized message type: %s" % value['type']) diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/datasources/QueueSizeDataSource.py b/ZenPacks/zenoss/OpenStackInfrastructure/datasources/QueueSizeDataSource.py index f53762c8..04625c53 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/datasources/QueueSizeDataSource.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/datasources/QueueSizeDataSource.py @@ -1,38 +1,24 @@ ############################################################################## # -# Copyright (C) Zenoss, Inc. 2014, all rights reserved. +# Copyright (C) Zenoss, Inc. 2014-2024, all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. # ############################################################################## -import logging -log = logging.getLogger('zen.OpenStack.QueueSizeStatus') - -from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks - -from zope.component import adapts, getUtility +from zope.component import adapts from zope.interface import implements -from Products.ZenEvents import ZenEventClasses - from ZenPacks.zenoss.PythonCollector.datasources.PythonDataSource import ( PythonDataSource, PythonDataSourcePlugin, PythonDataSourceInfo, IPythonDataSourceInfo) from ZenPacks.zenoss.OpenStackInfrastructure.utils import add_local_lib_path, result_errmsg -add_local_lib_path() - -from zenoss.protocols.interfaces import IAMQPConnectionInfo, IQueueSchema -from zenoss.protocols.twisted.amqp import AMQPFactory -import txamqp.client class QueueSizeDataSource(PythonDataSource): ''' - Datasource used to check the number of messages in this device's ceilometer - queues. + Deprecated. Need to delete with the next release. ''' ZENPACKID = 'ZenPacks.zenoss.OpenStackInfrastructure.' @@ -54,15 +40,14 @@ class QueueSizeDataSource(PythonDataSource): class IQueueSizeDataSourceInfo(IPythonDataSourceInfo): ''' - API Info interface for IQueueSizeDataSource. + Deprecated. Need to delete with the next release. ''' - pass class QueueSizeDataSourceInfo(PythonDataSourceInfo): ''' - API Info adapter factory for QueueSizeDataSource. + Deprecated. Need to delete with the next release. ''' implements(IQueueSizeDataSourceInfo) @@ -71,128 +56,8 @@ class QueueSizeDataSourceInfo(PythonDataSourceInfo): testable = False -# Persistent state -# Technically, we could share the same client for all devices, but -# in the interest of keeping things simple and consistent with the -# other datasources.. -amqp_client = {} # amqp_client[device.id] = AMQClient object - - class QueueSizeDataSourcePlugin(PythonDataSourcePlugin): - proxy_attributes = () - - @classmethod - def config_key(cls, datasource, context): - """ - Return list that is used to split configurations at the collector. - - This is a classmethod that is executed in zenhub. The datasource and - context parameters are the full objects. - """ - return ( - context.device().id, - datasource.getCycleTime(context), - datasource.plugin_classname, - ) - - @classmethod - def params(cls, datasource, context): - return {} - - @inlineCallbacks - def collect(self, config): - log.debug("Collect for OpenStack AMQP Queue Size (%s)" % config.id) - - results = {} - - self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo) - self._queueSchema = getUtility(IQueueSchema) - - # reuse existing connectio if there is one. - if config.id in amqp_client and amqp_client[config.id]: - amqp = amqp_client[config.id] - else: - amqp = AMQPFactory(self._amqpConnectionInfo, self._queueSchema) - yield amqp._onConnectionMade - amqp_client[config.id] = amqp - - for queuename in ('$OpenStackInboundPerf', '$OpenStackInboundEvent',): - queue = self._queueSchema.getQueue(queuename, replacements={'device': config.id}) - try: - info = yield amqp.channel.queue_declare(queue=queue.name, - passive=True) - results[queuename] = info.fields[1] - - except txamqp.client.Closed, e: - log.info("Unable to determine queue size for %s (queue does not exist)" % queue.name) - pass - - except Exception, e: - log.info("Unable to determine queue size for %s (%s)" % (queue.name, e)) - pass - - defer.returnValue(results) - - def _disconnect_amqp_client(self, config_id): - if config_id in amqp_client and amqp_client[config_id]: - amqp = amqp_client[config_id] - amqp.disconnect() - amqp.shutdown() - del amqp_client[config_id] - - def onSuccess(self, result, config): - data = self.new_data() - ds0 = config.datasources[0] - - queue_map = { - 'perfQueueCount': '$OpenStackInboundPerf', - 'eventQueueCount': '$OpenStackInboundEvent' - } - - for point in ds0.points: - if point.id not in queue_map: - log.error("Queue datapoint '%s' is not supported" % point.id) - continue - - if queue_map[point.id] not in result: - log.error("No queue count available for %s %s" % (config.id, queue_map[point.id])) - continue - - value = result[queue_map[point.id]] - data['values'][ds0.component][point.id] = (value, 'N') - - if len(data['values']): - data['events'].append({ - 'device': config.id, - 'component': ds0.component, - 'summary': 'OpenStack AMQP QueueSize: successful collection', - 'severity': ZenEventClasses.Clear, - 'eventKey': 'openstackCeilometerAMQPCollection', - 'eventClassKey': 'openstack-QueueSize', - }) - - return data - - def onError(self, result, config): - # just in case some error we don't recover from is occurring, throw - # out the connection. - self._disconnect_amqp_client(config.id) - - errmsg = 'OpenStack AMQP QueueSize: %s' % result_errmsg(result) - log.error('%s: %s', config.id, errmsg) - - data = self.new_data() - data['events'].append({ - 'device': config.id, - 'summary': errmsg, - 'severity': ZenEventClasses.Error, - 'eventKey': 'openstackCeilometerAMQPCollection', - 'eventClassKey': 'openstack-QueueSize', - }) - - return data - - def cleanup(self, config): - log.info("Disconnecting any open AMQP connections for OpenStack AMQP QueueDataSource (%s)" % config.id) - - self._disconnect_amqp_client(config.id) + """ + Deprecated. Need to delete with the next release. + """ + pass diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/dsplugins.py b/ZenPacks/zenoss/OpenStackInfrastructure/dsplugins.py index 3063f731..45fbd030 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/dsplugins.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/dsplugins.py @@ -7,10 +7,6 @@ # ############################################################################## -import logging -log = logging.getLogger('zen.OpenStack') - -import datetime from twisted.internet import defer from twisted.internet.defer import inlineCallbacks @@ -18,34 +14,16 @@ PythonDataSourcePlugin from Products.DataCollector.plugins.DataMaps import ObjectMap -from Products.ZenEvents import ZenEventClasses import zope.component -from zope.component import getUtility from Products.Five import zcml import Products.ZenMessaging.queuemessaging -from zenoss.protocols.interfaces import IAMQPConnectionInfo zcml.load_config('meta.zcml', zope.component) zcml.load_config('configure.zcml', zope.component) zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging) -from ZenPacks.zenoss.OpenStackInfrastructure.apiclients.txrabbitadminapi import RabbitMqAdminApiClient - -try: - import servicemigration as sm - sm.require("1.0.0") - VERSION5 = True -except ImportError: - VERSION5 = False - -# cache of the last password set for each user during rabbitmq provisioning -USER_PASSWORD = {} - -# and the last time it was checked -USER_LASTCHECK = {} - class MaintenanceDataSourcePlugin(PythonDataSourcePlugin): proxy_attributes = () @@ -84,140 +62,3 @@ def onSuccess(self, result, config): )) return data - - -class RabbitMQCeilometerCredentialsDataSourcePlugin(PythonDataSourcePlugin): - proxy_attributes = () - - @classmethod - def config_key(cls, datasource, context): - # We would ideally run one task per username to provision. - # (not per device), but because the task splitter runs per device, - # this results in duplicate tasks, which fail silently to be allocated. - # (See ZEN-27883) - # - # Instead, we will run it once per device, but only perform the - # collection step a maximum of once every 5 minutes. - return ( - context.device().id, - datasource.getCycleTime(context), - datasource.plugin_classname, - context.zOpenStackAMQPUsername - ) - - @classmethod - def params(cls, datasource, context): - return { - 'zOpenStackAMQPUsername': context.zOpenStackAMQPUsername, - 'zOpenStackAMQPPassword': context.zOpenStackAMQPPassword - } - - @defer.inlineCallbacks - def collect(self, config): - ds0 = config.datasources[0] - user = ds0.params['zOpenStackAMQPUsername'] - password = ds0.params['zOpenStackAMQPPassword'] - - if not VERSION5: - # This datasource is only applicable to v5. - defer.returnValue(None) - - if not user: - # the task splitter will still consider no user as a valid - # task differentiator. So, if this is the task for the non - # existant username, just exit without doing anything. - defer.returnValue(None) - - last_run = USER_LASTCHECK.get(user) - if last_run and datetime.datetime.now() - last_run < datetime.timedelta(minutes=5): - # This user has already been checked within the last - # 5 minutes. - log.debug( - "Skipping RabbitMQCeilometerCredentialsDataSourcePlugin for " - "user %s because it has already run recently", user) - defer.returnValue(None) - - USER_LASTCHECK[user] = \ - datetime.datetime.now() - - connectionInfo = getUtility(IAMQPConnectionInfo) - - if USER_PASSWORD.get(user, None) != password: - # password has changed, or not previously been set- go ahead - # and set that user up. (note that this will also verify - # the vhost and exchanges) - - yield self._verify_amqp_credentials( - vhost='/zenoss', - user=user, - password=password, - admin_host='localhost', - admin_port='45672', - admin_user=connectionInfo.user, - admin_password=connectionInfo.password - ) - - # Store password if we set it successfully. - USER_PASSWORD[user] = password - - def onSuccess(self, result, config): - return { - 'events': [{ - 'summary': 'RabbitMQ-Ceilometer: successful completion', - 'eventKey': 'RabbitMQCeilometerResult', - 'severity': ZenEventClasses.Clear, - 'eventClassKey': 'openstack-RabbitMQ' - }], - } - - def onError(self, result, config): - return { - 'events': [{ - 'summary': 'Error configuring RabbitMQ-Ceilometer: %s' % result, - 'eventKey': 'RabbitMQCeilometerResult', - 'severity': ZenEventClasses.Error, - 'eventClassKey': 'openstack-RabbitMQ' - }], - } - - @inlineCallbacks - def _verify_amqp_credentials(self, vhost, user, password, admin_host, admin_port, admin_user, admin_password): - client = RabbitMqAdminApiClient(admin_host, admin_port, admin_user, admin_password) - - log.info("Verifying access to RabbitMQ Management HTTP API") - if not (yield client.verify_access()): - raise Exception("Cannot access RabbitMQ Management HTTP API http://%s:%s as %s" % (admin_host, admin_port, admin_user)) - - log.info("Verifying vhost...") - success = yield client.does_vhost_exist(vhost) - if not success: - success = yield client.add_vhost(vhost) - if not success: - raise Exception("Unable to create vhost %s, cannot complete setup" % vhost) - - log.info("Verifying exchanges...") - for exchange in ('zenoss.openstack.ceilometer', ): - success = yield client.does_exchange_exist_on_vhost(exchange, vhost) - if not success: - success = yield client.add_exchange_to_vhost(exchange, vhost) - if not success: - raise Exception("Unable to create exchange %s on vhost %s, cannot complete setup" % (exchange, vhost)) - - log.info("Verifying user '%s'" % user) - existing_user = yield client.does_user_exist(user) - - # create user if missing, set password if it exists. - success = yield client.add_user(user, password) - if not success: - raise Exception("Unable to create or update user %s, cannot complete setup" % user) - - if not existing_user: - # If this is a new user, setup desired permissions - log.info("Setting up new amqp user's permissions") - success = yield client.delete_user_permissions_from_all_vhosts(user) - if success: - success = yield client.add_user_permissions_to_vhost(user, {"configure": "zenoss.openstack.*", - "write": "zenoss.openstack.*", - "read": "^$"}, vhost) - if not success: - raise Exception("Unable to set permissions for user %s on vhost %s, cannot complete setup" % (user, vhost)) diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/init_rabbitmq-ceil.sh b/ZenPacks/zenoss/OpenStackInfrastructure/libexec/init_rabbitmq-ceil.sh deleted file mode 100755 index 32dfdafd..00000000 --- a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/init_rabbitmq-ceil.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -set -e -SENTINEL=/tmp/rabbit_ceil_ok -if [ ! -e $SENTINEL ] ; then - USER=$1 - PASSWORD=$2 - #Tag with administrator in order to add ability to create and delete users, create and delete permissions through http api - rabbitmqctl add_user $USER $PASSWORD || true - rabbitmqctl change_password $USER $PASSWORD - rabbitmqctl set_user_tags $USER administrator management - rabbitmqctl add_vhost /zenoss || true - rabbitmqctl clear_permissions -p /zenoss $USER - rabbitmqctl set_permissions -p /zenoss $USER '.*' '.*' '.*' - rabbitmqadmin --port=45672 --vhost=/zenoss --username=$USER --password=$PASSWORD declare exchange name=zenoss.openstack.ceilometer type=topic - rabbitmqctl delete_user guest || true - touch $SENTINEL -fi diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_amqp_init.py b/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_amqp_init.py deleted file mode 100644 index c6a34186..00000000 --- a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_amqp_init.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python -########################################################################### -# -# This program is part of Zenoss Core, an open source monitoring platform. -# Copyright (C) 2014-2016, Zenoss Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 or (at your -# option) any later version as published by the Free Software Foundation. -# -# For complete information please visit: http://www.zenoss.com/oss/ -# -########################################################################### - -import logging -log = logging.getLogger('zen.OpenStack.amqp_init') - -import os -import subprocess -import sys -import uuid - -import transaction - -import Globals -from Products.ZenUtils.Utils import unused -unused(Globals) - -import zope.component -from zope.component import getUtility, getAdapter -from Products.Five import zcml - -from Products.ZenUtils.ZenScriptBase import ZenScriptBase -import Products.ZenMessaging.queuemessaging -from zenoss.protocols.interfaces import IAMQPConnectionInfo, IQueueSchema, IAMQPChannelAdapter -from zenoss.protocols.amqp import Publisher as BlockingPublisher - -try: - import servicemigration as sm - sm.require("1.0.0") - VERSION5 = True -except ImportError: - VERSION5 = False - - -def require_rabbitmq(): - print "Verifying rabbitmqctl access" - try: - with open(os.devnull, 'w') as devnull: - subprocess.check_call(["sudo", "rabbitmqctl", "list_vhosts"], stdout=devnull) - except subprocess.CalledProcessError, e: - sys.exit("Unable to exceute rabbitmqctl (%s)\nPlease execute this command on the correct host." % e) - - -def create_exchanges(): - print "Verifying exchanges" - - # Create all the AMQP exchanges we require. This should be run before we try - # to have ceilometer send zenoss any events. - zcml.load_config('meta.zcml', zope.component) - zcml.load_config('configure.zcml', zope.component) - zcml.load_config('configure.zcml', Products.ZenMessaging.queuemessaging) - - connectionInfo = getUtility(IAMQPConnectionInfo) - queueSchema = getUtility(IQueueSchema) - amqpClient = BlockingPublisher(connectionInfo, queueSchema) - channel = amqpClient.getChannel() - - for exchange in ('$OpenStackInbound', ): - exchangeConfig = queueSchema.getExchange(exchange) - print "Verifying configuration of exchange '%s' (%s)" % (exchange, exchangeConfig.name) - - # Declare the exchange to which the message is being sent - getAdapter(channel, IAMQPChannelAdapter).declareExchange(exchangeConfig) - - amqpClient.close() - - -def create_default_credentials(dmd): - print "Checking default credentials" - dc = dmd.getObjByPath("Devices/OpenStack/Infrastructure") - if not dc.getProperty('zOpenStackAMQPPassword'): - print "Generating random default zOpenStackAMQPPassword" - random_password = uuid.uuid4().bytes.encode("base64")[:15] - dc.setZenProperty('zOpenStackAMQPPassword', random_password) - - -def provision_user(dmd): - dc = dmd.getObjByPath("Devices/OpenStack/Infrastructure") - user = dc.getZ('zOpenStackAMQPUsername') - password = dc.getProperty('zOpenStackAMQPPassword') - - conninfo = getUtility(IAMQPConnectionInfo) - vhost = conninfo.vhost - - if not [x for x in subprocess.check_output(['sudo', 'rabbitmqctl', 'list_users']).split("\n") if x.startswith(user + "\t")]: - print "Adding user %s to rabbitmq" % user - try: - with open(os.devnull, 'w') as devnull: - subprocess.check_call(['sudo', 'rabbitmqctl', 'add_user', user, password], stdout=devnull) - except subprocess.CalledProcessError, e: - sys.exit("Unable to exceute rabbitmqctl (%s)" % e) - else: - # Set the password (unconditionally) - print "Updating user password" - try: - with open(os.devnull, 'w') as devnull: - subprocess.check_call(['sudo', 'rabbitmqctl', 'change_password', user, password], stdout=devnull) - except subprocess.CalledProcessError, e: - sys.exit("Unable to exceute rabbitmqctl (%s)" % e) - - # Fix the user permissions, too, just in case. - print "Verifying user permissions" - try: - with open(os.devnull, 'w') as devnull: - subprocess.check_call(['sudo', 'rabbitmqctl', '-p', vhost, 'clear_permissions', user], stdout=devnull) - subprocess.check_call(['sudo', 'rabbitmqctl', '-p', vhost, 'set_permissions', user, 'zenoss.openstack.*', 'zenoss.openstack.*', '^$'], stdout=devnull) - except subprocess.CalledProcessError, e: - sys.exit("Unable to exceute rabbitmqctl (%s)" % e) - - -if __name__ == '__main__': - if not VERSION5: - require_rabbitmq() - create_exchanges() - - # Log into ZODB.. - try: - script = ZenScriptBase() - script.connect() - dmd = script.dmd - except Exception, e: - sys.exit("Unable to connect to ZODB: %s" % e) - - create_default_credentials(dmd) - - if not VERSION5: - provision_user(dmd) - - rabbit_ip = {} - if VERSION5: - from Products.ZenUtils.controlplane.application import getConnectionSettings - from Products.ZenUtils.controlplane import ControlPlaneClient - - client = ControlPlaneClient(**getConnectionSettings()) - for svc_id in [x.id for x in client.queryServices() if x.name == 'RabbitMQ-Ceilometer']: - svc = client.getService(svc_id) - for ceil_endpoint in filter(lambda s: s['Name'] == 'rabbitmq_ceil', svc.getRawData()['Endpoints']): - try: - ip = ceil_endpoint['AddressAssignment']['IPAddr'] - collector = client.getService(svc.parentId).name - - rabbit_ip[collector] = ip - except Exception: - # no ip assignment or error determining what collector this is. - pass - - dc = dmd.getObjByPath("Devices/OpenStack/Infrastructure") - - transaction.commit() - - conninfo = getUtility(IAMQPConnectionInfo) - if VERSION5: - port = 55672 - else: - port = conninfo.port - - print """ -OpenStack Configuration ------------------------ -Add the following configuration to ceilometer.conf on all openstack nodes: -For Mitaka and newer versions, - - [DEFAULT] - meter_dispatchers = zenoss - event_dispatchers = zenoss - -For Newton and newer versions, - [event] - drop_unmatched_notifications = true - -For all versions, - [notification] - store_events=True - - [dispatcher_zenoss] - - zenoss_device = [device that the openstack environment is registered as in zenoss] - - amqp_port = {AMQP_PORT} - amqp_userid = {AMQP_USERID} - amqp_password = {AMQP_PASSWORD} - amqp_virtual_host = {AMQP_VIRTUAL_HOST} - amqp_hostname = [The correct AMQP system IP or hostname (see below)] - - """.format( - AMQP_PORT=port, - AMQP_USERID=dc.getZ('zOpenStackAMQPUsername'), - AMQP_PASSWORD=dc.getProperty('zOpenStackAMQPPassword'), - AMQP_VIRTUAL_HOST=conninfo.vhost, - ) - - if VERSION5: - if rabbit_ip: - print "* The correct IP address will depend upon the collector for the device:" - for collector_name, ip in rabbit_ip.iteritems(): - print " %s: %s" % (collector_name, ip) - else: - print "* No RabbitMQ-Ceilometer IP address assignments could be found." - print " - To correct this, in Control Center, click 'Assign' for the" - print " unassigned RabbitMQ-Ceilometer IP assignments." - else: - print """ - * Your current Zenoss amqp_hostname is set to '{AMQP_HOSTNAME}'. - - => If this is not the correct IP to access your rabbitmq server from - external hosts, you must specify the proper IP for amqp_hostname - in ceilometer.conf. - """.format(AMQP_HOSTNAME=conninfo.host) diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_helper.py b/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_helper.py index 194f3bf7..d1163dfe 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_helper.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/libexec/openstack_helper.py @@ -21,7 +21,7 @@ from optparse import OptionParser from twisted.internet.defer import inlineCallbacks, returnValue -from apiclients.session import SessionManager +from ZenPacks.zenoss.OpenStackInfrastructure.apiclients.session import SessionManager class OpenstackHelper(object): diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/poll_openstack.py b/ZenPacks/zenoss/OpenStackInfrastructure/libexec/poll_openstack.py index 5a5b99ce..350a4e67 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/libexec/poll_openstack.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/libexec/poll_openstack.py @@ -17,9 +17,9 @@ from twisted.internet.defer import inlineCallbacks, returnValue -from apiclients.exceptions import APIClientError -from apiclients.session import SessionManager -from apiclients.txapiclient import NovaClient, NeutronClient, CinderClient +from ZenPacks.zenoss.OpenStackInfrastructure.apiclients.exceptions import APIClientError +from ZenPacks.zenoss.OpenStackInfrastructure.apiclients.session import SessionManager +from ZenPacks.zenoss.OpenStackInfrastructure.apiclients.txapiclient import NovaClient, NeutronClient, CinderClient class OpenStackPoller(object): diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/migrate/RemoveRabbitMQCeilometerService.py b/ZenPacks/zenoss/OpenStackInfrastructure/migrate/RemoveRabbitMQCeilometerService.py new file mode 100644 index 00000000..0a6e009f --- /dev/null +++ b/ZenPacks/zenoss/OpenStackInfrastructure/migrate/RemoveRabbitMQCeilometerService.py @@ -0,0 +1,202 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +"""Remove RabbitMQ-Ceilometer service""" + +import logging +import Globals +from Acquisition import aq_base +# Platform imports +from Products.ZenModel.migrate.Migrate import Version +from Products.ZenModel.ZenPack import ZenPackMigration +from Products.Zuul.interfaces import ICatalogTool +from Products.ZenModel.RRDTemplate import RRDTemplate +from Products.ZenRelations.RelationshipBase import RelationshipBase +# ZenPack imports +from ZenPacks.zenoss.OpenStackInfrastructure.Instance import Instance +from ZenPacks.zenoss.OpenStackInfrastructure.Endpoint import Endpoint +from ZenPacks.zenoss.OpenStackInfrastructure.Vnic import Vnic +from service_migration import remove_migrate_zenpython + +try: + import servicemigration as sm + from Products.ZenUtils.application import ApplicationState + from Products.ZenUtils.controlplane.application import getConnectionSettings + from Products.ZenUtils.controlplane import ControlPlaneClient + from Products.ZenUtils.controlplane import ControlCenterError +except ImportError: + CONTROL_CENTER = False +else: + CONTROL_CENTER = True + +LOG = logging.getLogger("zen.migrate") + +# Name of service to remove. +SERVICE_NAME = "rabbitmq-ceilometer" + + +class RemoveRabbitMQCeilometerService(ZenPackMigration): + version = Version(4, 0, 1) + + def migrate(self, pack): + if pack.prevZenPackVersion is None: + # Do nothing if this is a fresh install of self.version. + return + if not CONTROL_CENTER: + return + + self.migrate_service(pack) + self.migrate_templates(pack.dmd) + self.rebuild_relations(pack) + self.migrate_zprops(pack.dmd) + + def rebuild_relations(self, pack): + results = ICatalogTool(pack.dmd.Devices.OpenStack.Infrastructure).search([Instance, Vnic, Endpoint]) + LOG.info("starting: %s total objects", results.total) + objects_migrated = 0 + + for brain in results: + try: + if self.updateRelations(brain.getObject()): + objects_migrated += 1 + except Exception: + LOG.exception( + "error updating relationships for %s", + brain.id) + + LOG.info("finished: %s of %s objects required migration", objects_migrated, results.total) + + def updateRelations(self, comp): + if isinstance(comp, Instance): + relations = Instance._relations + elif isinstance(comp, Endpoint): + relations = Endpoint._relations + elif isinstance(comp, Vnic): + relations = Vnic._relations + for relname in (x[0] for x in relations): + rel = getattr(aq_base(comp), relname, None) + if not rel or not isinstance(rel, RelationshipBase): + comp.buildRelations() + return True + + return False + + def migrate_zprops(self, dmd): + zprops_to_remove = ["zOpenStackAMQPUsername", "zOpenStackAMQPPassword"] + for zprop in zprops_to_remove: + if dmd.Devices.hasProperty(zprop): + try: + dmd.Devices._delProperty(zprop) + except Exception as e: + LOG.warn("Failed to delete zProperty - %s with a message: %s", zprop, e) + + def migrate_templates(self, dmd): + amqp_templates = [ + "Endpoint", + "Instance", + "Vnic" + ] + ds_types_to_delete = [ + "OpenStack Ceilometer AMQP", + "OpenStack Ceilometer Events AMQP", + "OpenStack Ceilometer Heartbeats AMQP", + "OpenStack AMQP Queue Size" + ] + ds_names_to_delete = [ + "rabbitmq-credentials", + "openstackQueues" + ] + templates = [] + results = ICatalogTool(dmd.Devices.OpenStack.Infrastructure).search(RRDTemplate) + if results.total == 0: + return + for result in results: + try: + template = result.getObject() + except Exception: + continue + if template.id in amqp_templates: + templates.append(template) + for template in templates: + ds_to_delete = [] + for ds in template.datasources(): + if ds.sourcetype in ds_types_to_delete or ds.id in ds_names_to_delete: + ds_to_delete.append(ds.id) + if ds_to_delete: + try: + template.manage_deleteRRDDataSources(ds_to_delete) + except Exception as e: + LOG.warn("Failed to delete datasource - %s with a message: %s", ds_to_delete, e) + # if the template is empty then remove it + if len(template.datasources()) == 0: + try: + template.deviceClass().manage_deleteRRDTemplates((template.id,)) + except Exception as e: + LOG.warn("Failed to delete template - %s with a message: %s", template.id, e) + + + def migrate_service(self, pack): + sm.require("1.0.0") + try: + ctx = sm.ServiceContext() + except Exception as e: + LOG.warn("Failed to remove %s service: %s", SERVICE_NAME, e) + return + services = get_services_ids(ctx) + if not services: + return + client = ControlPlaneClient(**getConnectionSettings()) + + # Stop and remove the old service and rabbitmqs_ceil endpoints in zenpython. + remove_migrate_zenpython() + remove_service(client, services) + + +def get_services_ids(ctx): + services = [] + """Return service to be removed or None.""" + for service in ctx.services: + if service.name.lower() == SERVICE_NAME: + services.append(service) + return services + + +def stop_service(client, service): + """Stop service.""" + service_id = service._Service__data['ID'] + + try: + status = client.queryServiceStatus(service_id) + except Exception as e: + LOG.warn("failed to get %s service status: %s", service.name, e) + else: + for instance_status in status.itervalues(): + if instance_status.status != ApplicationState.STOPPED: + try: + client.stopService(service_id) + except Exception as e: + LOG.warn("failed to stop %s service: %s", service.name, e) + + return + + +def remove_service(client, services): + """Remove service. Stopping it first if necessary.""" + for service in services: + stop_service(client, service) + + try: + client.deleteService(service._Service__data['ID']) + LOG.info("successfuly removed service from serviced: %s", service.name) + except ControlCenterError as e: + LOG.warn("failed to remove %s service: %s", service.name, e) + + + +RemoveRabbitMQCeilometerService() diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/-CONFIGS-/etc/rabbitmq/rabbitmq-env.conf b/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/-CONFIGS-/etc/rabbitmq/rabbitmq-env.conf deleted file mode 100644 index 85dd8983..00000000 --- a/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/-CONFIGS-/etc/rabbitmq/rabbitmq-env.conf +++ /dev/null @@ -1,5 +0,0 @@ -NODENAME=rabbit@rbt-ceil{{.InstanceID}} -NODE_IP_ADDRESS=0.0.0.0 -RABBITMQ_MNESIA_BASE=/var/lib/rabbitmq/mnesia-ceil-{{(parent .).Name}}{{if ne .InstanceID 0}}-node{{end}} -RABBITMQ_NODE_PORT=55672 - diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/-CONFIGS-/etc/rabbitmq/rabbitmq.config b/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/-CONFIGS-/etc/rabbitmq/rabbitmq.config deleted file mode 100644 index 36470649..00000000 --- a/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/-CONFIGS-/etc/rabbitmq/rabbitmq.config +++ /dev/null @@ -1,11 +0,0 @@ -[ - {rabbit, [ - {log_levels, [{connection, error}]}, - {cluster_nodes, {[{{with $service := .}}{{with $rabbits := .Instances}}{{range (each $rabbits) }}'rabbit@rbt-ceil{{.}}'{{if ne (plus 1 .) $rabbits}},{{end}}{{end}}{{end}}{{end}}], disc}} - ]}, - {rabbitmq_management, [{listener, [{port, 45672}]}]}, - {kernel, [ - {inet_dist_listen_max, 54001}, - {inet_dist_listen_min, 54001} - ]} -]. diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/RabbitMQ-Ceilometer.json b/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/RabbitMQ-Ceilometer.json deleted file mode 100644 index ccf1aa14..00000000 --- a/ZenPacks/zenoss/OpenStackInfrastructure/service_definition/RabbitMQ-Ceilometer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "servicePath": "/hub/collector", - "serviceDefinition": { - "CPUCommitment": 1, - "Command": "/usr/sbin/rabbitmq-server", - "ConfigFiles": { - "/etc/rabbitmq/rabbitmq-env.conf": { - "FileName": "/etc/rabbitmq/rabbitmq-env.conf", - "Owner": "root:root", - "Permissions": "0664" - }, - "/etc/rabbitmq/rabbitmq.config": { - "FileName": "/etc/rabbitmq/rabbitmq.config", - "Owner": "root:root", - "Permissions": "0664" - } - }, - "Description": "RabbitMQ server for OpenStackInfrastructure Ceilometer", - "Endpoints": [ - { - "AddressConfig": { - "Port": 45672, - "Protocol": "tcp" - }, - "Application": "openstack_rabbitmq_{{(parent .).Name}}_admin", - "Name": "rabbitmq_ceil admin", - "PortNumber": 45672, - "Protocol": "tcp", - "Purpose": "export" - }, - { - "AddressConfig": { - "Port": 55672, - "Protocol": "tcp" - }, - "Application": "openstack_rabbitmq_{{(parent .).Name}}", - "Name": "rabbitmq_ceil", - "PortNumber": 55672, - "Protocol": "tcp", - "Purpose": "export" - }, - { - "Application": "openstack_rabbitmq_{{(parent .).Name}}_epmd", - "Name": "rabbitmq_ceil_epmd", - "PortNumber": 54369, - "Protocol": "tcp", - "Purpose": "export" - }, - { - "Application": "openstack_rabbitmq_{{(parent .).Name}}_inet_dist", - "Name": "rabbitmq_ceil_inet_dist", - "PortNumber": 54001, - "Protocol": "tcp", - "Purpose": "export" - }, - { - "Application": "openstack_rabbitmq_{{(parent .).Name}}_epmd", - "Name": "rabbitmq_ceil_epmds", - "PortNumber": 54369, - "PortTemplate": "{{plus .InstanceID 24369}}", - "Protocol": "tcp", - "Purpose": "import_all", - "VirtualAddress": "rbtc{{.InstanceID}}:54369" - }, - { - "Application": "openstack_rabbitmq_{{(parent .).Name}}_inet_dist", - "Name": "rabbitmq_ceil_inet_dists", - "PortNumber": 54001, - "PortTemplate": "{{plus .InstanceID 49001}}", - "Protocol": "tcp", - "Purpose": "import_all", - "VirtualAddress": "rbtc{{.InstanceID}}:54001" - }, - { - "Application": "openstack_rabbitmq_{{(parent .).Name}}", - "Name": "rabbitmqs_ceil", - "PortNumber": 55672, - "PortTemplate": "{{plus .InstanceID 35672}}", - "Protocol": "tcp", - "Purpose": "import_all", - "VirtualAddress": "rbtc{{.InstanceID}}:55672" - } - ], - "Environment": [ - "ERL_EPMD_PORT=54369" - ], - "HealthChecks": { - "admin": { - "Interval": 10.0, - "Script": "curl -s -u {{(getContext . \"global.conf.amqpuser\")}}:{{(getContext . \"global.conf.amqppassword\")}} http://localhost:45672/api/vhosts | grep -q /zenoss" - }, - "publishing": { - "Interval": 10.0, - "Script": "curl -s -u {{(getContext . \"global.conf.amqpuser\")}}:{{(getContext . \"global.conf.amqppassword\")}} http://localhost:45672/api/aliveness-test/%%2Fzenoss | grep -q '{\"status\":\"ok\"}'" - }, - "ceilometer-setup": { - "Interval": 15.0, - "Script": "init_rabbitmq-ceil.sh {{(getContext . \"global.conf.amqpuser\")}} {{(getContext . \"global.conf.amqppassword\")}}" - } - }, - "Hostname": "rbt-ceil{{.InstanceID}}", - "ImageID": "", - "Instances": { - "min": 1 - }, - "Launch": "auto", - "LogConfigs": [ - { - "path": "/var/log/rabbitmq/rabbit@localhost.log", - "type": "rabbitmq" - }, - { - "path": "/var/log/rabbitmq/rabbit@localhost-sasl.log", - "type": "rabbitmq_sasl" - } - ], - "Name": "RabbitMQ-Ceilometer", - "Prereqs": [ - { - "Name": "Cluster Leader Start", - "Script": "[ $(hostname) == 'rbt-ceil0' ] || rabbitmqctl -n rabbit@rbt-ceil0 status | grep nodedown || sleep 5" - } - ], - "Privileged": true, - "RAMCommitment": "256M", - "Services": [], - "Snapshot": { - "Pause": "PAUSE_CHECK_TIMEOUT=60 ${ZENHOME:-/opt/zenoss}/bin/quiesce-rabbitmq.sh pause", - "Resume": "${ZENHOME:-/opt/zenoss}/bin/quiesce-rabbitmq.sh resume" - }, - "Tags": [ - "daemon", - "collector" - ], - "Volumes": [ - { - "#####": "drwxr-x--- 4 rabbitmq rabbitmq 4096 Feb 25 21:25 /var/lib/rabbitmq", - "ContainerPath": "/var/lib/rabbitmq", - "Owner": "rabbitmq:rabbitmq", - "Permission": "0750", - "ResourcePath": "rabbitmq/ceil-{{(parent .).Name}}{{.InstanceID}}" - } - ] - } -} diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/service_migration.py b/ZenPacks/zenoss/OpenStackInfrastructure/service_migration.py index e8c51b74..83dda0db 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/service_migration.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/service_migration.py @@ -22,30 +22,6 @@ import copy import os -def fix_service_healthcheck_path(): - # When the zenpack is installed or upgraded, the path to the healthcheck - # script will need to be updated to reflect the current zenpack path. - - if not VERSION5: - return - - try: - ctx = sm.ServiceContext() - except sm.ServiceMigrationError: - log.info("Couldn't generate service context, skipping.") - return - - for svc in ctx.services: - if svc.name != 'RabbitMQ-Ceilometer': - continue - - for check in svc.healthChecks: - if check.name == 'ceilometer-setup': - log.info("Enabling %s healthcheck for %s", check.name, ctx.getServicePath(svc)) - script_path = zenpack_path('libexec/init_rabbitmq-ceil.sh') - check.script = script_path + ' {{(getContext . "global.conf.amqpuser")}} {{(getContext . "global.conf.amqppassword")}}' - ctx.commit() - def force_update_configs(zenpack, service_name, config_filenames): # update the specified config filenames to match those provided in the @@ -76,97 +52,6 @@ def force_update_configs(zenpack, service_name, config_filenames): ctx.commit() -def install_migrate_zenpython(): - if not VERSION5: - return - - try: - ctx = sm.ServiceContext() - except sm.ServiceMigrationError: - log.info("Couldn't generate service context, skipping.") - return - - # Create the endpoints we're importing. - mgmt_endpoint = { - "ApplicationTemplate": "openstack_rabbitmq_{{(parent .).Name}}_admin", - "Name": "rabbitmqadmins_ceil", - "PortNumber": 45672, - "PortTemplate": "{{plus .InstanceID 45672}}", - "Protocol": "tcp", - "Purpose": "import_all" - } - mgmt_endpoint_lc = {key.lower(): value for (key, value) in mgmt_endpoint.iteritems()} - try: - rabbit_mgmt = sm.Endpoint(**mgmt_endpoint_lc) - except TypeError: - mgmt_endpoint_lc = {key.lower(): value for (key, value) in mgmt_endpoint.iteritems() if key != "PortTemplate"} - amqp_endpoint = { - "ApplicationTemplate": "openstack_rabbitmq_{{(parent .).Name}}", - "Name": "rabbitmqs_ceil", - "PortNumber": 55672, - "PortTemplate": "{{plus .InstanceID 55672}}", - "Protocol": "tcp", - "Purpose": "import_all" - } - amqp_endpoint_lc = {key.lower(): value for (key, value) in amqp_endpoint.iteritems()} - try: - rabbit_amqp = sm.Endpoint(**amqp_endpoint_lc) - except TypeError: - amqp_endpoint_lc = {key.lower(): value for (key, value) in amqp_endpoint.iteritems() if key != "PortTemplate"} - - commit = False - zpythons = filter(lambda s: s.name == "zenpython", ctx.services) - log.info("Found %i services named 'zenpython'" % len(zpythons)) - for zpython in zpythons: - collector = ctx.getServiceParent(zpython).name - rbtamqp_imports = filter(lambda ep: ep.name == "rabbitmqs_ceil" and ep.purpose == "import_all", zpython.endpoints) - if len(rbtamqp_imports) > 0: - log.info("Service %s already has a rabbitmqs_ceil endpoint." % ctx.getServicePath(zpython)) - else: - log.info("Adding a rabbitmqs_ceil import endpoint to service '%s'." % ctx.getServicePath(zpython)) - rabbit_amqp = sm.Endpoint(**amqp_endpoint_lc) - rabbit_amqp.__data = copy.deepcopy(amqp_endpoint) - rabbit_amqp.application = "openstack_rabbitmq_{}".format(collector) - zpython.endpoints.append(rabbit_amqp) - commit = True - rbtmgmt_imports = filter(lambda ep: ep.name == "rabbitmqadmins_ceil" and ep.purpose == "import_all", zpython.endpoints) - if len(rbtmgmt_imports) > 0: - log.info("Service %s already has a rabbitmqadmins_ceil endpoint." % ctx.getServicePath(zpython)) - else: - log.info("Adding a rabbitmqadmins_ceil import endpoint to service '%s'." % ctx.getServicePath(zpython)) - rabbit_mgmt = sm.Endpoint(**mgmt_endpoint_lc) - rabbit_mgmt.__data = copy.deepcopy(mgmt_endpoint) - rabbit_mgmt.application = "openstack_rabbitmq_{}_admin".format(collector) - zpython.endpoints.append(rabbit_mgmt) - commit = True - - # Fix application names for existing rabbitmq-ceilometer and zenpython endpoints - # naming convention. - services = filter(lambda s: s.name == "zenpython" or s.name == "RabbitMQ-Ceilometer", ctx.services) - change_count = 0 - for service in services: - collector = ctx.getServiceParent(service).name - for endpoint in service.endpoints: - if endpoint.application.startswith("rabbitmq_"): - new_application_name = endpoint.application.replace("rabbitmq_", "openstack_rabbitmq_") - log.debug("%s/%s: Changing endpoint from %s to %s", collector, service.name, endpoint.application, new_application_name) - endpoint.application = new_application_name - commit = True - change_count += 1 - if endpoint.applicationtemplate.startswith("rabbitmq_"): - new_application_template = endpoint.applicationtemplate.replace("rabbitmq_", "openstack_rabbitmq_") - log.debug("%s/%s: Changing endpoint template from %s to %s", collector, service.name, endpoint.applicationtemplate, new_application_template) - endpoint.applicationtemplate = new_application_template - commit = True - change_count += 1 - - if change_count: - log.info("Upgraded %d endpoint names and templates to new naming convention.", change_count) - - if commit: - ctx.commit() - - def remove_migrate_zenpython(): if not VERSION5: return diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/tests/test_cvim.py b/ZenPacks/zenoss/OpenStackInfrastructure/tests/test_cvim.py index 4fa4d747..2116fd3c 100755 --- a/ZenPacks/zenoss/OpenStackInfrastructure/tests/test_cvim.py +++ b/ZenPacks/zenoss/OpenStackInfrastructure/tests/test_cvim.py @@ -48,7 +48,7 @@ def afterSetUp(self): self.linux_dc = self.dmd.Devices.createOrganizer('/Server/SSH/Linux/NovaHost') self.hostmap = HostMap() - @crochet.wait_for(timeout=30) + @crochet.wait_for(timeout=60) def perform_mapping(self): # This has to be run through the twisted reactor. return self.hostmap.perform_mapping() diff --git a/ZenPacks/zenoss/OpenStackInfrastructure/zenpack.yaml b/ZenPacks/zenoss/OpenStackInfrastructure/zenpack.yaml index f439e213..3e97855d 100644 --- a/ZenPacks/zenoss/OpenStackInfrastructure/zenpack.yaml +++ b/ZenPacks/zenoss/OpenStackInfrastructure/zenpack.yaml @@ -89,15 +89,6 @@ zProperties: containers. Provide the container names (or a pattern to match them) here, or leave blank in a non-containerized openstack environment. - zOpenStackAMQPUsername: - type: string - default: openstack - label: Username for ceilometer AMQP integration - description: See documentation for details. - zOpenStackAMQPPassword: - type: password - label: Password for ceilometer AMQP integration - description: See documentation for details. zOpenStackHostLocalDomain: type: string default: '' @@ -227,9 +218,6 @@ device_classes: description: 'OpenStackInfrastructure: Endpoint Monitoring' targetPythonClass: ZenPacks.zenoss.OpenStackInfrastructure.Endpoint datasources: - events: - type: OpenStack Ceilometer Events AMQP - cycletime: 30 neutronAgentStatus: type: OpenStack Neutron Agent Status cycletime: 30 @@ -237,10 +225,6 @@ device_classes: type: Python plugin_classname: ZenPacks.zenoss.OpenStackInfrastructure.dsplugins.MaintenanceDataSourcePlugin cycletime: 7200 - rabbitmq-credentials: - type: Python - plugin_classname: ZenPacks.zenoss.OpenStackInfrastructure.dsplugins.RabbitMQCeilometerCredentialsDataSourcePlugin - cycletime: 300 openstack: type: COMMAND commandTemplate: ${here/ZenPackManager/packs/ZenPacks.zenoss.OpenStackInfrastructure/path}/libexec/poll_openstack.py @@ -384,12 +368,6 @@ device_classes: poolQoSSupportCount: aliases: {pool_qos_support__count: null} parser: ZenPacks.zenoss.OpenStackInfrastructure.parsers.endpoint - openstackQueues: - type: OpenStack AMQP Queue Size - cycletime: 300 - datapoints: - eventQueueCount: {} - perfQueueCount: {} serviceStatus: type: OpenStack Nova Service Status cycletime: 30 @@ -512,17 +490,6 @@ device_classes: Total: dpName: openstack_networkTotalCount format: '%7.0lf' - Queues: - units: count - miny: 0 - graphpoints: - Event: - dpName: openstackQueues_eventQueueCount - format: '%7.0lf' - Performance: - dpName: openstackQueues_perfQueueCount - format: '%7.0lf' - color: 98df8a Routers: units: count miny: 0 diff --git a/docs/body.md b/docs/body.md index 84a00f83..036e43a6 100644 --- a/docs/body.md +++ b/docs/body.md @@ -1018,6 +1018,18 @@ option requires that your browser have https connectivity to the Zenoss collecto external SSH access may not be available by default, and additional configuration may be required to achieve it. +* Starting from OpenStackInfrastructure version 4.0.1, the RabbitMQ-ceilometer + service has been removed from the ZenPack. As a result, events and performance + data are no longer collected from OpenStack Ceilometer via AMQM. OpenStack + Ceilometer collection have been fully migrated to the zenopenstack service. + The following OpenStackInfrastructure AMQP data source types are deleted + during the upgrade: + + - OpenStack Ceilometer AMQP + - OpenStack Ceilometer Events AMQP + - OpenStack Ceilometer Heartbeats AMQP + - OpenStack AMQP Queue Size" + Zenoss Analytics ---------------- @@ -1203,6 +1215,21 @@ Known Issues Changes ------- +4.0.1 + +- Removed RabbitMQ-ceilometer service from the ZenPack (ZPS-8941) +- Fixed execute permission for OSI devices (ZPS-8939) +- Tested with Zenoss Resource Manager 6.7.0, Zenoss Cloud and Service Impact 5.7.0 + +4.0.0 + +- Added support for OSP 16.x +- Increased Read timeout (ZPS-7352) +- Modeling needs to succeed when cinder service is not available. (ZPS-7362) +- Host name is constructed with informative string unique to the endpoint and host combination (ZPS-7318) +- Added support for Queens, Rocky, Ussuri, Train, Victoria and Red Hat OpenStack Platform (RHOSP) version 13, 14 and 16 +- Tested with Zenoss Resource Manager 6.5.0, 6.6.0, Zenoss Cloud and Service Impact 5.5.3 + 3.0.1 - Add support for Twisted library update (ZPS-6975) diff --git a/docs/releases.md b/docs/releases.md index 89b743cf..171247b4 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,20 +1,14 @@ Releases -------- -Version 4.0.0- Download -: Released on 2021/06/04 -: Requires PythonCollector ZenPack,OpenStack (Tenant View) ZenPack,Linux Monitor ZenPack -: Requires PythonCollector ZenPack,OpenStack (Tenant View) ZenPack,Linux Monitor ZenPack -: Tested with Zenoss Resource Manager 6.6.0, Zenoss Cloud and Service Impact 5.5.3 -Version 3.0.1- Download -: Released on 2020/06/05 +Version 4.0.1 - Download +: Released on 2024/08/06 : Requires PythonCollector ZenPack,OpenStack (Tenant View) ZenPack,Linux Monitor ZenPack : Requires PythonCollector ZenPack,OpenStack (Tenant View) ZenPack,Linux Monitor ZenPack -: Tested with Zenoss Resource Manager 6.4.1, Zenoss Cloud and Service Impact 5.5.1 +: Compatible with Zenoss 6.x and Zenoss Cloud -Version 2.4.2- Download -: Released on 2018/08/15 +Version 4.0.0 - Download +: Released on 2021/06/04 : Requires PythonCollector ZenPack,OpenStack (Tenant View) ZenPack,Linux Monitor ZenPack : Requires PythonCollector ZenPack,OpenStack (Tenant View) ZenPack,Linux Monitor ZenPack -: Compatible with Zenoss 4.2 - 6.2 - +: Compatible with Zenoss 6.x and Zenoss Cloud