From 7a5d196c4171815ce1c6a31e767393d043a50d11 Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Wed, 30 Aug 2023 15:07:22 +0000 Subject: [PATCH 01/10] Start 1.0.13 Hotfix: Version 1.0.12 -> 1.0.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index becb7f4..56dab97 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # or saved. Do not modify them directly here. # NB: PACKAGES is deprecated NAME = "ZenPacks.zenoss.PostgreSQL" -VERSION = "1.0.12" +VERSION = "1.0.13" AUTHOR = "Zenoss" LICENSE = "GPLv2" NAMESPACE_PACKAGES = ['ZenPacks', 'ZenPacks.zenoss'] From 433784c80e1112513b2e5764beb31fe3162b774f Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Wed, 30 Aug 2023 15:11:04 +0000 Subject: [PATCH 02/10] Move changes from develop to hotfix 1.0.13 Implements ZPS-8554. Move changes in scope of the ticket from develop to hotfix. --- ZenPacks/zenoss/PostgreSQL/__init__.py | 27 ++++++++++++++----- .../modeler/plugins/zenoss/PostgreSQL.py | 18 ++++++++++--- docs/body.md | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index 5963237..aac3b5b 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -12,6 +12,7 @@ ########################################################################### import logging + log = logging.getLogger('zen.PostgreSQL') import os @@ -22,6 +23,7 @@ from Products.ZenRelations.RelSchema import ToManyCont, ToOne from Products.ZenUtils.Utils import monkeypatch, zenPath + class ZenPack(ZenPackBase): packZProperties = [ ('zPostgreSQLPort', 5432, 'int'), @@ -29,8 +31,16 @@ class ZenPack(ZenPackBase): ('zPostgreSQLPassword', '', 'password'), ('zPostgreSQLUseSSL', False, 'boolean'), ('zPostgreSQLDefaultDB', 'postgres', 'string'), + ('zPostgreSQLTableRegex', '', 'string'), ] + packZProperties_data = { + 'zPostgreSQLTableRegex': { + 'description': "Filter tables from all databases if name matches regex provided", + 'label': "Regex Table Filter", + 'type': "string"}, + } + def install(self, app): super(ZenPack, self).install(app) self.patchPostgreSQLDriver() @@ -40,10 +50,10 @@ def remove(self, app, leaveObjects=False): if not leaveObjects: # Remove our custom relations addition. Device._relations = tuple( - [ x for x in Device._relations if x[0] != 'pgDatabases' ]) + [x for x in Device._relations if x[0] != 'pgDatabases']) self.updateExistingRelations(app.zport.dmd) - + # Revert all pg8000 library patches self.patchPostgreSQLDriver(revert=True) @@ -57,21 +67,22 @@ def updateExistingRelations(self, dmd): def patchPostgreSQLDriver(self, revert=False): log.info('Patching pg8000 core library') patch_dir = self.path('lib') - + # Getting a list of all patches which will be applied patches_list = [file for file in os.listdir(patch_dir) if file.endswith('.patch')] - - cmd = "patch -p0 -d %s -i %s" if not revert else "patch -p0 -R -d %s -i %s" + + cmd = "patch -p0 -d %s -i %s" if not revert else "patch -p0 -R -d %s -i %s" for patch in patches_list: os.system(cmd % ( os.path.join(patch_dir, 'pg8000'), os.path.join(patch_dir, patch) )) + # Allow PostgreSQL databases to be related to any device. Device._relations += ( ('pgDatabases', ToManyCont(ToOne, - 'ZenPacks.zenoss.PostgreSQL.Database.Database', 'server')), + 'ZenPacks.zenoss.PostgreSQL.Database.Database', 'server')), ) # We need to filter components by id instead of name. @@ -79,6 +90,7 @@ def patchPostgreSQLDriver(self, revert=False): "\"(device = '%s' and component = '%s')\"" " % (me.device().getDmdKey(), me.id)") + @monkeypatch('Products.ZenModel.Device.Device') def setPostgreSQL(self, active=True): if not active: @@ -96,11 +108,12 @@ def setPostgreSQL(self, active=True): if device.zCommandCommandTimeout < 180: device.setZenProperty('zCommandCommandTimeout', 180) + @monkeypatch('Products.ZenModel.Device.Device') def getPostgreSQL(self): device = self.primaryAq() if 'PostgreSQLServer' in device.zDeviceTemplates \ - and device.zCommandCommandTimeout >= 180: + and device.zCommandCommandTimeout >= 180: return True return False diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 9649e74..c870c7f 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -12,6 +12,8 @@ ########################################################################### import logging +import re + log = logging.getLogger('zen.PostgreSQL') from Products.DataCollector.plugins.CollectorPlugin import PythonPlugin @@ -20,6 +22,7 @@ from ZenPacks.zenoss.PostgreSQL.util import PgHelper + class PostgreSQL(PythonPlugin): deviceProperties = PythonPlugin.deviceProperties + ( 'zPostgreSQLPort', @@ -27,6 +30,7 @@ class PostgreSQL(PythonPlugin): 'zPostgreSQLPassword', 'zPostgreSQLUseSSL', 'zPostgreSQLDefaultDB', + 'zPostgreSQLTableRegex', ) def collect(self, device, unused): @@ -39,6 +43,7 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} + pattern = re.compile(getattr(device, 'zPostgreSQLTableRegex', '').strip()) log.info("Getting database list") try: @@ -55,8 +60,15 @@ def collect(self, device, unused): log.info("Getting tables list for {0}".format(dbName)) try: - results['databases'][dbName]['tables'] = \ - pg.getTablesInDatabase(dbName) + + tables = pg.getTablesInDatabase(dbName) + if pattern.pattern: + for key in tables.keys(): + if not pattern.match(key): + del tables[key] + + results['databases'][dbName]['tables'] = tables + except Exception, ex: log.warn("Error getting tables list for {0}: {1}".format( dbName, ex)) @@ -69,7 +81,7 @@ def process(self, devices, results, unused): if results is None: return None - maps = [ self.objectMap(dict(setPostgreSQL=True)) ] + maps = [self.objectMap(dict(setPostgreSQL=True))] databases = [] for dbName, dbDetail in results['databases'].items(): diff --git a/docs/body.md b/docs/body.md index 2e54c86..fbaa2bb 100644 --- a/docs/body.md +++ b/docs/body.md @@ -82,6 +82,7 @@ individual devices. - *zPostgreSQLUsername* - Must be a superuser. Default: postgres - *zPostgreSQLPassword* - Password for user. No default. - *zPostgreSQLDefaultDB* - Default database. Default: postgres + - *zPostgreSQLTableRegex* - Filter tables of all databases with Regex. Default: "" In addition to setting these properties you must add the ''zenoss.PostgreSQL'' modeler plugin to a device class or individual device. This modeler plugin will From ab03562734c9c96bcf21a94b1a1b6d8f722a4d95 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Thu, 31 Aug 2023 17:47:08 +0000 Subject: [PATCH 03/10] ZPS-8554: filter table names with regex lines Zprop --- ZenPacks/zenoss/PostgreSQL/__init__.py | 8 ++++++ .../modeler/plugins/zenoss/PostgreSQL.py | 12 ++++++++- ZenPacks/zenoss/PostgreSQL/util.py | 25 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index 5963237..389868f 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -29,8 +29,16 @@ class ZenPack(ZenPackBase): ('zPostgreSQLPassword', '', 'password'), ('zPostgreSQLUseSSL', False, 'boolean'), ('zPostgreSQLDefaultDB', 'postgres', 'string'), + ('zPostgreSQLTableRegex', [], 'lines'), ] + packZProperties_data = { + 'zPostgreSQLTableRegex': { + 'description': "List of regular expressions (matched against table names) to control which tables are NOT modeled from All databases.", + 'label': "Regex Table Filter", + 'type': "lines" }, + } + def install(self, app): super(ZenPack, self).install(app) self.patchPostgreSQLDriver() diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 1a872df..9a4e44d 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -18,7 +18,7 @@ from Products.DataCollector.plugins.DataMaps import ObjectMap, RelationshipMap from Products.ZenUtils.Utils import prepId -from ZenPacks.zenoss.PostgreSQL.util import PgHelper +from ZenPacks.zenoss.PostgreSQL.util import PgHelper, exclude_patterns_list, is_suppressed class PostgreSQL(PythonPlugin): deviceProperties = PythonPlugin.deviceProperties + ( @@ -39,6 +39,7 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} + exclude_patterns = exclude_patterns_list(getattr(device, 'zPostgreSQLTableRegex', [])) log.info("Getting database list") try: @@ -56,6 +57,15 @@ def collect(self, device, unused): log.info("Getting tables list for {0}".format(dbName)) try: results['databases'][dbName]['tables'] = pg.getTablesInDatabase(dbName) + + tables = pg.getTablesInDatabase(dbName) + if pattern.pattern: + for key in tables.keys(): + if is_suppressed(key, exclude_patterns): + del tables[key] + + results['databases'][dbName]['tables'] = tables + except Exception, ex: log.warn("Error getting tables list for {0}: {1}".format( dbName, ex)) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index d701d2e..7de85d2 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -545,3 +545,28 @@ def getTableStatsForDatabase(self, db): cursor.close() return tableStats + +def exclude_patterns_list(excludes): + exclude_patterns = [] + + for exclude in excludes: + exclude = exclude.strip() + if exclude == "" or exclude.startswith("#"): + continue + + try: + exclude_patterns.append(re.compile(exclude)) + except Exception: + LOG.warn("Invalid zPostgreSQLTableRegex value: '%s', this modeling filter will not be applied.", exclude) + continue + + return exclude_patterns + + +def is_suppressed(item, exclude_patterns): + + for exclude_pattern in exclude_patterns: + if exclude_pattern.search(item): + return True + + return False From 2431adcd8112f1329b714ea252477983a3907fc7 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Thu, 31 Aug 2023 18:26:28 +0000 Subject: [PATCH 04/10] ZPS-8554: check against exclude_patterns --- ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 9a4e44d..784d93d 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -59,7 +59,7 @@ def collect(self, device, unused): results['databases'][dbName]['tables'] = pg.getTablesInDatabase(dbName) tables = pg.getTablesInDatabase(dbName) - if pattern.pattern: + if exclude_patterns: for key in tables.keys(): if is_suppressed(key, exclude_patterns): del tables[key] From 325f525c9252321e45f6aa5013312c532a9e3dc6 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Tue, 5 Sep 2023 06:41:55 +0000 Subject: [PATCH 05/10] ZPS-8554: add missing imports --- ZenPacks/zenoss/PostgreSQL/util.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index d701d2e..288d14c 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -15,7 +15,9 @@ import math import sys import time - +import re +import logging +LOG = logging.getLogger('zen.Nutanix.utils') def addLocalLibPath(): """ @@ -545,3 +547,28 @@ def getTableStatsForDatabase(self, db): cursor.close() return tableStats + +def exclude_patterns_list(excludes): + exclude_patterns = [] + + for exclude in excludes: + exclude = exclude.strip() + if exclude == "" or exclude.startswith("#"): + continue + + try: + exclude_patterns.append(re.compile(exclude)) + except Exception: + LOG.warn("Invalid zPostgreSQLTableRegex value: '%s', this modeling filter will not be applied.", exclude) + continue + + return exclude_patterns + + +def is_suppressed(item, exclude_patterns): + + for exclude_pattern in exclude_patterns: + if exclude_pattern.search(item): + return True + + return False From dbaae0208402e75ce7cf853133cee4049056322c Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Tue, 5 Sep 2023 06:46:02 +0000 Subject: [PATCH 06/10] ZPS-8554: update logger name --- ZenPacks/zenoss/PostgreSQL/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index 288d14c..8a5823c 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -17,7 +17,7 @@ import time import re import logging -LOG = logging.getLogger('zen.Nutanix.utils') +LOG = logging.getLogger('zen.PostgreSQL.utils') def addLocalLibPath(): """ From e5751327bcd1ebd48eb779368475b9e9b4f13a16 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Wed, 6 Sep 2023 18:14:42 +0000 Subject: [PATCH 07/10] ZPS-8692: check for db before device() --- ZenPacks/zenoss/PostgreSQL/Table.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/Table.py b/ZenPacks/zenoss/PostgreSQL/Table.py index 4733463..345ff30 100644 --- a/ZenPacks/zenoss/PostgreSQL/Table.py +++ b/ZenPacks/zenoss/PostgreSQL/Table.py @@ -60,5 +60,7 @@ class Table(DeviceComponent, ManagedEntity, CollectedOrModeledMixin): event_key = "ComponentId" def device(self): - return self.database().device() - + db = self.database() + if db: + return db.device() + return None From 8e6db3ebe43beef1894d36c35fb46c15ab3ac060 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Wed, 6 Sep 2023 18:20:57 +0000 Subject: [PATCH 08/10] ZPS-8692: ZPS-8688: update docs --- docs/body.md | 4 ++++ docs/releases.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/body.md b/docs/body.md index 2e54c86..fe1fe8a 100644 --- a/docs/body.md +++ b/docs/body.md @@ -164,6 +164,10 @@ zSnmpMonitorIgnore property to True and remodel. Changes --------------- +1.0.13 + +* Make PostgreSQL table modeling optional (ZPS-8554) + 1.0.12 * Resolved isuue with error in pg8000 library on Ubuntu OS (ZPS-7424) diff --git a/docs/releases.md b/docs/releases.md index e9a3fce..424476f 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,6 +1,10 @@ Releases -------- +Version 1.0.13-Download +Released on 2023/9/7 +Tested with Zenoss 6.7.0, Zenoss Cloud and Service Impact 5.6.0 + Version 1.0.12-Download Released on 2021/1/14 Compatible with Zenoss 6.4.1 - 6.5.0, Zenoss Cloud and Service Impact 5.5.3 From 0eef3a8c7181065fd8ea40995e8b900e6b5768c3 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Thu, 7 Sep 2023 14:01:12 +0000 Subject: [PATCH 09/10] ZPS-8692: ZPS-8688: fix changelog --- docs/body.md | 1 + docs/releases.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/body.md b/docs/body.md index fe1fe8a..07362c3 100644 --- a/docs/body.md +++ b/docs/body.md @@ -167,6 +167,7 @@ Changes 1.0.13 * Make PostgreSQL table modeling optional (ZPS-8554) +* Tested with Zenoss Cloud, Zenoss 6.7.0 and Service Impact 5.6.0 1.0.12 diff --git a/docs/releases.md b/docs/releases.md index 424476f..80ce64a 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -3,7 +3,7 @@ Releases Version 1.0.13-Download Released on 2023/9/7 -Tested with Zenoss 6.7.0, Zenoss Cloud and Service Impact 5.6.0 +Compatible with Zenoss 6.x, Zenoss Cloud and Service Impact 5.6.1 Version 1.0.12-Download Released on 2021/1/14 From 4c54d4ef9cd09eb0c6e6e37c943d84f9528f8fba Mon Sep 17 00:00:00 2001 From: vsaliieva Date: Tue, 12 Sep 2023 15:00:51 +0000 Subject: [PATCH 10/10] Finish 1.0.13 Hotfix: Version 1.0.12 -> 1.0.13 --- docs/body.md | 2 +- docs/releases.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/body.md b/docs/body.md index 1a83895..84ffdba 100644 --- a/docs/body.md +++ b/docs/body.md @@ -168,7 +168,7 @@ Changes 1.0.13 * Make PostgreSQL table modeling optional (ZPS-8554) -* Tested with Zenoss Cloud, Zenoss 6.7.0 and Service Impact 5.6.0 +* Tested with Zenoss Cloud, Zenoss 6.7.0 and Service Impact 5.6.1 1.0.12 diff --git a/docs/releases.md b/docs/releases.md index 80ce64a..03599f0 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,7 +2,7 @@ Releases -------- Version 1.0.13-Download -Released on 2023/9/7 +Released on 2023/9/12 Compatible with Zenoss 6.x, Zenoss Cloud and Service Impact 5.6.1 Version 1.0.12-Download