From 8a7e4120ac287b6449c0699510f932947dfa8768 Mon Sep 17 00:00:00 2001 From: Chet Luther Date: Wed, 29 Oct 2014 12:09:51 -0500 Subject: [PATCH 01/20] Post Release: Version 1.0.8 -> 1.0.9dev. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf6b689..49b23c3 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # or saved. Do not modify them directly here. # NB: PACKAGES is deprecated NAME = "ZenPacks.zenoss.PostgreSQL" -VERSION = "1.0.8" +VERSION = "1.0.9dev" AUTHOR = "Zenoss" LICENSE = "GPLv2" NAMESPACE_PACKAGES = ['ZenPacks', 'ZenPacks.zenoss'] From 9fe43de7f357368f2170284d6337adf2d89941b6 Mon Sep 17 00:00:00 2001 From: Philip Carinhas Date: Thu, 30 Oct 2014 13:54:21 -0500 Subject: [PATCH 02/20] release: version 1.0.7 -> 1.0.8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4b2f7b..bf6b689 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # or saved. Do not modify them directly here. # NB: PACKAGES is deprecated NAME = "ZenPacks.zenoss.PostgreSQL" -VERSION = "1.0.8dev" +VERSION = "1.0.8" AUTHOR = "Zenoss" LICENSE = "GPLv2" NAMESPACE_PACKAGES = ['ZenPacks', 'ZenPacks.zenoss'] From 62a8cd23d0194753f2c41190b6bceccd8683d27e Mon Sep 17 00:00:00 2001 From: Philip Carinhas Date: Thu, 30 Oct 2014 14:02:56 -0500 Subject: [PATCH 03/20] post release: 1.0.8 -> 1.0.9dev --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf6b689..49b23c3 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # or saved. Do not modify them directly here. # NB: PACKAGES is deprecated NAME = "ZenPacks.zenoss.PostgreSQL" -VERSION = "1.0.8" +VERSION = "1.0.9dev" AUTHOR = "Zenoss" LICENSE = "GPLv2" NAMESPACE_PACKAGES = ['ZenPacks', 'ZenPacks.zenoss'] From a36371b33e558d79ae2767777111d5754b086684 Mon Sep 17 00:00:00 2001 From: Philip Carinhas Date: Thu, 6 Nov 2014 16:34:15 -0600 Subject: [PATCH 04/20] Remove all PIDs for this query --- ZenPacks/zenoss/PostgreSQL/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index b044c33..f750836 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -434,6 +434,7 @@ def getLocks(self): " FROM pg_database AS d" " INNER JOIN pg_locks AS l ON l.database = d.oid" " WHERE NOT d.datistemplate AND d.datallowconn" + " AND pid <> pg_backend_pid()" ) locks.update(locksTemplate) From 65e587b7987ee744249625ee68124003b3fc1f3a Mon Sep 17 00:00:00 2001 From: Philip Carinhas Date: Fri, 6 Jan 2017 10:25:37 -0600 Subject: [PATCH 05/20] Add Changes to Docs --- README.mediawiki | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.mediawiki b/README.mediawiki index f5d68ae..f7e52a2 100644 --- a/README.mediawiki +++ b/README.mediawiki @@ -122,3 +122,13 @@ SELECT relname, relid, schemaname, pg_total_relation_size(relid) AS total_size FROM pg_stat_user_tables + +== Changes == + +;1.0.9 +* Filter PIDs for lock query (ZEN-15165) + +;1.0.8 +* Handle null data by skipping it (ZEN-14276) +* Update pg8000 lib to 1.9.14 (ZEN-12752) + From a404299c620c9b5b36d01afcc7fe0d644bbc48ba Mon Sep 17 00:00:00 2001 From: Philip Carinhas Date: Fri, 6 Jan 2017 10:28:48 -0600 Subject: [PATCH 06/20] Start Develop 1.1.0 branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 49b23c3..6614dd4 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # or saved. Do not modify them directly here. # NB: PACKAGES is deprecated NAME = "ZenPacks.zenoss.PostgreSQL" -VERSION = "1.0.9dev" +VERSION = "1.1.0" AUTHOR = "Zenoss" LICENSE = "GPLv2" NAMESPACE_PACKAGES = ['ZenPacks', 'ZenPacks.zenoss'] From 42722af2acdae144b98307fdd3391ecb87bae55f Mon Sep 17 00:00:00 2001 From: Philip Carinhas Date: Tue, 10 Jan 2017 15:47:49 -0600 Subject: [PATCH 07/20] Add docs for dedicated db user access. Fixes ZPS-312 * Describe how to add dedicated user * Add statement_timeout description --- README.mediawiki | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/README.mediawiki b/README.mediawiki index f7e52a2..332f419 100644 --- a/README.mediawiki +++ b/README.mediawiki @@ -45,7 +45,7 @@ The following metrics will be collected every 5 minutes by default. * Tuple Rate Metrics: Sequentially Read/sec, Index Fetched/sec, Inserted/sec, Updated/sec, Hot Updated/sec, Deleted/sec * Tuple Metrics: Live, Dead -== Usage == +== Usage, Security and Setup == Once the PostgreSQL ZenPack is installed you will have the following new configuration properties which should be set either for device classes or @@ -54,7 +54,7 @@ individual devices. ;Configuration Properties * ''zPostgreSQLPort'' - Port where PostgreSQL is listening. Default: 5432 * ''zPostgreSQLUseSSL'' - Whether to use SSL or not. Default: False -* ''zPostgreSQLUsername'' - Must be a superuser. Default: postgres +* ''zPostgreSQLUsername'' - Must have read access to databases. Default: postgres * ''zPostgreSQLPassword'' - Password for user. No default. In addition to setting these properties you must add the ''zenoss.PostgreSQL'' @@ -63,6 +63,46 @@ will discover all databases and tables using the connectivity information provided through the above settings. Each database and table will automatically be monitored. +=== Security Considerations === + +In order to increase security, we recommend setting up a dedicated user account +to perform all queries. + +In addition, to avoid hung database processes, setting a *statement_timeout* to +15 seconds for the dedicated Zenoss user. This causes all commands that do not +finish within 15 seconds to timeout and ensures PostgreSQL will not run out of +connection resources due to Zenoss activity. + +In the following example we assume the dedicated user is named '''zenoss'''. +We further will use commands that are exclusive to the 9.X version of +PostgreSQL. Analagous commands can be found for older versions. + +==== Dedicated User Setup ==== + +* From the PostgreSQL superuser account, first create a dedicated user named "zenoss" via commandline tool: + + createuser -S -D -R -e zenoss + +* Next issue the following SQL command as the database superuser: + + +-- Grant SELECT on all existing tables in SCHEMA public: +-- Note: This only affects tables that have already been created. + + GRANT SELECT ON ALL TABLES IN SCHEMA public TO zenoss; + +-- Automatically have default roles assigned to new objects in future: + + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO zenoss; + +-- Finally set the connection_timeout for the user. + + ALTER ROLE zenoss SET statement_timeout=15000; + + +You should now be able to issue any of the commands in the following section +using this dedicated PostgreSQL user. + === PostgreSQL Server Impact === Zenoss will run the following queries every five (5) minutes. These queries From 7b3ebb66fa810ef332190176c208ab7349db4c7e Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Wed, 3 May 2023 04:59:48 +0000 Subject: [PATCH 08/20] ZPS-8554: add regex zProp to filter unmatching table names --- ZenPacks/zenoss/PostgreSQL/__init__.py | 1 + .../modeler/plugins/zenoss/PostgreSQL.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index 5963237..9f4b0cf 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -29,6 +29,7 @@ class ZenPack(ZenPackBase): ('zPostgreSQLPassword', '', 'password'), ('zPostgreSQLUseSSL', False, 'boolean'), ('zPostgreSQLDefaultDB', 'postgres', 'string'), + ('zPostgreSQLTableRegex', '', 'string'), ] def install(self, app): diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 9649e74..c17f9bb 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -12,6 +12,7 @@ ########################################################################### import logging +import re log = logging.getLogger('zen.PostgreSQL') from Products.DataCollector.plugins.CollectorPlugin import PythonPlugin @@ -27,6 +28,7 @@ class PostgreSQL(PythonPlugin): 'zPostgreSQLPassword', 'zPostgreSQLUseSSL', 'zPostgreSQLDefaultDB', + 'zPostgreSQLTableRegex', ) def collect(self, device, unused): @@ -39,6 +41,7 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} + pattern = re.compile(device.zPostgreSQLTableRegex) log.info("Getting database list") try: @@ -55,8 +58,15 @@ def collect(self, device, unused): log.info("Getting tables list for {0}".format(dbName)) try: - results['databases'][dbName]['tables'] = \ - pg.getTablesInDatabase(dbName) + + tables = pg.getTablesInDatabase(dbName) + if pattern.pattern != '': + for key in tables.keys(): + if not pattern.match(key): + del tables[key] + + results['databases'][dbName]['tables'] = tables + except Exception, ex: log.warn("Error getting tables list for {0}: {1}".format( dbName, ex)) From e9891dd5d170251b7c40eb8dfefa3570234d31b2 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Wed, 31 May 2023 05:30:27 +0000 Subject: [PATCH 09/20] ZPS-8554: review fixes --- ZenPacks/zenoss/PostgreSQL/__init__.py | 7 +++++++ .../zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py | 4 ++-- docs/body.md | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index 9f4b0cf..bb0c641 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -32,6 +32,13 @@ class ZenPack(ZenPackBase): ('zPostgreSQLTableRegex', '', 'string'), ] + packZProperties_data = { + 'zPostgreSQLTableRegex': { + 'description': "Filter tables from all databases if name matches regex provided", + 'label': "Regex Table Filter", + 'type': "string" }, + } + def install(self, app): super(ZenPack, self).install(app) self.patchPostgreSQLDriver() diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index c17f9bb..92b306d 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -41,7 +41,7 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} - pattern = re.compile(device.zPostgreSQLTableRegex) + pattern = re.compile(getattr(device, 'zPostgreSQLTableRegex', '').strip()) log.info("Getting database list") try: @@ -60,7 +60,7 @@ def collect(self, device, unused): try: tables = pg.getTablesInDatabase(dbName) - if pattern.pattern != '': + if pattern.pattern: for key in tables.keys(): if not pattern.match(key): del tables[key] diff --git a/docs/body.md b/docs/body.md index 2e54c86..fbaa2bb 100644 --- a/docs/body.md +++ b/docs/body.md @@ -82,6 +82,7 @@ individual devices. - *zPostgreSQLUsername* - Must be a superuser. Default: postgres - *zPostgreSQLPassword* - Password for user. No default. - *zPostgreSQLDefaultDB* - Default database. Default: postgres + - *zPostgreSQLTableRegex* - Filter tables of all databases with Regex. Default: "" In addition to setting these properties you must add the ''zenoss.PostgreSQL'' modeler plugin to a device class or individual device. This modeler plugin will From 375d940f87bf8170d05cbba2bacfb0584b6d4706 Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Wed, 30 Aug 2023 14:57:17 +0000 Subject: [PATCH 10/20] Revert the changes in develope branch Implements ZPS-8554. Revert changes in the dev branch and move them to hotfix. --- ZenPacks/zenoss/PostgreSQL/__init__.py | 8 -------- .../PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py | 13 +------------ docs/body.md | 1 - 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index bb0c641..5963237 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -29,16 +29,8 @@ class ZenPack(ZenPackBase): ('zPostgreSQLPassword', '', 'password'), ('zPostgreSQLUseSSL', False, 'boolean'), ('zPostgreSQLDefaultDB', 'postgres', 'string'), - ('zPostgreSQLTableRegex', '', 'string'), ] - packZProperties_data = { - 'zPostgreSQLTableRegex': { - 'description': "Filter tables from all databases if name matches regex provided", - 'label': "Regex Table Filter", - 'type': "string" }, - } - def install(self, app): super(ZenPack, self).install(app) self.patchPostgreSQLDriver() diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 92b306d..1a872df 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -12,7 +12,6 @@ ########################################################################### import logging -import re log = logging.getLogger('zen.PostgreSQL') from Products.DataCollector.plugins.CollectorPlugin import PythonPlugin @@ -28,7 +27,6 @@ class PostgreSQL(PythonPlugin): 'zPostgreSQLPassword', 'zPostgreSQLUseSSL', 'zPostgreSQLDefaultDB', - 'zPostgreSQLTableRegex', ) def collect(self, device, unused): @@ -41,7 +39,6 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} - pattern = re.compile(getattr(device, 'zPostgreSQLTableRegex', '').strip()) log.info("Getting database list") try: @@ -58,15 +55,7 @@ def collect(self, device, unused): log.info("Getting tables list for {0}".format(dbName)) try: - - tables = pg.getTablesInDatabase(dbName) - if pattern.pattern: - for key in tables.keys(): - if not pattern.match(key): - del tables[key] - - results['databases'][dbName]['tables'] = tables - + results['databases'][dbName]['tables'] = pg.getTablesInDatabase(dbName) except Exception, ex: log.warn("Error getting tables list for {0}: {1}".format( dbName, ex)) diff --git a/docs/body.md b/docs/body.md index fbaa2bb..2e54c86 100644 --- a/docs/body.md +++ b/docs/body.md @@ -82,7 +82,6 @@ individual devices. - *zPostgreSQLUsername* - Must be a superuser. Default: postgres - *zPostgreSQLPassword* - Password for user. No default. - *zPostgreSQLDefaultDB* - Default database. Default: postgres - - *zPostgreSQLTableRegex* - Filter tables of all databases with Regex. Default: "" In addition to setting these properties you must add the ''zenoss.PostgreSQL'' modeler plugin to a device class or individual device. This modeler plugin will From 7a5d196c4171815ce1c6a31e767393d043a50d11 Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Wed, 30 Aug 2023 15:07:22 +0000 Subject: [PATCH 11/20] Start 1.0.13 Hotfix: Version 1.0.12 -> 1.0.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index becb7f4..56dab97 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # or saved. Do not modify them directly here. # NB: PACKAGES is deprecated NAME = "ZenPacks.zenoss.PostgreSQL" -VERSION = "1.0.12" +VERSION = "1.0.13" AUTHOR = "Zenoss" LICENSE = "GPLv2" NAMESPACE_PACKAGES = ['ZenPacks', 'ZenPacks.zenoss'] From 433784c80e1112513b2e5764beb31fe3162b774f Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Wed, 30 Aug 2023 15:11:04 +0000 Subject: [PATCH 12/20] Move changes from develop to hotfix 1.0.13 Implements ZPS-8554. Move changes in scope of the ticket from develop to hotfix. --- ZenPacks/zenoss/PostgreSQL/__init__.py | 27 ++++++++++++++----- .../modeler/plugins/zenoss/PostgreSQL.py | 18 ++++++++++--- docs/body.md | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index 5963237..aac3b5b 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -12,6 +12,7 @@ ########################################################################### import logging + log = logging.getLogger('zen.PostgreSQL') import os @@ -22,6 +23,7 @@ from Products.ZenRelations.RelSchema import ToManyCont, ToOne from Products.ZenUtils.Utils import monkeypatch, zenPath + class ZenPack(ZenPackBase): packZProperties = [ ('zPostgreSQLPort', 5432, 'int'), @@ -29,8 +31,16 @@ class ZenPack(ZenPackBase): ('zPostgreSQLPassword', '', 'password'), ('zPostgreSQLUseSSL', False, 'boolean'), ('zPostgreSQLDefaultDB', 'postgres', 'string'), + ('zPostgreSQLTableRegex', '', 'string'), ] + packZProperties_data = { + 'zPostgreSQLTableRegex': { + 'description': "Filter tables from all databases if name matches regex provided", + 'label': "Regex Table Filter", + 'type': "string"}, + } + def install(self, app): super(ZenPack, self).install(app) self.patchPostgreSQLDriver() @@ -40,10 +50,10 @@ def remove(self, app, leaveObjects=False): if not leaveObjects: # Remove our custom relations addition. Device._relations = tuple( - [ x for x in Device._relations if x[0] != 'pgDatabases' ]) + [x for x in Device._relations if x[0] != 'pgDatabases']) self.updateExistingRelations(app.zport.dmd) - + # Revert all pg8000 library patches self.patchPostgreSQLDriver(revert=True) @@ -57,21 +67,22 @@ def updateExistingRelations(self, dmd): def patchPostgreSQLDriver(self, revert=False): log.info('Patching pg8000 core library') patch_dir = self.path('lib') - + # Getting a list of all patches which will be applied patches_list = [file for file in os.listdir(patch_dir) if file.endswith('.patch')] - - cmd = "patch -p0 -d %s -i %s" if not revert else "patch -p0 -R -d %s -i %s" + + cmd = "patch -p0 -d %s -i %s" if not revert else "patch -p0 -R -d %s -i %s" for patch in patches_list: os.system(cmd % ( os.path.join(patch_dir, 'pg8000'), os.path.join(patch_dir, patch) )) + # Allow PostgreSQL databases to be related to any device. Device._relations += ( ('pgDatabases', ToManyCont(ToOne, - 'ZenPacks.zenoss.PostgreSQL.Database.Database', 'server')), + 'ZenPacks.zenoss.PostgreSQL.Database.Database', 'server')), ) # We need to filter components by id instead of name. @@ -79,6 +90,7 @@ def patchPostgreSQLDriver(self, revert=False): "\"(device = '%s' and component = '%s')\"" " % (me.device().getDmdKey(), me.id)") + @monkeypatch('Products.ZenModel.Device.Device') def setPostgreSQL(self, active=True): if not active: @@ -96,11 +108,12 @@ def setPostgreSQL(self, active=True): if device.zCommandCommandTimeout < 180: device.setZenProperty('zCommandCommandTimeout', 180) + @monkeypatch('Products.ZenModel.Device.Device') def getPostgreSQL(self): device = self.primaryAq() if 'PostgreSQLServer' in device.zDeviceTemplates \ - and device.zCommandCommandTimeout >= 180: + and device.zCommandCommandTimeout >= 180: return True return False diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 9649e74..c870c7f 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -12,6 +12,8 @@ ########################################################################### import logging +import re + log = logging.getLogger('zen.PostgreSQL') from Products.DataCollector.plugins.CollectorPlugin import PythonPlugin @@ -20,6 +22,7 @@ from ZenPacks.zenoss.PostgreSQL.util import PgHelper + class PostgreSQL(PythonPlugin): deviceProperties = PythonPlugin.deviceProperties + ( 'zPostgreSQLPort', @@ -27,6 +30,7 @@ class PostgreSQL(PythonPlugin): 'zPostgreSQLPassword', 'zPostgreSQLUseSSL', 'zPostgreSQLDefaultDB', + 'zPostgreSQLTableRegex', ) def collect(self, device, unused): @@ -39,6 +43,7 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} + pattern = re.compile(getattr(device, 'zPostgreSQLTableRegex', '').strip()) log.info("Getting database list") try: @@ -55,8 +60,15 @@ def collect(self, device, unused): log.info("Getting tables list for {0}".format(dbName)) try: - results['databases'][dbName]['tables'] = \ - pg.getTablesInDatabase(dbName) + + tables = pg.getTablesInDatabase(dbName) + if pattern.pattern: + for key in tables.keys(): + if not pattern.match(key): + del tables[key] + + results['databases'][dbName]['tables'] = tables + except Exception, ex: log.warn("Error getting tables list for {0}: {1}".format( dbName, ex)) @@ -69,7 +81,7 @@ def process(self, devices, results, unused): if results is None: return None - maps = [ self.objectMap(dict(setPostgreSQL=True)) ] + maps = [self.objectMap(dict(setPostgreSQL=True))] databases = [] for dbName, dbDetail in results['databases'].items(): diff --git a/docs/body.md b/docs/body.md index 2e54c86..fbaa2bb 100644 --- a/docs/body.md +++ b/docs/body.md @@ -82,6 +82,7 @@ individual devices. - *zPostgreSQLUsername* - Must be a superuser. Default: postgres - *zPostgreSQLPassword* - Password for user. No default. - *zPostgreSQLDefaultDB* - Default database. Default: postgres + - *zPostgreSQLTableRegex* - Filter tables of all databases with Regex. Default: "" In addition to setting these properties you must add the ''zenoss.PostgreSQL'' modeler plugin to a device class or individual device. This modeler plugin will From ab03562734c9c96bcf21a94b1a1b6d8f722a4d95 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Thu, 31 Aug 2023 17:47:08 +0000 Subject: [PATCH 13/20] ZPS-8554: filter table names with regex lines Zprop --- ZenPacks/zenoss/PostgreSQL/__init__.py | 8 ++++++ .../modeler/plugins/zenoss/PostgreSQL.py | 12 ++++++++- ZenPacks/zenoss/PostgreSQL/util.py | 25 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/__init__.py b/ZenPacks/zenoss/PostgreSQL/__init__.py index 5963237..389868f 100644 --- a/ZenPacks/zenoss/PostgreSQL/__init__.py +++ b/ZenPacks/zenoss/PostgreSQL/__init__.py @@ -29,8 +29,16 @@ class ZenPack(ZenPackBase): ('zPostgreSQLPassword', '', 'password'), ('zPostgreSQLUseSSL', False, 'boolean'), ('zPostgreSQLDefaultDB', 'postgres', 'string'), + ('zPostgreSQLTableRegex', [], 'lines'), ] + packZProperties_data = { + 'zPostgreSQLTableRegex': { + 'description': "List of regular expressions (matched against table names) to control which tables are NOT modeled from All databases.", + 'label': "Regex Table Filter", + 'type': "lines" }, + } + def install(self, app): super(ZenPack, self).install(app) self.patchPostgreSQLDriver() diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 1a872df..9a4e44d 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -18,7 +18,7 @@ from Products.DataCollector.plugins.DataMaps import ObjectMap, RelationshipMap from Products.ZenUtils.Utils import prepId -from ZenPacks.zenoss.PostgreSQL.util import PgHelper +from ZenPacks.zenoss.PostgreSQL.util import PgHelper, exclude_patterns_list, is_suppressed class PostgreSQL(PythonPlugin): deviceProperties = PythonPlugin.deviceProperties + ( @@ -39,6 +39,7 @@ def collect(self, device, unused): device.zPostgreSQLDefaultDB) results = {} + exclude_patterns = exclude_patterns_list(getattr(device, 'zPostgreSQLTableRegex', [])) log.info("Getting database list") try: @@ -56,6 +57,15 @@ def collect(self, device, unused): log.info("Getting tables list for {0}".format(dbName)) try: results['databases'][dbName]['tables'] = pg.getTablesInDatabase(dbName) + + tables = pg.getTablesInDatabase(dbName) + if pattern.pattern: + for key in tables.keys(): + if is_suppressed(key, exclude_patterns): + del tables[key] + + results['databases'][dbName]['tables'] = tables + except Exception, ex: log.warn("Error getting tables list for {0}: {1}".format( dbName, ex)) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index d701d2e..7de85d2 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -545,3 +545,28 @@ def getTableStatsForDatabase(self, db): cursor.close() return tableStats + +def exclude_patterns_list(excludes): + exclude_patterns = [] + + for exclude in excludes: + exclude = exclude.strip() + if exclude == "" or exclude.startswith("#"): + continue + + try: + exclude_patterns.append(re.compile(exclude)) + except Exception: + LOG.warn("Invalid zPostgreSQLTableRegex value: '%s', this modeling filter will not be applied.", exclude) + continue + + return exclude_patterns + + +def is_suppressed(item, exclude_patterns): + + for exclude_pattern in exclude_patterns: + if exclude_pattern.search(item): + return True + + return False From 2431adcd8112f1329b714ea252477983a3907fc7 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Thu, 31 Aug 2023 18:26:28 +0000 Subject: [PATCH 14/20] ZPS-8554: check against exclude_patterns --- ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py index 9a4e44d..784d93d 100644 --- a/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py +++ b/ZenPacks/zenoss/PostgreSQL/modeler/plugins/zenoss/PostgreSQL.py @@ -59,7 +59,7 @@ def collect(self, device, unused): results['databases'][dbName]['tables'] = pg.getTablesInDatabase(dbName) tables = pg.getTablesInDatabase(dbName) - if pattern.pattern: + if exclude_patterns: for key in tables.keys(): if is_suppressed(key, exclude_patterns): del tables[key] From 325f525c9252321e45f6aa5013312c532a9e3dc6 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Tue, 5 Sep 2023 06:41:55 +0000 Subject: [PATCH 15/20] ZPS-8554: add missing imports --- ZenPacks/zenoss/PostgreSQL/util.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index d701d2e..288d14c 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -15,7 +15,9 @@ import math import sys import time - +import re +import logging +LOG = logging.getLogger('zen.Nutanix.utils') def addLocalLibPath(): """ @@ -545,3 +547,28 @@ def getTableStatsForDatabase(self, db): cursor.close() return tableStats + +def exclude_patterns_list(excludes): + exclude_patterns = [] + + for exclude in excludes: + exclude = exclude.strip() + if exclude == "" or exclude.startswith("#"): + continue + + try: + exclude_patterns.append(re.compile(exclude)) + except Exception: + LOG.warn("Invalid zPostgreSQLTableRegex value: '%s', this modeling filter will not be applied.", exclude) + continue + + return exclude_patterns + + +def is_suppressed(item, exclude_patterns): + + for exclude_pattern in exclude_patterns: + if exclude_pattern.search(item): + return True + + return False From dbaae0208402e75ce7cf853133cee4049056322c Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Tue, 5 Sep 2023 06:46:02 +0000 Subject: [PATCH 16/20] ZPS-8554: update logger name --- ZenPacks/zenoss/PostgreSQL/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZenPacks/zenoss/PostgreSQL/util.py b/ZenPacks/zenoss/PostgreSQL/util.py index 288d14c..8a5823c 100644 --- a/ZenPacks/zenoss/PostgreSQL/util.py +++ b/ZenPacks/zenoss/PostgreSQL/util.py @@ -17,7 +17,7 @@ import time import re import logging -LOG = logging.getLogger('zen.Nutanix.utils') +LOG = logging.getLogger('zen.PostgreSQL.utils') def addLocalLibPath(): """ From e5751327bcd1ebd48eb779368475b9e9b4f13a16 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Wed, 6 Sep 2023 18:14:42 +0000 Subject: [PATCH 17/20] ZPS-8692: check for db before device() --- ZenPacks/zenoss/PostgreSQL/Table.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ZenPacks/zenoss/PostgreSQL/Table.py b/ZenPacks/zenoss/PostgreSQL/Table.py index 4733463..345ff30 100644 --- a/ZenPacks/zenoss/PostgreSQL/Table.py +++ b/ZenPacks/zenoss/PostgreSQL/Table.py @@ -60,5 +60,7 @@ class Table(DeviceComponent, ManagedEntity, CollectedOrModeledMixin): event_key = "ComponentId" def device(self): - return self.database().device() - + db = self.database() + if db: + return db.device() + return None From 8e6db3ebe43beef1894d36c35fb46c15ab3ac060 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Wed, 6 Sep 2023 18:20:57 +0000 Subject: [PATCH 18/20] ZPS-8692: ZPS-8688: update docs --- docs/body.md | 4 ++++ docs/releases.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/body.md b/docs/body.md index 2e54c86..fe1fe8a 100644 --- a/docs/body.md +++ b/docs/body.md @@ -164,6 +164,10 @@ zSnmpMonitorIgnore property to True and remodel. Changes --------------- +1.0.13 + +* Make PostgreSQL table modeling optional (ZPS-8554) + 1.0.12 * Resolved isuue with error in pg8000 library on Ubuntu OS (ZPS-7424) diff --git a/docs/releases.md b/docs/releases.md index e9a3fce..424476f 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,6 +1,10 @@ Releases -------- +Version 1.0.13-Download +Released on 2023/9/7 +Tested with Zenoss 6.7.0, Zenoss Cloud and Service Impact 5.6.0 + Version 1.0.12-Download Released on 2021/1/14 Compatible with Zenoss 6.4.1 - 6.5.0, Zenoss Cloud and Service Impact 5.5.3 From 0eef3a8c7181065fd8ea40995e8b900e6b5768c3 Mon Sep 17 00:00:00 2001 From: zendwhiting Date: Thu, 7 Sep 2023 14:01:12 +0000 Subject: [PATCH 19/20] ZPS-8692: ZPS-8688: fix changelog --- docs/body.md | 1 + docs/releases.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/body.md b/docs/body.md index fe1fe8a..07362c3 100644 --- a/docs/body.md +++ b/docs/body.md @@ -167,6 +167,7 @@ Changes 1.0.13 * Make PostgreSQL table modeling optional (ZPS-8554) +* Tested with Zenoss Cloud, Zenoss 6.7.0 and Service Impact 5.6.0 1.0.12 diff --git a/docs/releases.md b/docs/releases.md index 424476f..80ce64a 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -3,7 +3,7 @@ Releases Version 1.0.13-Download Released on 2023/9/7 -Tested with Zenoss 6.7.0, Zenoss Cloud and Service Impact 5.6.0 +Compatible with Zenoss 6.x, Zenoss Cloud and Service Impact 5.6.1 Version 1.0.12-Download Released on 2021/1/14 From 4c54d4ef9cd09eb0c6e6e37c943d84f9528f8fba Mon Sep 17 00:00:00 2001 From: vsaliieva Date: Tue, 12 Sep 2023 15:00:51 +0000 Subject: [PATCH 20/20] Finish 1.0.13 Hotfix: Version 1.0.12 -> 1.0.13 --- docs/body.md | 2 +- docs/releases.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/body.md b/docs/body.md index 1a83895..84ffdba 100644 --- a/docs/body.md +++ b/docs/body.md @@ -168,7 +168,7 @@ Changes 1.0.13 * Make PostgreSQL table modeling optional (ZPS-8554) -* Tested with Zenoss Cloud, Zenoss 6.7.0 and Service Impact 5.6.0 +* Tested with Zenoss Cloud, Zenoss 6.7.0 and Service Impact 5.6.1 1.0.12 diff --git a/docs/releases.md b/docs/releases.md index 80ce64a..03599f0 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,7 +2,7 @@ Releases -------- Version 1.0.13-Download -Released on 2023/9/7 +Released on 2023/9/12 Compatible with Zenoss 6.x, Zenoss Cloud and Service Impact 5.6.1 Version 1.0.12-Download