From 08a03e712224d20e7faef7f15dc89e536c9dec35 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 26 Sep 2024 16:03:27 -0500 Subject: [PATCH] Defer creating the event client and queue. The event queue initializer looks up event transformers too early so defer creating the queue until after connecting to ZenHub. ZEN-35073 --- Products/ZenEvents/zentrap/app.py | 3 +- Products/ZenEvents/zentrap/filterspec.py | 34 +- Products/ZenEvents/zentrap/handlers.py | 7 +- Products/ZenEvents/zentrap/receiver.py | 2 +- .../zentrap/tests/test_filterspec.py | 436 +++++++++++------- Products/ZenEvents/zentrap/trapfilter.py | 21 +- Products/ZenHub/PBDaemon.py | 28 +- Products/ZenHub/tests/test_PBDaemon.py | 5 +- 8 files changed, 326 insertions(+), 210 deletions(-) diff --git a/Products/ZenEvents/zentrap/app.py b/Products/ZenEvents/zentrap/app.py index f20ad7821d..a28d54a112 100644 --- a/Products/ZenEvents/zentrap/app.py +++ b/Products/ZenEvents/zentrap/app.py @@ -57,7 +57,7 @@ class TrapDaemon(PBDaemon): def __init__(self, *args, **kwargs): super(TrapDaemon, self).__init__(*args, **kwargs) - self.configCycleInterval = 20 * 60 # seconds + self.configCycleInterval = 2 * 60 # seconds self.cycleInterval = 5 * 60 # seconds self.__lastCounterEventTime = time.time() @@ -233,7 +233,6 @@ def _start_receiver(self): reactor.addSystemEventTrigger( "before", "shutdown", self._receiver.stop ) - reactor.addSystemEventTrigger( "after", "shutdown", self._displayStatistics ) diff --git a/Products/ZenEvents/zentrap/filterspec.py b/Products/ZenEvents/zentrap/filterspec.py index 038b54e035..4b7172d3ba 100644 --- a/Products/ZenEvents/zentrap/filterspec.py +++ b/Products/ZenEvents/zentrap/filterspec.py @@ -100,7 +100,7 @@ def update_from_string(self, trapFilters, reset=True): self._filtersDefined = 0 != numFiltersDefined if self._filtersDefined: log.debug( - "Finished reading filter configuration. Lines parsed:%s, " + "finished reading filter configuration. Lines parsed:%s, " "Filters defined:%s [v1Traps:%d, v1Filters:%d, " "v2Filters:%d]", lineNumber, @@ -110,13 +110,13 @@ def update_from_string(self, trapFilters, reset=True): len(self._v2Filters), ) else: - log.warn("No zentrap filters defined.") + log.warn("no zentrap filters defined.") return events def _reset(self): - self._v1Traps = {} - self._v1Filters = {} - self._v2Filters = {} + self._v1Traps.clear() + self._v1Filters.clear() + self._v2Filters.clear() self._filtersDefined = False def _parseFilterDefinition(self, line, lineNumber): @@ -165,7 +165,7 @@ def _parseFilterDefinition(self, line, lineNumber): return None except Exception: errorMessage = ( - "Could not compile collector expression {!r} on " + "could not compile collector expression {!r} on " "line {}".format(collectorRegex, lineNumber) ) log.error(errorMessage) @@ -439,13 +439,14 @@ def __init__( self.genericTrap = genericTrap def __eq__(self, other): - if isinstance(other, GenericTrapFilterDefinition): - return self.genericTrap == other.genericTrap - else: - return False + if not isinstance(other, GenericTrapFilterDefinition): + return NotImplemented + return self.genericTrap == other.genericTrap def __ne__(self, other): - return not self.__eq__(other) + if not isinstance(other, GenericTrapFilterDefinition): + return NotImplemented + return self.genericTrap != other.genericTrap def __hash__(self): return hash(self.genericTrap) @@ -462,13 +463,14 @@ def levels(self): return countOidLevels(self.oid) def __eq__(self, other): - if isinstance(other, OIDBasedFilterDefinition): - return self.oid == other.oid - else: - return False + if not isinstance(other, OIDBasedFilterDefinition): + return NotImplemented + return self.oid == other.oid def __ne__(self, other): - return not self.__eq__(other) + if not isinstance(other, OIDBasedFilterDefinition): + return NotImplemented + return self.oid != other.oid def __hash__(self): return hash(self.oid) diff --git a/Products/ZenEvents/zentrap/handlers.py b/Products/ZenEvents/zentrap/handlers.py index 212db876f0..bb2212a1af 100644 --- a/Products/ZenEvents/zentrap/handlers.py +++ b/Products/ZenEvents/zentrap/handlers.py @@ -86,16 +86,15 @@ def __call__(self, addr, pdu, starttime): ) result["zenoss.trap_source_ip"] = addr[0] + community = self.getCommunity(pdu) + self.sendTrapEvent(result, community, eventType, starttime) log.debug( - "asyncHandleTrap: eventType=%s oid=%s snmpVersion=%s", + "handled trap event-type=%s oid=%s snmp-version=%s", eventType, result["oid"], result["snmpVersion"], ) - community = self.getCommunity(pdu) - self.sendTrapEvent(result, community, eventType, starttime) - def sendTrapEvent(self, result, community, eventType, starttime): summary = "snmp trap %s" % eventType log.debug(summary) diff --git a/Products/ZenEvents/zentrap/receiver.py b/Products/ZenEvents/zentrap/receiver.py index 3b1130d577..e3bdc907ca 100644 --- a/Products/ZenEvents/zentrap/receiver.py +++ b/Products/ZenEvents/zentrap/receiver.py @@ -25,7 +25,7 @@ sockaddr_in6, ) -log = logging.getLogger("zen.zentrap.server") +log = logging.getLogger("zen.zentrap.receiver") class Receiver(object): diff --git a/Products/ZenEvents/zentrap/tests/test_filterspec.py b/Products/ZenEvents/zentrap/tests/test_filterspec.py index db472a894f..a87af606e4 100644 --- a/Products/ZenEvents/zentrap/tests/test_filterspec.py +++ b/Products/ZenEvents/zentrap/tests/test_filterspec.py @@ -7,6 +7,8 @@ # ############################################################################## +from __future__ import print_function + # runtests -v -t unit Products.ZenEvents -m testTrapFilter import logging @@ -25,74 +27,74 @@ class OIDBasedFilterDefinitionTest(TestCase): def testEQByOID(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") - self.assert_(base1 == base2) + self.assertEqual(base1, base2) def testEQByOIDFails(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = OIDBasedFilterDefinition(0, "include", "5.4.3.2.1") - self.assert_(base1 != base2) + self.assertNotEqual(base1, base2) def testEQByOIDIgnoresAction(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = OIDBasedFilterDefinition(0, "exclude", "1.2.3.4.5") - self.assert_(base1 == base2) + self.assertEqual(base1, base2) def testEQByOIDFailsForDifferentClass(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = BaseFilterDefinition(0, "include") - self.assert_(base1 != base2) + self.assertNotEqual(base1, base2) def testHash(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") - self.assert_(base1.__hash__() == base2.__hash__()) + self.assertEqual(hash(base1), hash(base2)) def testHashFails(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = OIDBasedFilterDefinition(0, "include", "5.4.3.2.1") - self.assert_(base1.__hash__() != base2.__hash__()) + self.assertNotEqual(hash(base1), hash(base2)) def testHashIgnoresAction(self): base1 = OIDBasedFilterDefinition(0, "include", "1.2.3.4.5") base2 = OIDBasedFilterDefinition(0, "exclude", "1.2.3.4.5") - self.assert_(base1.__hash__() == base2.__hash__()) + self.assertEqual(hash(base1), hash(base2)) class GenericTrapFilterDefinitionTest(TestCase): def testEQByOID(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = GenericTrapFilterDefinition(0, "include", "1") - self.assert_(base1 == base2) + self.assertEqual(base1, base2) def testEQByOIDFails(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = GenericTrapFilterDefinition(0, "include", "5") - self.assert_(base1 != base2) + self.assertNotEqual(base1, base2) def testEQByOIDIgnoresAction(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = GenericTrapFilterDefinition(0, "exclude", "1") - self.assert_(base1 == base2) + self.assertEqual(base1, base2) def testEQByOIDFailsForDifferentClass(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = BaseFilterDefinition(0, "include") - self.assert_(base1 != base2) + self.assertNotEqual(base1, base2) def testHash(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = GenericTrapFilterDefinition(0, "include", "1") - self.assertEquals(base1.__hash__(), base2.__hash__()) + self.assertEqual(hash(base1), hash(base2)) def testHashFails(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = GenericTrapFilterDefinition(0, "include", "2") - self.assertNotEquals(base1.__hash__(), base2.__hash__()) + self.assertNotEqual(hash(base1), hash(base2)) def testHashIgnoresAction(self): base1 = GenericTrapFilterDefinition(0, "include", "1") base2 = GenericTrapFilterDefinition(0, "exclude", "1") - self.assert_(base1.__hash__() == base2.__hash__()) + self.assertEqual(hash(base1), hash(base2)) class FilterSpecificationTest(TestCase): @@ -105,99 +107,99 @@ def tearDown(t): def testValidateOIDForGlob(t): results = t.spec._validateOID("*") - t.assertEquals(results, None) + t.assertEqual(results, None) results = t.spec._validateOID("1.2.*") - t.assertEquals(results, None) + t.assertEqual(results, None) def testValidateOIDFailsForEmptyString(t): results = t.spec._validateOID("") - t.assertEquals(results, "Empty OID is invalid") + t.assertEqual(results, "Empty OID is invalid") def testValidateOIDFailsForSimpleNumber(t): results = t.spec._validateOID("123") - t.assertEquals(results, "At least one '.' required") + t.assertEqual(results, "At least one '.' required") def testValidateOIDFailsForInvalidChars(t): results = t.spec._validateOID("1.2.3-5.*") - t.assertEquals( + t.assertEqual( results, "Invalid character found; only digits, '.' and '*' allowed", ) def testValidateOIDFailsForDoubleDots(t): results = t.spec._validateOID("1.2..3") - t.assertEquals(results, "Consecutive '.'s not allowed") + t.assertEqual(results, "Consecutive '.'s not allowed") def testValidateOIDFailsForInvalidGlobbing(t): results = t.spec._validateOID("1.2.3.*.5.*") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("1.*.5") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("1.5*") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("*.") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("*.1") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("*.*") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("5*") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID("*5") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) results = t.spec._validateOID(".*") - t.assertEquals( + t.assertEqual( results, "When using '*', only a single '*' at the end of OID is allowed", ) def testParseFilterDefinitionForEmptyLine(t): results = t.spec._parseFilterDefinition("", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, "Incomplete filter definition") + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, "Incomplete filter definition") def testParseFilterDefinitionForIncompleteLine(t): results = t.spec._parseFilterDefinition("a b", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, "Incomplete filter definition") + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, "Incomplete filter definition") def testParseFilterDefinitionForInvalidAction(t): results = t.spec._parseFilterDefinition("invalid V1 ignored", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals( + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual( results, "Invalid action 'invalid'; the only valid actions are " "'include' or 'exclude'", @@ -205,8 +207,8 @@ def testParseFilterDefinitionForInvalidAction(t): def testParseFilterDefinitionForInvalidVersion(t): results = t.spec._parseFilterDefinition("include V4 ignored", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals( + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual( results, "Invalid SNMP version 'V4'; the only valid versions are " "'v1' or 'v2' or 'v3'", @@ -214,170 +216,170 @@ def testParseFilterDefinitionForInvalidVersion(t): def testParseFilterDefinitionForInvalidV1Definition(t): results = t.spec._parseFilterDefinition("include V1 .", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") def testParseFilterDefinitionForCaseInsensitiveDefinition(t): results = t.spec._parseFilterDefinition("InClude v1 3", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, None) + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, None) def testParseFilterDefinitionForValidV1Definition(t): results = t.spec._parseFilterDefinition("include V1 3", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, None) + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, None) def testParseFilterDefinitionForInvalidV2Definition(t): results = t.spec._parseFilterDefinition("include V2 .", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") def testParseFilterDefinitionForValidV2Definition(t): results = t.spec._parseFilterDefinition("include V2 .1.3.6.1.4.*", 99) - # t.assertEquals(t.spec._eventService.sendEvent.called, False) - t.assertEquals(results, None) + # t.assertEqual(t.spec._eventService.sendEvent.called, False) + t.assertEqual(results, None) def testParseFilterDefinitionForInvalidV3Definition(t): results = t.spec._parseFilterDefinition("include V3 .", 99) - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") def testParseFilterDefinitionForValidV3Definition(t): results = t.spec._parseFilterDefinition("include V3 .1.3.6.1.4.*", 99) - t.assertEquals(results, None) + t.assertEqual(results, None) def testParseV1FilterDefinitionForGenericTrap(t): results = t.spec._parseV1FilterDefinition(99, "include", ["0"], ".*") - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 1) - t.assertEquals(len(t.spec._v1Filters), 0) - t.assertEquals(len(t.spec._v2Filters), 0) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 1) + t.assertEqual(len(t.spec._v1Filters), 0) + t.assertEqual(len(t.spec._v2Filters), 0) genericTrapDefinition = t.spec._v1Traps["0"] t.assertIsNotNone(genericTrapDefinition) - t.assertEquals(genericTrapDefinition.lineNumber, 99) - t.assertEquals(genericTrapDefinition.action, "include") - t.assertEquals(genericTrapDefinition.genericTrap, "0") + t.assertEqual(genericTrapDefinition.lineNumber, 99) + t.assertEqual(genericTrapDefinition.action, "include") + t.assertEqual(genericTrapDefinition.genericTrap, "0") # Now add another to make sure we can parse more than one results = t.spec._parseV1FilterDefinition(100, "exclude", ["5"], ".*") - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 2) - t.assertEquals(len(t.spec._v1Filters), 0) - t.assertEquals(len(t.spec._v2Filters), 0) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 2) + t.assertEqual(len(t.spec._v1Filters), 0) + t.assertEqual(len(t.spec._v2Filters), 0) genericTrapDefinition = t.spec._v1Traps["5"] t.assertIsNotNone(genericTrapDefinition) - t.assertEquals(genericTrapDefinition.lineNumber, 100) - t.assertEquals(genericTrapDefinition.action, "exclude") - t.assertEquals(genericTrapDefinition.genericTrap, "5") + t.assertEqual(genericTrapDefinition.lineNumber, 100) + t.assertEqual(genericTrapDefinition.action, "exclude") + t.assertEqual(genericTrapDefinition.genericTrap, "5") def testParseV1FilterDefinitionEnterpriseSpecificTrap(t): results = t.spec._parseV1FilterDefinition( 99, "include", ["1.2.3.*"], ".*" ) - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 1) - t.assertEquals(len(t.spec._v2Filters), 0) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 1) + t.assertEqual(len(t.spec._v2Filters), 0) oidLevels = 4 mapByLevel = t.spec._v1Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 1) + t.assertEqual(len(mapByLevel), 1) filterDef = mapByLevel["1.2.3.*"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 99) - t.assertEquals(filterDef.action, "include") - t.assertEquals(filterDef.oid, "1.2.3.*") - t.assertEquals(filterDef.specificTrap, None) + t.assertEqual(filterDef.lineNumber, 99) + t.assertEqual(filterDef.action, "include") + t.assertEqual(filterDef.oid, "1.2.3.*") + t.assertEqual(filterDef.specificTrap, None) # Add another 4-level OID results = t.spec._parseV1FilterDefinition( 100, "exclude", ["1.2.3.4", "25"], ".*" ) - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 1) - t.assertEquals(len(t.spec._v2Filters), 0) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 1) + t.assertEqual(len(t.spec._v2Filters), 0) mapByLevel = t.spec._v1Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 2) + t.assertEqual(len(mapByLevel), 2) filterDef = mapByLevel["1.2.3.4-25"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 100) - t.assertEquals(filterDef.action, "exclude") - t.assertEquals(filterDef.oid, "1.2.3.4") - t.assertEquals(filterDef.specificTrap, "25") + t.assertEqual(filterDef.lineNumber, 100) + t.assertEqual(filterDef.action, "exclude") + t.assertEqual(filterDef.oid, "1.2.3.4") + t.assertEqual(filterDef.specificTrap, "25") # Add a different specific trap for the same OID results = t.spec._parseV1FilterDefinition( 101, "exclude", ["1.2.3.4", "99"], ".*" ) - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 1) - t.assertEquals(len(t.spec._v2Filters), 0) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 1) + t.assertEqual(len(t.spec._v2Filters), 0) mapByLevel = t.spec._v1Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 3) + t.assertEqual(len(mapByLevel), 3) filterDef = mapByLevel["1.2.3.4-99"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 101) - t.assertEquals(filterDef.action, "exclude") - t.assertEquals(filterDef.oid, "1.2.3.4") - t.assertEquals(filterDef.specificTrap, "99") + t.assertEqual(filterDef.lineNumber, 101) + t.assertEqual(filterDef.action, "exclude") + t.assertEqual(filterDef.oid, "1.2.3.4") + t.assertEqual(filterDef.specificTrap, "99") # Add another single-level OID results = t.spec._parseV1FilterDefinition(101, "exclude", ["*"], ".*") - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 2) - t.assertEquals(len(t.spec._v2Filters), 0) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 2) + t.assertEqual(len(t.spec._v2Filters), 0) oidLevels = 1 mapByLevel = t.spec._v1Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 1) + t.assertEqual(len(mapByLevel), 1) filterDef = mapByLevel["*"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 101) - t.assertEquals(filterDef.action, "exclude") - t.assertEquals(filterDef.oid, "*") - t.assertEquals(filterDef.specificTrap, None) + t.assertEqual(filterDef.lineNumber, 101) + t.assertEqual(filterDef.action, "exclude") + t.assertEqual(filterDef.oid, "*") + t.assertEqual(filterDef.specificTrap, None) def testParseV1FilterDefinitionFailsForTooManyArgs(t): results = t.spec._parseV1FilterDefinition( 99, "include", ["0", "1", "2"], ".*" ) - t.assertEquals( + t.assertEqual( results, "Too many fields found; at most 4 fields allowed for V1 filters", ) def testParseV1FilterDefinitionFailsForEmptyOID(t): results = t.spec._parseV1FilterDefinition(99, "include", [], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") results = t.spec._parseV1FilterDefinition(99, "include", [""], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") results = t.spec._parseV1FilterDefinition(99, "include", ["."], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") results = t.spec._parseV1FilterDefinition(99, "include", ["..."], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") def testParseV1FilterDefinitionFailsForInvalidOID(t): results = t.spec._parseV1FilterDefinition( 99, "include", ["invalidOID"], ".*" ) - t.assertEquals( + t.assertEqual( results, "'invalidOID' is not a valid OID: Invalid character found; " "only digits, '.' and '*' allowed", @@ -385,24 +387,24 @@ def testParseV1FilterDefinitionFailsForInvalidOID(t): def testParseV1FilterDefinitionFailsForInvalidTrap(t): results = t.spec._parseV1FilterDefinition(99, "include", ["a"], ".*") - t.assertEquals(results, "Invalid generic trap 'a'; must be one of 0-5") + t.assertEqual(results, "Invalid generic trap 'a'; must be one of 0-5") results = t.spec._parseV1FilterDefinition(99, "include", ["6"], ".*") - t.assertEquals(results, "Invalid generic trap '6'; must be one of 0-5") + t.assertEqual(results, "Invalid generic trap '6'; must be one of 0-5") def testParseV1FilterDefinitionFailsForConflictingTrap(t): results = t.spec._parseV1FilterDefinition(99, "include", ["1"], ".*") - t.assertEquals(results, None) + t.assertEqual(results, None) results = t.spec._parseV1FilterDefinition(100, "include", ["1"], ".*") - t.assertEquals( + t.assertEqual( results, "Generic trap '1' conflicts with previous definition at line 99", ) # Verify we find a conflict for generic traps where the action differs results = t.spec._parseV1FilterDefinition(100, "exclude", ["1"], ".*") - t.assertEquals( + t.assertEqual( results, "Generic trap '1' conflicts with previous definition at line 99", ) @@ -411,12 +413,12 @@ def testParseV1FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV1FilterDefinition( 99, "include", [".1.3.6.1.4.5", "2"], ".*" ) - t.assertEquals(results, None) + t.assertEqual(results, None) results = t.spec._parseV1FilterDefinition( 100, "include", [".1.3.6.1.4.5", "2"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.5' conflicts with previous definition at line 99", ) @@ -425,7 +427,7 @@ def testParseV1FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV1FilterDefinition( 100, "exclude", [".1.3.6.1.4.5", "2"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.5' conflicts with previous definition at line 99", ) @@ -433,13 +435,13 @@ def testParseV1FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV1FilterDefinition( 101, "include", [".1.3.6.1.4.*"], ".*" ) - t.assertEquals(results, None) + t.assertEqual(results, None) # Verify we find a conflict for glob-based OIDs results = t.spec._parseV1FilterDefinition( 102, "include", [".1.3.6.1.4.*"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.*' conflicts with previous definition at line 101", ) @@ -449,7 +451,7 @@ def testParseV1FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV1FilterDefinition( 102, "exclude", [".1.3.6.1.4.*"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.*' conflicts with previous definition at line 101", ) @@ -458,13 +460,13 @@ def testParseV1FilterDefinitionFailsForEnterpriseSpecificGlob(t): results = t.spec._parseV1FilterDefinition( 99, "include", [".1.3.6.1.4.5.*", "23"], ".*" ) - t.assertEquals(results, "Specific trap not allowed with globbed OID") + t.assertEqual(results, "Specific trap not allowed with globbed OID") def testParseV1FilterDefinitionFailsForInvalidEnterpriseSpecificTrap(t): results = t.spec._parseV1FilterDefinition( 99, "include", [".1.3.6.1.4.5", "abc"], ".*" ) - t.assertEquals( + t.assertEqual( results, "Specific trap 'abc' invalid; must be non-negative integer", ) @@ -472,7 +474,7 @@ def testParseV1FilterDefinitionFailsForInvalidEnterpriseSpecificTrap(t): results = t.spec._parseV1FilterDefinition( 99, "include", [".1.3.6.1.4.5", "-1"], ".*" ) - t.assertEquals( + t.assertEqual( results, "Specific trap '-1' invalid; must be non-negative integer" ) @@ -480,92 +482,92 @@ def testParseV1FilterDefinitionForSpecificOid(t): results = t.spec._parseV1FilterDefinition( 99, "include", [".1.3.6.1.4.5"], ".*" ) - t.assertEquals(results, None) + t.assertEqual(results, None) def testParseV2FilterDefinition(t): results = t.spec._parseV2FilterDefinition( 99, "include", ["1.2.3.*"], ".*" ) - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 0) - t.assertEquals(len(t.spec._v2Filters), 1) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 0) + t.assertEqual(len(t.spec._v2Filters), 1) oidLevels = 4 mapByLevel = t.spec._v2Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 1) + t.assertEqual(len(mapByLevel), 1) filterDef = mapByLevel["1.2.3.*"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 99) - t.assertEquals(filterDef.action, "include") - t.assertEquals(filterDef.oid, "1.2.3.*") + t.assertEqual(filterDef.lineNumber, 99) + t.assertEqual(filterDef.action, "include") + t.assertEqual(filterDef.oid, "1.2.3.*") # Add another 4-level OID results = t.spec._parseV2FilterDefinition( 100, "exclude", ["1.2.3.4"], ".*" ) - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 0) - t.assertEquals(len(t.spec._v2Filters), 1) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 0) + t.assertEqual(len(t.spec._v2Filters), 1) mapByLevel = t.spec._v2Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 2) + t.assertEqual(len(mapByLevel), 2) filterDef = mapByLevel["1.2.3.4"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 100) - t.assertEquals(filterDef.action, "exclude") - t.assertEquals(filterDef.oid, "1.2.3.4") + t.assertEqual(filterDef.lineNumber, 100) + t.assertEqual(filterDef.action, "exclude") + t.assertEqual(filterDef.oid, "1.2.3.4") # Add another single-level OID results = t.spec._parseV2FilterDefinition(101, "exclude", ["*"], ".*") - t.assertEquals(results, None) - t.assertEquals(len(t.spec._v1Traps), 0) - t.assertEquals(len(t.spec._v1Filters), 0) - t.assertEquals(len(t.spec._v2Filters), 2) + t.assertEqual(results, None) + t.assertEqual(len(t.spec._v1Traps), 0) + t.assertEqual(len(t.spec._v1Filters), 0) + t.assertEqual(len(t.spec._v2Filters), 2) oidLevels = 1 mapByLevel = t.spec._v2Filters[oidLevels] t.assertIsNotNone(mapByLevel) - t.assertEquals(len(mapByLevel), 1) + t.assertEqual(len(mapByLevel), 1) filterDef = mapByLevel["*"] t.assertIsNotNone(filterDef) - t.assertEquals(filterDef.lineNumber, 101) - t.assertEquals(filterDef.action, "exclude") - t.assertEquals(filterDef.oid, "*") + t.assertEqual(filterDef.lineNumber, 101) + t.assertEqual(filterDef.action, "exclude") + t.assertEqual(filterDef.oid, "*") def testParseV2FilterDefinitionFailsForTooManyArgs(t): results = t.spec._parseV2FilterDefinition( 99, "include", ["0", "1"], ".*" ) - t.assertEquals( + t.assertEqual( results, "Too many fields found; at most 3 fields allowed for V2 filters", ) def testParseV2FilterDefinitionFailsForEmptyOID(t): results = t.spec._parseV2FilterDefinition(99, "include", [], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") results = t.spec._parseV2FilterDefinition(99, "include", [""], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") results = t.spec._parseV2FilterDefinition(99, "include", ["."], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") results = t.spec._parseV2FilterDefinition(99, "include", ["..."], ".*") - t.assertEquals(results, "'' is not a valid OID: Empty OID is invalid") + t.assertEqual(results, "'' is not a valid OID: Empty OID is invalid") def testParseV2FilterDefinitionFailsForInvalidOID(t): results = t.spec._parseV2FilterDefinition( 99, "include", ["invalidOID"], ".*" ) - t.assertEquals( + t.assertEqual( results, "'invalidOID' is not a valid OID: Invalid character found; " "only digits, '.' and '*' allowed", @@ -575,12 +577,12 @@ def testParseV2FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV2FilterDefinition( 99, "include", [".1.3.6.1.4.5"], ".*" ) - t.assertEquals(results, None) + t.assertEqual(results, None) results = t.spec._parseV2FilterDefinition( 100, "include", [".1.3.6.1.4.5"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.5' conflicts with previous definition at line 99", ) @@ -589,7 +591,7 @@ def testParseV2FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV2FilterDefinition( 100, "exclude", [".1.3.6.1.4.5"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.5' conflicts with previous definition at line 99", ) @@ -597,13 +599,13 @@ def testParseV2FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV2FilterDefinition( 101, "include", [".1.3.6.1.4.*"], ".*" ) - t.assertEquals(results, None) + t.assertEqual(results, None) # Verify we find a conflict for glob-based OIDs results = t.spec._parseV2FilterDefinition( 102, "include", [".1.3.6.1.4.*"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.*' conflicts with previous definition at line 101", ) @@ -613,17 +615,125 @@ def testParseV2FilterDefinitionFailsForConflictingOID(t): results = t.spec._parseV2FilterDefinition( 102, "exclude", [".1.3.6.1.4.*"], ".*" ) - t.assertEquals( + t.assertEqual( results, "OID '1.3.6.1.4.*' conflicts with previous definition at line 101", ) -def test_suite(): - from unittest import TestSuite, makeSuite +class TestPersistentFilterSpecProperties(TestCase): + """ + Test that the `v1traps`, `v1filters`, and `v2filters` properties are + updated after a call to `update_from_string` with a different config. + """ + + _rules_v1 = "\n".join( + ( + "include v1 0", + "include v1 1", + "include v1 2", + "include v1 3", + "include v1 4", + "include v1 5", + "include v1 *", + "include v2 *", + ) + ) + + _rules_v2 = "\n".join( + ( + "include v1 0", + "include v1 1", + "exclude v1 2", + "include v1 3", + "include v1 4", + "include v1 5", + "include v1 *", + "include v2 *", + "exclude v1 1.3.6.1.4.1.9.9.41.2.0.*", + "exclude v2 1.3.6.1.4.1.9.9.43.1.3.*", + ) + ) - suite = TestSuite() - suite.addTest(makeSuite(OIDBasedFilterDefinitionTest)) - suite.addTest(makeSuite(GenericTrapFilterDefinitionTest)) - suite.addTest(makeSuite(FilterSpecificationTest)) - return suite + def setUp(t): + t.spec = FilterSpecification("localhost") + t.spec.update_from_string(t._rules_v1) + + def test_v1trap_property(t): + reference = t.spec.v1traps + original = dict(reference) + t.spec.update_from_string(t._rules_v2) + + t.assertEqual(len(original), len(t.spec.v1traps)) + t.assertEqual(len(reference), len(t.spec.v1traps)) + t.assertFalse( + all( + (kr == kl) and _eq_v1generic(vr, vl) + for (kr, vr), (kl, vl) in zip( + sorted(original.items()), sorted(t.spec.v1traps.items()) + ) + ) + ) + t.assertTrue( + all( + (kr == kl) and _eq_v1generic(vr, vl) + for (kr, vr), (kl, vl) in zip( + sorted(reference.items()), sorted(t.spec.v1traps.items()) + ) + ) + ) + + def test_v1filters_property(t): + reference = t.spec.v1filters + original = dict(reference) + t.spec.update_from_string(t._rules_v2) + + t.assertNotEqual(len(original), len(t.spec.v1filters)) + t.assertEqual(len(reference), len(t.spec.v1filters)) + t.assertTrue( + all( + (kr == kl) and _eq_v1enterprise(vr, vl) + for (kr, vr), (kl, vl) in zip( + sorted(reference.items()), sorted(t.spec.v1filters.items()) + ) + ) + ) + + def test_v2filters_property(t): + reference = t.spec.v2filters + original = dict(reference) + t.spec.update_from_string(t._rules_v2) + + t.assertNotEqual(len(original), len(t.spec.v2filters)) + t.assertEqual(len(reference), len(t.spec.v2filters)) + t.assertTrue( + all( + (kr == kl) and _eq_v2(vr, vl) + for (kr, vr), (kl, vl) in zip( + sorted(reference.items()), sorted(t.spec.v2filters.items()) + ) + ) + ) + + +def _eq_v1generic(lv, rv): + return (lv.genericTrap == rv.genericTrap) and (lv.action == rv.action) + + +def _eq_v1enterprise(lv, rv): + return all( + (k1 == k2) + and (v1.action == v2.action) + and (v1.oid == v2.oid) + and (v1.specificTrap == v2.specificTrap) + for (k1, v1), (k2, v2) in zip(sorted(lv.items()), sorted(rv.items())) + ) + + +def _eq_v2(lv, rv): + return all( + (k1 == k2) + and (v1.action == v2.action) + and (v1.oid == v2.oid) + for (k1, v1), (k2, v2) in zip(sorted(lv.items()), sorted(rv.items())) + ) diff --git a/Products/ZenEvents/zentrap/trapfilter.py b/Products/ZenEvents/zentrap/trapfilter.py index e657b6f76a..28896590bb 100644 --- a/Products/ZenEvents/zentrap/trapfilter.py +++ b/Products/ZenEvents/zentrap/trapfilter.py @@ -119,6 +119,7 @@ def _dropEvent(self, event): (tf for tf in self._filters if tf.is_valid(event)), None ) if trapfilter: + log.debug("using trap filter %r", trapfilter) return trapfilter(event) log.error("dropping unknown trap event=%r", event) return True @@ -179,7 +180,7 @@ def __call__(self, event): ) return True - result = _check_definitions( + return _check_definitions( ( getter() for getter in ( @@ -190,9 +191,6 @@ def __call__(self, event): ) ) ) - if result: - log.debug("drop specific v1 trap trap=%s", event) - return result def _getSpecificTrapDefinition(self, event, enterpriseOID): specificTrap = event.get("snmpV1SpecificTrap", None) @@ -201,14 +199,14 @@ def _getSpecificTrapDefinition(self, event, enterpriseOID): key = "".join([enterpriseOID, "-", str(specificTrap)]) definition = _findFilterByLevel(key, self._definitions) if definition: - log.debug("matched definition %s", definition) + log.debug("matched [specific-trap] definition %s", definition) return definition def _getWildCardDefinition(self, enterpriseOID): key = "".join([enterpriseOID, "-", "*"]) definition = _findFilterByLevel(key, self._definitions) if definition: - log.debug("matched definition %s", definition) + log.debug("matched [wildcard] definition %s", definition) return definition def _getGlobMatchDefinition(self, enterpriseOID): @@ -216,7 +214,7 @@ def _getGlobMatchDefinition(self, enterpriseOID): enterpriseOID, self._definitions ) if definition: - log.debug("matched definition %s", definition) + log.debug("matched [glob] definition %s", definition) return definition @@ -226,7 +224,7 @@ def is_valid(self, event): def __call__(self, event): oid = event["oid"] - result = _check_definitions( + return _check_definitions( ( getter() for getter in ( @@ -236,21 +234,18 @@ def __call__(self, event): ) ) ) - if result: - log.debug("drop v2 trap trap=%s", event) - return result def _getExactMatchDefinition(self, oid): # First, try an exact match on the OID definition = _findFilterByLevel(oid, self._definitions) if definition: - log.debug("matched definition %s", definition) + log.debug("matched [exact] definition %s", definition) return definition def _getGlobMatchDefinition(self, oid): definition = _findClosestGlobbedFilter(oid, self._definitions) if definition: - log.debug("matched definition %s", definition) + log.debug("matched [glob] definition %s", definition) return definition diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index 0ebbb718f5..e3cc239d5a 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -130,7 +130,6 @@ def __init__( for evt in self.startEvent, self.stopEvent: evt.update(details) - self.__eventqueue = EventQueueManager(self.options, self.log) self._metrologyReporter = None self.__publisher = publisher @@ -138,12 +137,8 @@ def __init__( self.__metric_writer = None self.__derivative_tracker = None - self.__eventclient = EventClient( - self.options, - self.__eventqueue, - self.generateEvent, - lambda: self.getService("EventService"), - ) + self.__eventqueue = None + self.__eventclient = None self.__recordQueuedEventsCountLoop = task.LoopingCall( self.__record_queued_events_count ) @@ -186,7 +181,7 @@ def services(self): return self.__zhclient.services def __record_queued_events_count(self): - if self.rrdStats.name: + if self.rrdStats.name and self.__eventqueue is not None: self.rrdStats.gauge("eventQueueLength", len(self.__eventqueue)) def generateEvent(self, event, **kw): @@ -273,13 +268,19 @@ def eventService(self): return self.getServiceNow("EventService") def sendEvents(self, events): + if self.__eventclient is None: + return return self.__eventclient.sendEvents(events) def sendHeartbeat(self, event): + if self.__eventclient is None: + return self.__eventclient.sendHeartbeat(event) @defer.inlineCallbacks def sendEvent(self, event, **kw): + if self.__eventclient is None: + return yield self.__eventclient.sendEvent(event, **kw) def getServiceNow(self, svcName): @@ -427,16 +428,23 @@ def _started(self): @defer.inlineCallbacks def _stop(self): - if self.options.cycle: + if self.__eventclient is not None: self.__eventclient.sendEvent(self.stopEvent) yield self.__eventclient.stop() self.log.debug("stopped event client") yield self.__zhclient.stop() def _setup_event_client(self): + self.__eventqueue = EventQueueManager(self.options, self.log) + self.__eventclient = EventClient( + self.options, + self.__eventqueue, + self.generateEvent, + lambda: self.getService("EventService"), + ) self.__eventclient.start() - self.__recordQueuedEventsCountLoop.start(2.0, now=False) self.__eventclient.sendEvent(self.startEvent) + self.__recordQueuedEventsCountLoop.start(2.0, now=False) self.log.info("started event client") def _setup_stats_recording(self): diff --git a/Products/ZenHub/tests/test_PBDaemon.py b/Products/ZenHub/tests/test_PBDaemon.py index b52045751d..6ff97d0199 100644 --- a/Products/ZenHub/tests/test_PBDaemon.py +++ b/Products/ZenHub/tests/test_PBDaemon.py @@ -82,7 +82,7 @@ def test___init__( ls = _getLocalServer.return_value ls.add_resource.assert_called_once_with("zenhub", ANY) - EventQueueManager.assert_called_with(PBDaemon.options, PBDaemon.log) + EventQueueManager.assert_not_called() # Check lots of attributes, should verify that they are needed t.assertEqual(pbd._thresholds, Thresholds.return_value) @@ -415,6 +415,7 @@ def test_stop(t): def test_sendEvents(t): ec = t.EventClient.return_value + t.pbd._setup_event_client() events = [{"name": "evt_a"}, {"name": "evt_b"}] d = t.pbd.sendEvents(events) @@ -426,6 +427,7 @@ def test_sendEvent(t): ec = t.EventClient.return_value sendEvent = Mock(name="sendEvent") ec.sendEvent = sendEvent + t.pbd._setup_event_client() event = {"name": "event"} d = t.pbd.sendEvent(event, newkey="newkey") @@ -458,6 +460,7 @@ def test_postStatisticsImpl(t): def test_postStatistics(t): ec = t.EventClient.return_value ec.counters = collections.Counter() + t.pbd._setup_event_client() # sets rrdStats, then calls postStatisticsImpl t.pbd.rrdStats = Mock(name="rrdStats", spec_set=["counter"]) ctrs = {"c1": 3, "c2": 5}