From 33cc04f7bfb1af606ef0ea5dcce40c87a1c85e23 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 5 May 2023 16:12:18 -0500 Subject: [PATCH 001/147] Finish reformatting code in DataCollector using black. --- .../ApplyDataMap/incrementalupdate.py | 2 +- .../ApplyDataMap/tests/test_add_directive.py | 9 +++------ .../ApplyDataMap/tests/test_datamaputils.py | 19 +++++++++++-------- .../tests/test_incrementalupdate.py | 6 +++++- .../DataCollector/ApplyDataMap/tests/utils.py | 1 - Products/DataCollector/Plugins.py | 4 +--- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/Products/DataCollector/ApplyDataMap/incrementalupdate.py b/Products/DataCollector/ApplyDataMap/incrementalupdate.py index 993d81abf1..513306f4ab 100644 --- a/Products/DataCollector/ApplyDataMap/incrementalupdate.py +++ b/Products/DataCollector/ApplyDataMap/incrementalupdate.py @@ -279,7 +279,7 @@ def _add(self): for objId, obj in self.relationship.objectItemsAll() if objId == self._target_id ), - _NOTSET + _NOTSET, ) if self._target is _NOTSET: changed = True diff --git a/Products/DataCollector/ApplyDataMap/tests/test_add_directive.py b/Products/DataCollector/ApplyDataMap/tests/test_add_directive.py index eb280b30f2..52c0ab4c93 100644 --- a/Products/DataCollector/ApplyDataMap/tests/test_add_directive.py +++ b/Products/DataCollector/ApplyDataMap/tests/test_add_directive.py @@ -13,12 +13,11 @@ from ..applydatamap import ApplyDataMap, ObjectMap -PATH = {'src': 'Products.DataCollector.ApplyDataMap.applydatamap'} +PATH = {"src": "Products.DataCollector.ApplyDataMap.applydatamap"} class TestImplicitAdd(BaseTestCase): - """Test ApplyDataMap directives. - """ + """Test ApplyDataMap directives.""" def afterSetUp(t): super(TestImplicitAdd, t).afterSetUp() @@ -52,7 +51,6 @@ def test_implicit_update_twice_with_same_data(t): class TestExplicitAdd(BaseTestCase): - def afterSetUp(t): super(TestExplicitAdd, t).afterSetUp() t.om1 = ObjectMap( @@ -125,8 +123,7 @@ def test_explicit_add_false_with_existing_device_with_changes(t): class TestAddSequence(BaseTestCase): - """Test ApplyDataMap directives. - """ + """Test ApplyDataMap directives.""" def afterSetUp(t): super(TestAddSequence, t).afterSetUp() diff --git a/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py b/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py index 600f0b989d..04e90b2f17 100644 --- a/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py +++ b/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py @@ -8,6 +8,9 @@ ############################################################################## from base64 import b64encode + +import six + from mock import Mock, sentinel, patch from Products.DataCollector.plugins.DataMaps import ObjectMap @@ -313,22 +316,22 @@ def test_encoding(t): ), "utf-8": ObjectMap( { - "a": u"\xe0".encode("utf-8"), - "b": u"\xe0".encode("utf-8"), - "c": u"\xe0".encode("utf-8"), + "a": six.text_type("\xe0").encode("utf-8"), + "b": six.text_type("\xe0").encode("utf-8"), + "c": six.text_type("\xe0").encode("utf-8"), } ), "latin-1": ObjectMap( { - "a": u"\xe0".encode("latin-1"), - "b": u"\xe0".encode("latin-1"), - "c": u"\xe0".encode("latin-1"), + "a": six.text_type("\xe0").encode("latin-1"), + "b": six.text_type("\xe0").encode("latin-1"), + "c": six.text_type("\xe0").encode("latin-1"), } ), "utf-16": ObjectMap( { - "a": u"\xff\xfeabcdef".encode("utf-16"), - "b": u"\xff\xfexyzwow".encode("utf-16"), + "a": six.text_type("\xff\xfeabcdef").encode("utf-16"), + "b": six.text_type("\xff\xfexyzwow").encode("utf-16"), # (water, z, G clef), UTF-16 encoded, # little-endian with BOM "c": r"\xff\xfe\x34\x6c\x7a\x00\x34\xd8\x13\xdd", diff --git a/Products/DataCollector/ApplyDataMap/tests/test_incrementalupdate.py b/Products/DataCollector/ApplyDataMap/tests/test_incrementalupdate.py index 52dbfd144b..db4155e938 100644 --- a/Products/DataCollector/ApplyDataMap/tests/test_incrementalupdate.py +++ b/Products/DataCollector/ApplyDataMap/tests/test_incrementalupdate.py @@ -60,7 +60,11 @@ def setup_mock_environment(t): t.relationship = Mock( name="relationship", spec_set=[ - t.target.id, "_getOb", "hasobject", "_setObject", "objectItemsAll" + t.target.id, + "_getOb", + "hasobject", + "_setObject", + "objectItemsAll", ], ) setattr(t.relationship, t.target.id, t.target) diff --git a/Products/DataCollector/ApplyDataMap/tests/utils.py b/Products/DataCollector/ApplyDataMap/tests/utils.py index e1de5f23d9..49ed59f2c3 100644 --- a/Products/DataCollector/ApplyDataMap/tests/utils.py +++ b/Products/DataCollector/ApplyDataMap/tests/utils.py @@ -13,7 +13,6 @@ class BaseTestCase(TestCase): - def setUp(t): logging.disable(logging.CRITICAL) diff --git a/Products/DataCollector/Plugins.py b/Products/DataCollector/Plugins.py index 0f211fac4a..0506300a92 100644 --- a/Products/DataCollector/Plugins.py +++ b/Products/DataCollector/Plugins.py @@ -287,9 +287,7 @@ def getPluginLoaders(self, packs): if modname not in self.loadedZenpacks: self.loadedZenpacks.append(modname) modPathPrefix = ".".join( - (modname,) - + self.packPath - + (self.lastModName,) + (modname,) + self.packPath + (self.lastModName,) ) factory = PackLoaderFactory(OsWalker(), modPathPrefix) package = pack.path(*self.packPath + (self.lastModName,)) From cfb8340c23edc0c330fb331e1e83a8916c3710ba Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 5 May 2023 16:15:57 -0500 Subject: [PATCH 002/147] Update Jobber w/Python3-compatible changes --- Products/Jobber/config.py | 3 +- Products/Jobber/interfaces.py | 48 ++++++++++++++++++------------- Products/Jobber/metadirectives.py | 20 +++++++------ 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 80f685daaa..7f79295d39 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -11,7 +11,7 @@ from Products.ZenUtils.config import Config, ConfigLoader from Products.ZenUtils.GlobalConfig import getGlobalConfiguration -from Products.ZenUtils.RedisUtils import DEFAULT_REDIS_URL, getRedisUrl +from Products.ZenUtils.RedisUtils import DEFAULT_REDIS_URL from Products.ZenUtils.Utils import zenPath __all__ = ("Celery", "ZenJobs") @@ -33,7 +33,6 @@ "concurrent-jobs": 1, "job-hard-time-limit": 21600, # 6 hours "job-soft-time-limit": 18000, # 5 hours - "redis-url": DEFAULT_REDIS_URL, } diff --git a/Products/Jobber/interfaces.py b/Products/Jobber/interfaces.py index 9881667d31..2e1beee9ad 100644 --- a/Products/Jobber/interfaces.py +++ b/Products/Jobber/interfaces.py @@ -9,6 +9,8 @@ from __future__ import absolute_import +import six + from celery import states from zope.interface import Interface from zope.schema import Bool, Choice, Datetime, TextLine, Timedelta @@ -19,60 +21,66 @@ class IJobRecord(Interface): """ """ jobid = TextLine( - title=u"Job ID", - description=u"The Job's unique identifier", + title=six.text_type("Job ID"), + description=six.text_type("The Job's unique identifier"), ) name = TextLine( - title=u"Name", - description=u"The full class name of the job", + title=six.text_type("Name"), + description=six.text_type("The full class name of the job"), ) summary = TextLine( - title=u"Summary", - description=u"A brief and general summary of the job's function", + title=six.text_type("Summary"), + description=six.text_type( + "A brief and general summary of the job's function" + ), ) description = TextLine( - title=u"Description", - description=u"A description of what this job will do", + title=six.text_type("Description"), + description=six.text_type("A description of what this job will do"), ) userid = TextLine( - title=u"User ID", - description=u"The user that created the job", + title=six.text_type("User ID"), + description=six.text_type("The user that created the job"), ) logfile = TextLine( - title=u"Logfile", - description=u"Path to this job's log file.", + title=six.text_type("Logfile"), + description=six.text_type("Path to this job's log file."), ) status = Choice( - title=u"Status", - description=u"The current status of the job", + title=six.text_type("Status"), + description=six.text_type("The current status of the job"), vocabulary=SimpleVocabulary.fromValues(states.ALL_STATES), ) created = Datetime( - title=u"Created", description=u"When the job was created" + title=six.text_type("Created"), + description=six.text_type("When the job was created"), ) started = Datetime( - title=u"Started", description=u"When the job began executing" + title=six.text_type("Started"), + description=six.text_type("When the job began executing"), ) finished = Datetime( - title=u"Finished", description=u"When the job finished executing" + title=six.text_type("Finished"), + description=six.text_type("When the job finished executing"), ) duration = Timedelta( - title=u"Duration", description=u"How long the job has run" + title=six.text_type("Duration"), + description=six.text_type("How long the job has run"), ) complete = Bool( - title=u"Complete", - description=u"True if the job has finished running", + title=six.text_type("Complete"), + description=six.text_type("True if the job has finished running"), ) def abort(): diff --git a/Products/Jobber/metadirectives.py b/Products/Jobber/metadirectives.py index 16b6b93908..accb3b0e51 100644 --- a/Products/Jobber/metadirectives.py +++ b/Products/Jobber/metadirectives.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, unicode_literals +import six + from zope.configuration.fields import GlobalObject from zope.interface import Interface from zope.schema import TextLine @@ -18,13 +20,13 @@ class IJob(Interface): """Registers a ZenJobs Job class.""" class_ = GlobalObject( - title=u"Job Class", - description=u"The class of the job to register", + title=six.text_type("Job Class"), + description=six.text_type("The class of the job to register"), ) name = TextLine( - title=u"Name", - description=u"Optional name of the job", + title=six.text_type("Name"), + description=six.text_type("Optional name of the job"), required=False, ) @@ -33,11 +35,13 @@ class ICelerySignal(Interface): """Registers a Celery signal handler.""" name = TextLine( - title=u"Name", - description=u"The signal receiving a handler", + title=six.text_type("Name"), + description=six.text_type("The signal receiving a handler"), ) handler = TextLine( - title=u"Handler", - description=u"Classpath to the function handling the signal", + title=six.text_type("Handler"), + description=six.text_type( + "Classpath to the function handling the signal" + ), ) From 25d6f9d0e172c6090dbfc0e98796fafcad70b801 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 5 May 2023 16:17:24 -0500 Subject: [PATCH 003/147] Reformat ZenCollector test code using black --- Products/ZenCollector/tests/test_cyberark.py | 56 +++++++++++--------- Products/ZenCollector/tests/test_daemon.py | 1 - 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Products/ZenCollector/tests/test_cyberark.py b/Products/ZenCollector/tests/test_cyberark.py index e0d275d1a6..2890543bcc 100644 --- a/Products/ZenCollector/tests/test_cyberark.py +++ b/Products/ZenCollector/tests/test_cyberark.py @@ -29,19 +29,18 @@ CyberArkProperty, ) -PATH = {'src': 'Products.ZenCollector.cyberark'} +PATH = {"src": "Products.ZenCollector.cyberark"} class TestFunctions(TestCase): - def setUp(t): t.log_patcher = patch("{src}.log".format(**PATH), autospec=True) t.log = t.log_patcher.start() t.addCleanup(t.log_patcher.stop) t.getGlobalConfiguration_patcher = patch( - '{src}.getGlobalConfiguration'.format(**PATH), - name='getGlobalConfiguration', + "{src}.getGlobalConfiguration".format(**PATH), + name="getGlobalConfiguration", autospec=True, ) t.getGlobalConfiguration = t.getGlobalConfiguration_patcher.start() @@ -98,7 +97,6 @@ def test_get_cyberark_success(t, _manager): class TestCyberArk(TestCase): - class Conf(object): def __init__(self, query): self.configId = "dev1" @@ -111,7 +109,8 @@ def setUp(t): t.addCleanup(t.log_patcher.stop) t.queryUtility_patcher = patch( - "{src}.queryUtility".format(**PATH), autospec=True, + "{src}.queryUtility".format(**PATH), + autospec=True, ) t.queryUtility = t.queryUtility_patcher.start() t.addCleanup(t.queryUtility_patcher.stop) @@ -190,14 +189,14 @@ def test_update_config_no_props(t): class TestCyberArkManager(TestCase): - def setUp(t): t.log_patcher = patch("{src}.log".format(**PATH), autospec=True) t.log = t.log_patcher.start() t.addCleanup(t.log_patcher.stop) t.queryUtility_patcher = patch( - "{src}.queryUtility".format(**PATH), autospec=True, + "{src}.queryUtility".format(**PATH), + autospec=True, ) t.queryUtility = t.queryUtility_patcher.start() t.addCleanup(t.queryUtility_patcher.stop) @@ -277,9 +276,13 @@ def test_update_error(t): "Bad CyberArk query " "status=%s %s device=%s zproperty=%s query=%s " "ErrorCode=%s ErrorMsg=%s", - status, httplib.responses.get(status), - devId, zprop, query, - "4E", "object not found", + status, + httplib.responses.get(status), + devId, + zprop, + query, + "4E", + "object not found", ) mgr.add(devId, zprop, query) @@ -306,8 +309,11 @@ def test_update_unexpected_error(t): "Bad CyberArk query " "status=%s %s device=%s zproperty=%s query=%s " "result=%s", - status, httplib.responses.get(status), - devId, zprop, query, + status, + httplib.responses.get(status), + devId, + zprop, + query, "Unexpected format", ) @@ -333,7 +339,10 @@ def test_update_failure(t): expected = call( "Failed to execute CyberArk query - %s " "device=%s zproperty=%s query=%s", - ex, devId, zprop, query, + ex, + devId, + zprop, + query, ) mgr.add(devId, zprop, query) @@ -346,26 +355,28 @@ def test_update_failure(t): class TestCyberArkClient(TestCase): - def setUp(t): t.log_patcher = patch("{src}.log".format(**PATH), autospec=True) t.log = t.log_patcher.start() t.addCleanup(t.log_patcher.stop) t.load_certificates_patcher = patch( - "{src}.load_certificates".format(**PATH), autospec=True, + "{src}.load_certificates".format(**PATH), + autospec=True, ) t.load_certificates = t.load_certificates_patcher.start() t.addCleanup(t.load_certificates_patcher.stop) t.agent_patcher = patch( - "{src}.client.Agent".format(**PATH), autospec=True, + "{src}.client.Agent".format(**PATH), + autospec=True, ) t.agent = t.agent_patcher.start() t.addCleanup(t.agent_patcher.stop) t.readBody_patcher = patch( - "{src}.client.readBody".format(**PATH), autospec=True, + "{src}.client.readBody".format(**PATH), + autospec=True, ) t.readBody = t.readBody_patcher.start() t.addCleanup(t.readBody_patcher.stop) @@ -419,10 +430,7 @@ def test_request(t): t.assertEqual(result, expected_result) t.assertEqual(code, expected_code) ag.request.assert_called_once_with( - "GET", - "https://vault/bar/baz?appid=foo&object=foo", - None, - None + "GET", "https://vault/bar/baz?appid=foo&object=foo", None, None ) def test_request_with_extra_path(t): @@ -439,7 +447,7 @@ def test_request_with_extra_path(t): "GET", "https://vault/alias/bar/baz?appid=foo&object=foo", None, - None + None, ) def test_request_failure(t): @@ -474,7 +482,6 @@ def go_boom(*args, **kw): class TestCyberArkProperty(TestCase): - def test_init(t): dev = "device1" zprop = "prop1" @@ -548,7 +555,6 @@ def test_init(t): class TestLoadCertificates(TestCase): - def setUp(t): t.FilePath_patcher = patch( "{src}.FilePath".format(**PATH), diff --git a/Products/ZenCollector/tests/test_daemon.py b/Products/ZenCollector/tests/test_daemon.py index ac375fab8c..a6d6660e28 100644 --- a/Products/ZenCollector/tests/test_daemon.py +++ b/Products/ZenCollector/tests/test_daemon.py @@ -105,7 +105,6 @@ def test__pauseUnreachableDevices(t): class _Capture(object): - def __init__(self): self.err = None From 9203af609528a9be4969dde2c84c96e3198c4cc8 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 5 May 2023 16:51:34 -0500 Subject: [PATCH 004/147] Reformatted ZenCallHome code using the black formatting tool --- Products/ZenCallHome/CallHomeStatus.py | 2 +- Products/ZenCallHome/HostData.py | 150 +++++++------- Products/ZenCallHome/VersionHistory.py | 54 ++--- Products/ZenCallHome/ZenossAppData.py | 147 ++++++++------ Products/ZenCallHome/__init__.py | 48 +++-- Products/ZenCallHome/callhome.py | 187 +++++++++++------- .../tests/testCallHomeGeneration.py | 153 +++++++------- .../ZenCallHome/tests/testVersionHistory.py | 58 +++--- Products/ZenCallHome/transport/__init__.py | 155 ++++++++------- .../ZenCallHome/transport/crypt/__init__.py | 44 +++-- Products/ZenCallHome/transport/cycler.py | 66 ++++--- .../transport/methods/browserjs.py | 53 ++--- .../transport/methods/directpost.py | 20 +- .../transport/methods/versioncheck.py | 19 +- 14 files changed, 638 insertions(+), 518 deletions(-) diff --git a/Products/ZenCallHome/CallHomeStatus.py b/Products/ZenCallHome/CallHomeStatus.py index 1918c56818..b23a8fb87e 100644 --- a/Products/ZenCallHome/CallHomeStatus.py +++ b/Products/ZenCallHome/CallHomeStatus.py @@ -45,7 +45,7 @@ def create_redis_client(redis_url): return client def _connected_to_redis(self): - """ ensures we have a connection to redis """ + """ensures we have a connection to redis""" if self._redis_client is None: now = time.time() if ( diff --git a/Products/ZenCallHome/HostData.py b/Products/ZenCallHome/HostData.py index 4dcf4a9cf4..4e5a2733c2 100644 --- a/Products/ZenCallHome/HostData.py +++ b/Products/ZenCallHome/HostData.py @@ -7,44 +7,50 @@ # ############################################################################## - -import string -import os +import logging import math +import os import platform import socket +import string +from functools import total_ordering from subprocess import Popen, PIPE -from Products.ZenCallHome import IHostData, IZenossEnvData -from zope.interface import implements -import logging +from zope.interface import implementer + +from . import IHostData, IZenossEnvData + log = logging.getLogger("zen.callhome") -LOCAL_HOSTNAMES = ["localhost", - "localhost.localdomain", - socket.gethostname(), - socket.getfqdn()] +LOCAL_HOSTNAMES = [ + "localhost", + "localhost.localdomain", + socket.gethostname(), + socket.getfqdn(), +] +@implementer(IHostData) class PlatformData(object): - implements(IHostData) - def callHomeData(self): distro = " ".join(platform.linux_distribution()) processor = platform.processor() system = platform.system() release = platform.release() - yield ("OS", - "{distro} {processor} " - "({system} kernel {release})".format(**locals())) + yield ( + "OS", + "{distro} {processor} " + "({system} kernel {release})".format(**locals()), + ) +@implementer(IHostData) class ProcFileData(object): """ Used to gather proc file statistics for call home """ - implements(IHostData) + _proc_file = None _parser = None @@ -80,7 +86,6 @@ def _createParser(self): class ProcFileParser(object): - @classmethod def _parse_key_value(cls, line): if not line.strip(): @@ -89,7 +94,6 @@ def _parse_key_value(cls, line): class CpuinfoParser(ProcFileParser): - def __init__(self): self._processors = [] self._processor = None @@ -112,8 +116,10 @@ def output(self): cores += count dct = dict(tuples) cache_size = convert_kb(dct["cache size"]) - yield ("CPU", - "{dct[model name]} ({cache_size} cache)".format(**locals())) + yield ( + "CPU", + "{dct[model name]} ({cache_size} cache)".format(**locals()), + ) yield "CPU Cores", cores def _summarize(self): @@ -136,11 +142,10 @@ class CpuProcFileData(ProcFileData): _parser = CpuinfoParser def _ioErrorOutputHandler(self): - yield 'CPU Cores', 'Not available' + yield "CPU Cores", "Not available" class MemoryStat(object): - def __init__(self, label, total_key, free_key): self.label = label self._total = [total_key, None] @@ -152,15 +157,17 @@ def set(self, key, value): stat[1] = convert_kb(value, key.endswith("Total")) def __repr__(self): - return ("{self._free[1]} of " - "{self._total[1]} available").format(**locals()) + return ("{self._free[1]} of " "{self._total[1]} available").format( + **locals() + ) class MeminfoParser(ProcFileParser): - def __init__(self): - self._stats = [MemoryStat("Memory", "MemTotal", "MemFree"), - MemoryStat("Swap", "SwapTotal", "SwapFree")] + self._stats = [ + MemoryStat("Memory", "MemTotal", "MemFree"), + MemoryStat("Swap", "SwapTotal", "SwapFree"), + ] def parse(self, line): key, value = self._parse_key_value(line) @@ -179,13 +186,14 @@ class MemProcFileData(ProcFileData): def _ioErrorOutputHandler(self): for stat in self._parser()._stats: - yield stat.label, 'Not available' + yield stat.label, "Not available" class CommandData(object): """ Base class for executing and return data based on executing a command """ + _args = [] _parser = None @@ -215,20 +223,27 @@ def _osErrorOutputHandler(self): return tuple() +@total_ordering class FilesystemInfo(object): - def __init__(self, mounted_on="", size=None, avail=None): self.mounted_on = mounted_on self.size = size self.avail = avail self.supporting = [] - def __cmp__(self, other_fs_info): - return cmp(self.mounted_on, other_fs_info.mounted_on) + def __eq__(self, other): + return self.mounted_on == other.mounted_on + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return self.mounted_on < other.mounted_on def __repr__(self): - repr_ = ("'{self.mounted_on}', " - "{self.avail} of {self.size} available").format(**locals()) + repr_ = ( + "'{self.mounted_on}', " "{self.avail} of {self.size} available" + ).format(**locals()) if self.supporting: supporting = ", ".join(self.supporting) repr_ = "{repr_} (supports {supporting})".format(**locals()) @@ -236,23 +251,26 @@ def __repr__(self): class DfParser(object): - def __init__(self): - self._zenoss_mounts = {zenhome.environ_key: "", - zendshome.environ_key: "", - rabbitmq_mnesia_base.environ_key: ""} + self._zenoss_mounts = { + zenhome.environ_key: "", + zendshome.environ_key: "", + rabbitmq_mnesia_base.environ_key: "", + } self._filesystems = [] def parse(self, line): if not line.startswith("/"): return filesystem, size, used, avail, use_pct, mounted_on = line.split() - fs_info = FilesystemInfo(mounted_on, convert_kb(size), convert_kb( - avail, False)) + fs_info = FilesystemInfo( + mounted_on, convert_kb(size), convert_kb(avail, False) + ) for environ_var in zenhome, zendshome, rabbitmq_mnesia_base: if environ_var.value is not None and environ_var.value.startswith( - mounted_on): + mounted_on + ): key = environ_var.environ_key if len(mounted_on) > len(self._zenoss_mounts[key]): fs_info.supporting.append(key) @@ -266,8 +284,8 @@ def output(self): yield "Filesystem", str(filesystem) +@implementer(IHostData) class DfData(CommandData): - implements(IHostData) _args = ["df", "-Pk"] _parser = DfParser @@ -276,9 +294,9 @@ def _osErrorOutputHandler(self): yield "Filesystem", "Not Available" +@implementer(IHostData) class HostId(CommandData): - implements(IHostData) - _args = ['hostid'] + _args = ["hostid"] def __init__(self): self._parser = HostId @@ -296,7 +314,6 @@ def _osErrorOutputHandler(self): class RpmParser(object): - def __init__(self, key): self._output = None self._key = key @@ -306,22 +323,19 @@ def parse(self, line): @property def output(self): - label = 'RPM' + label = "RPM" if self._key: label = "%s - %s" % (label, self._key) yield label, self._output class RPMData(CommandData): - def __init__(self, rpm_arg): super(RPMData, self).__init__() self._rpm_arg = rpm_arg - if ( - os.path.exists("/etc/redhat-release") - or - os.path.exists("/etc/SuSe-release") - ): + if os.path.exists("/etc/redhat-release") or os.path.exists( + "/etc/SuSe-release" + ): self._rpm_support = True self._args = ["rpm", "-q", rpm_arg] else: @@ -332,7 +346,7 @@ def _createParser(self): return RpmParser(self._rpm_arg) def _osErrorOutputHandler(self): - label = 'RPM' + label = "RPM" if self._rpm_arg: label = "%s - %s" % (label, self._rpm_arg) if self._rpm_support: @@ -342,33 +356,30 @@ def _osErrorOutputHandler(self): yield label, value +@implementer(IZenossEnvData) class ZenossRPMData(RPMData): - implements(IZenossEnvData) - def __init__(self): - super(ZenossRPMData, self).__init__('zenoss') + super(ZenossRPMData, self).__init__("zenoss") +@implementer(IZenossEnvData) class ZenDSRPMData(RPMData): - implements(IZenossEnvData) - def __init__(self): - super(ZenDSRPMData, self).__init__('zends') + super(ZenDSRPMData, self).__init__("zends") +@implementer(IZenossEnvData) class CoreZenpackRPMData(RPMData): - implements(IZenossEnvData) - def __init__(self): - super(CoreZenpackRPMData, self).__init__('zenoss-core-zenpacks') + super(CoreZenpackRPMData, self).__init__("zenoss-core-zenpacks") +@implementer(IZenossEnvData) class EnterpriseZenpackRPMData(RPMData): - implements(IZenossEnvData) - def __init__(self): super(EnterpriseZenpackRPMData, self).__init__( - 'zenoss-enterprise-zenpacks') + "zenoss-enterprise-zenpacks" + ) class Zenhome(object): @@ -437,29 +448,26 @@ def generate(self): rabbitmq_mnesia_base = RabbitmqMnesiaBase() +@implementer(IZenossEnvData) class ZenHomeData(object): - implements(IZenossEnvData) - def callHomeData(self): return zenhome.generate() +@implementer(IZenossEnvData) class ZenDSHomeData(object): - implements(IZenossEnvData) - def callHomeData(self): return zendshome.generate() +@implementer(IZenossEnvData) class RabbitData(object): - implements(IZenossEnvData) - def callHomeData(self): return rabbitmq_mnesia_base.generate() def convert_kb(kb_str, round_up=True): - units = ['YB', 'ZB', 'EB', 'PB', 'TB', 'GB', 'MB', 'KB'] + units = ["YB", "ZB", "EB", "PB", "TB", "GB", "MB", "KB"] quantity = int(kb_str.translate(None, string.ascii_letters)) # 5 percent fudge factor for rounding up while quantity > (1024 - (1024 * 0.05)): diff --git a/Products/ZenCallHome/VersionHistory.py b/Products/ZenCallHome/VersionHistory.py index 872edfc164..cc14379b94 100644 --- a/Products/ZenCallHome/VersionHistory.py +++ b/Products/ZenCallHome/VersionHistory.py @@ -7,24 +7,24 @@ # ############################################################################## - -from zope.interface import implements -from Products.ZenCallHome import IVersionHistoryCallHomeCollector -from Products.ZenCallHome.callhome import (REPORT_DATE_KEY, - VERSION_HISTORIES_KEY) import logging + +from zope.interface import implementer + +from . import IVersionHistoryCallHomeCollector +from .callhome import REPORT_DATE_KEY, VERSION_HISTORIES_KEY + log = logging.getLogger("zen.callhome") VERSION_START_KEY = "Version Start" +@implementer(IVersionHistoryCallHomeCollector) class VersionHistoryCallHomeCollector(object): """ - Superclass for version history collectors that - provides some basic functionality if you - provide the code to get the current version + Superclass for version history collectors that provides some basic + functionality if you provide the code to get the current version. """ - implements(IVersionHistoryCallHomeCollector) def __init__(self, versionedEntity): self._entity = versionedEntity @@ -39,15 +39,14 @@ def addVersionHistory(self, dmd, callHomeData): def getCurrentVersion(self, dmd, callHomeData): """ - implement this to determine the current - version. probably available in the - callhome data. + Implement this to determine the current version. + Probably available in the callhome data. """ raise NotImplementedError() def createVersionHistoryRecord(self, dmd, callHomeData): """ - Create a record object with the date + Create a record object with the date. """ reportDate = callHomeData[REPORT_DATE_KEY] record = {VERSION_START_KEY: reportDate} @@ -56,19 +55,20 @@ def createVersionHistoryRecord(self, dmd, callHomeData): class KeyedVersionHistoryCallHomeCollector(VersionHistoryCallHomeCollector): """ - If version info can be pulled from the callhome - data by simple keys, then this class handles - all the work. + If version info can be pulled from the callhome data by simple keys, + then this class handles all the work. """ def __init__(self, versionedEntity, historyRecordKeys=[]): - super(KeyedVersionHistoryCallHomeCollector, - self).__init__(versionedEntity) + super(KeyedVersionHistoryCallHomeCollector, self).__init__( + versionedEntity + ) self._historyRecordKeys = historyRecordKeys def createVersionHistoryRecord(self, dmd, callHomeData): - record = super(KeyedVersionHistoryCallHomeCollector, - self).createVersionHistoryRecord(dmd, callHomeData) + record = super( + KeyedVersionHistoryCallHomeCollector, self + ).createVersionHistoryRecord(dmd, callHomeData) if self._historyRecordKeys: for hrKey, targetKey in self._historyRecordKeys.iteritems(): value = self.getKeyedValue(hrKey, callHomeData) @@ -77,7 +77,7 @@ def createVersionHistoryRecord(self, dmd, callHomeData): return record def getKeyedValue(self, hrKey, callHomeData): - key_list = hrKey.split('.') + key_list = hrKey.split(".") currObj = callHomeData for key in key_list: currObj = currObj.get(key, None) @@ -87,16 +87,18 @@ def getKeyedValue(self, hrKey, callHomeData): class ZenossVersionHistoryCallHomeCollector( - KeyedVersionHistoryCallHomeCollector): - """ - """ + KeyedVersionHistoryCallHomeCollector +): + """ """ + ZENOSS_VERSION_HISTORY_KEY = "Zenoss" ZENOSS_VERSION_HISTORY_RECORD_KEYS = {} def __init__(self): super(ZenossVersionHistoryCallHomeCollector, self).__init__( self.ZENOSS_VERSION_HISTORY_KEY, - self.ZENOSS_VERSION_HISTORY_RECORD_KEYS) + self.ZENOSS_VERSION_HISTORY_RECORD_KEYS, + ) def getCurrentVersion(self, dmd, callHomeData): - return self.getKeyedValue('Zenoss App Data.Zenoss', callHomeData) + return self.getKeyedValue("Zenoss App Data.Zenoss", callHomeData) diff --git a/Products/ZenCallHome/ZenossAppData.py b/Products/ZenCallHome/ZenossAppData.py index a5fa333317..13d0cc3de5 100644 --- a/Products/ZenCallHome/ZenossAppData.py +++ b/Products/ZenCallHome/ZenossAppData.py @@ -7,52 +7,60 @@ # ############################################################################## - +import logging import time -from Products.ZenCallHome import (IZenossData, IDeviceResource, - IDeviceCpuCount, IDeviceType, - IVirtualDeviceType) -from zope.interface import implements -from zope.component import subscribers, getAdapters -from Products.Zuul import getFacade -from Products.ZenModel.DeviceComponent import DeviceComponent + from itertools import chain -import logging -from Products.Zuul.catalog.interfaces import IModelCatalogTool from zenoss.protocols.services.zep import ZepConnectionError -from . import IDeviceLink +from zope.interface import implementer +from zope.component import subscribers, getAdapters + +from Products.ZenModel.DeviceComponent import DeviceComponent +from Products.Zuul import getFacade +from Products.Zuul.catalog.interfaces import IModelCatalogTool + +from . import ( + IDeviceCpuCount, + IDeviceLink, + IDeviceResource, + IDeviceType, + IVirtualDeviceType, + IZenossData, +) log = logging.getLogger("zen.callhome") +@implementer(IZenossData) class ZenossAppData(object): - implements(IZenossData) - def callHomeData(self, dmd): self.dmd = dmd self._catalog = IModelCatalogTool(self.dmd) - stats = (self.server_key, - self.google_key, - self.all_versions, - self.event_classes, - self.event_count, - self.reports, - self.templates, - self.systems, - self.groups, - self.locations, - self.total_collectors, - self.zenpacks, - self.user_count, - self.product_count, - self.product_name, - self.components) + stats = ( + self.server_key, + self.google_key, + self.all_versions, + self.event_classes, + self.event_count, + self.reports, + self.templates, + self.systems, + self.groups, + self.locations, + self.total_collectors, + self.zenpacks, + self.user_count, + self.product_count, + self.product_name, + self.components, + ) return chain.from_iterable(map(lambda fn: fn(), stats)) def components(self): - brains = self._catalog.search(types=(DeviceComponent,), - facets_for_field=["meta_type"]) + brains = self._catalog.search( + types=(DeviceComponent,), facets_for_field=["meta_type"] + ) if brains.facets and brains.facets.get("meta_type"): facets = brains.facets["meta_type"] comps = facets.get_values() @@ -66,7 +74,8 @@ def product_name(self): def product_count(self): manufacturers = self.dmd.Manufacturers.objectValues( - spec='Manufacturer') + spec="Manufacturer" + ) prodCount = 0 for m in manufacturers: prodCount += m.products.countObjects() @@ -85,16 +94,21 @@ def google_key(self): def zenpacks(self): for zenpack in self.dmd.ZenPackManager.packs(): - yield ("Zenpack", - "{zenpack.id} {zenpack.version}".format(**locals())) + yield ( + "Zenpack", + "{zenpack.id} {zenpack.version}".format(**locals()), + ) def all_versions(self): - zenoss_version, cc_version = self.dmd.About.getZenossVersion(), self.dmd.About.getControlCenterVersion() + zenoss_version, cc_version = ( + self.dmd.About.getZenossVersion(), + self.dmd.About.getControlCenterVersion(), + ) yield zenoss_version.name, zenoss_version.full() yield cc_version.name, cc_version.full() def event_classes(self): - yield 'Evt Mappings', self.dmd.Events.countInstances() + yield "Evt Mappings", self.dmd.Events.countInstances() def reports(self): yield "Reports", self.dmd.Reports.countReports() @@ -116,21 +130,25 @@ def total_collectors(self): yield "Collectors", len(results) def event_count(self): - zep = getFacade('zep', self.dmd) + zep = getFacade("zep", self.dmd) try: - yield ("Event Count", - zep.countEventsSince(time.time() - 24 * 60 * 60)) + yield ( + "Event Count", + zep.countEventsSince(time.time() - 24 * 60 * 60), + ) except ZepConnectionError: yield "Event Count: last 24hr", "Not Available" -VM_MACS = {"00:0C:29": 'VMware Guest', - "00:50:56": 'VMware Guest', - "00:16:3e": 'Xen Guest'} +VM_MACS = { + "00:0C:29": "VMware Guest", + "00:50:56": "VMware Guest", + "00:16:3e": "Xen Guest", +} -class MacAddressVirtualDeviceType(object): - implements(IVirtualDeviceType) +@implementer(IVirtualDeviceType) +class MacAddressVirtualDeviceType(object): def __init__(self, device): self._device = device self._vmType = None @@ -143,9 +161,8 @@ def vmType(self): return self._vmType +@implementer(IDeviceType) class DeviceType(object): - implements(IDeviceType) - def __init__(self, device): self._device = device self._isVM = None @@ -163,7 +180,7 @@ def isVM(self): return self._isVM def type(self): - dType = 'Physical' + dType = "Physical" if self._isVM is None: self.isVM() if self._vmType: @@ -171,9 +188,8 @@ def type(self): return dType +@implementer(IDeviceResource) class DeviceTypeCounter(object): - implements(IDeviceResource) - def __init__(self, device): self._device = device @@ -192,22 +208,22 @@ def _get_type(self): return dev.type(), dev.isVM() +@implementer(IDeviceResource) class DeviceClassProductionStateCount(object): - implements(IDeviceResource) - def __init__(self, device): self._device = device def processDevice(self, stats): - key = "%s: %s" % (self._device.getDeviceClassPath(), - self._device.getProductionStateString()) + key = "%s: %s" % ( + self._device.getDeviceClassPath(), + self._device.getProductionStateString(), + ) stats.setdefault(key, 0) stats[key] += 1 +@implementer(IDeviceCpuCount) class DeviceCpuCounter(object): - implements(IDeviceCpuCount) - def __init__(self, device): self._device = device @@ -218,9 +234,8 @@ def cpuCount(self): return 0 +@implementer(IZenossData) class ZenossResourceData(object): - implements(IZenossData) - def __init__(self): self._dmd = None self._catalog = None @@ -233,24 +248,30 @@ def callHomeData(self, dmd): yield key, value def _process_devices(self): - stats = {'Device Count': 0, - 'Decommissioned Devices': 0, - 'CPU Cores': 0} + stats = { + "Device Count": 0, + "Decommissioned Devices": 0, + "CPU Cores": 0, + } LINKED_DEVICES = "Linked Devices" if LINKED_DEVICES not in stats: stats[LINKED_DEVICES] = 0 for device in self._dmd.Devices.getSubDevicesGen_recursive(): - stats['Device Count'] += 1 + stats["Device Count"] += 1 if device.getProductionState() < 0: stats["Decommissioned Devices"] += 1 cpuCount = IDeviceCpuCount(device).cpuCount() log.debug("Devices %s has %s cpu cores", device, cpuCount) - stats['CPU Cores'] += cpuCount + stats["CPU Cores"] += cpuCount for adapter in subscribers([device], IDeviceResource): adapter.processDevice(stats) found_linked = False for name, adapter in getAdapters((device,), IDeviceLink): - if adapter.linkedDevice() and adapter.linkedDevice().device().getProductionState() > 0: + if ( + adapter.linkedDevice() + and adapter.linkedDevice().device().getProductionState() + > 0 + ): key = "%s - %s" % (LINKED_DEVICES, name) if key not in stats: stats[key] = 0 diff --git a/Products/ZenCallHome/__init__.py b/Products/ZenCallHome/__init__.py index def0578128..460661ffc6 100644 --- a/Products/ZenCallHome/__init__.py +++ b/Products/ZenCallHome/__init__.py @@ -13,78 +13,75 @@ class ICallHomeCollector(Interface): """ - Implementers provide call home data + Implementers provide call home data. """ def generateData(self): """ - Generate data to be sent via call home - @return: dictionary of data to be sent. - values keyed by "_ERRORS_" should - be objects that will be attached - at the top level. + Generate data to be sent via call home. + + @return: Dictionary of data to be sent. Values keyed by "_ERRORS_" + should be objects that will be attached at the top level. @rtype: dict """ class IMasterCallHomeCollector(Interface): """ - Implementers provide call home data when collected on zenoss master + Implementers provide call home data when collected on zenoss master. """ def generateData(self, dmd): """ - Generate data to be sent via call home - @param dmd: databse connection - @return: dictionary of data to be sent - values keyed by "_ERRORS_" should - be objects that will be attached - at the top level. + Generate data to be sent via call home. + + @param dmd: Databse connection + @return: Dictionary of data to be sent values keyed by "_ERRORS_" + should be objects that will be attached at the top level. @rtype: dict """ class IVersionHistoryCallHomeCollector(Interface): """ - Implementers provide version history records + Implementers provide version history records. """ def addVersionHistory(self, dmd, callHomeData): """ - Create records to be added to version history - @param the callhome data that will be modified - then sent + Create records to be added to version history. + @param the callhome data that will be modified then sent. """ class IHostData(Interface): """ - Used to gather Host machine statistics for call home + Used to gather Host machine statistics for call home. """ def callHomeData(self): """ - @return:: name, value pairs of host stats for call home - @rtype: list or generator of tuples + @return:: name, value pairs of host stats for call home. + @rtype: list or generator of tuples. """ class IZenossData(Interface): """ - Used to gather Zenoss statistics for call home + Used to gather Zenoss statistics for call home. """ def callHomeData(self, dmd): """ - @param: dmd connection - @return: name, value pairs of Zenoss instance stats for call home - @rtype: list or generator of tuples + @param: dmd connection. + @return: name, value pairs of Zenoss instance stats for call home. + @rtype: list or generator of tuples. """ class IZenossEnvData(Interface): """ - Used to gather the Zenoss environment data for call home + Used to gather the Zenoss environment data for call home. """ def callHomeData(self): @@ -148,6 +145,7 @@ class IVirtualDeviceType(Interface): Subscription adapter. Determine the virtual machine type of a device if any. More than one impl can be registered per Device """ + def vmType(self): """ @return the type of virtual machine or None if not a virtual diff --git a/Products/ZenCallHome/callhome.py b/Products/ZenCallHome/callhome.py index 6958eef461..91684c824a 100755 --- a/Products/ZenCallHome/callhome.py +++ b/Products/ZenCallHome/callhome.py @@ -7,23 +7,27 @@ # ############################################################################## +from __future__ import print_function +import logging import json + from datetime import datetime -from zope.interface import implements +from zope.interface import implementer from zope.component import getUtilitiesFor -from Products.ZenCallHome.transport import CallHome - -from Products.ZenCallHome import (IZenossData, IHostData, IZenossEnvData, - ICallHomeCollector, - IMasterCallHomeCollector, - IVersionHistoryCallHomeCollector) -from Products.ZenUtils.ZenScriptBase import ZenScriptBase +from Products.ZenCallHome import ( + ICallHomeCollector, + IHostData, + IMasterCallHomeCollector, + IVersionHistoryCallHomeCollector, + IZenossData, + IZenossEnvData, +) from Products.ZenCallHome.CallHomeStatus import CallHomeStatus +from Products.ZenUtils.ZenScriptBase import ZenScriptBase -import logging log = logging.getLogger("zen.callhome") ERROR_KEY = "_ERROR_" @@ -32,9 +36,7 @@ VERSION_HISTORIES_KEY = "Version History" - class CallHomeCollector(object): - def __init__(self, utilityClass): self._utilityClass = utilityClass self._needsDmd = False @@ -47,8 +49,12 @@ def generateData(self, dmd=None): args.append(dmd) for name, utilClass in getUtilitiesFor(self._utilityClass): try: - log.debug("Getting data from %s %s, args: %s", - name, utilClass, str(args)) + log.debug( + "Getting data from %s %s, args: %s", + name, + utilClass, + str(args), + ) util = utilClass() for key, val in util.callHomeData(*args): log.debug("Data: %s | %s", key, val) @@ -62,14 +68,18 @@ def generateData(self, dmd=None): stats[key] = val except Exception as e: errorObject = dict( - source=utilClass.__name__, - key=name, - callhome_collector=self.__class__.__name__, - exception=str(e)) - log.warn("Continuing after catching exception while " - "generating callhome data for collector " - "%(callhome_collector)s (%(source)s:%(key)s : " - "%(exception)s", errorObject) + source=utilClass.__name__, + key=name, + callhome_collector=self.__class__.__name__, + exception=str(e), + ) + log.warn( + "Continuing after catching exception while " + "generating callhome data for collector " + "%(callhome_collector)s (%(source)s:%(key)s : " + "%(exception)s", + errorObject, + ) errors.append(errorObject) returnValue = {self._key: stats} if errors: @@ -77,11 +87,11 @@ def generateData(self, dmd=None): return returnValue +@implementer(IMasterCallHomeCollector) class ZenossDataCallHomeCollector(CallHomeCollector): """ Gathers data from all IZenossData utilities registered """ - implements(IMasterCallHomeCollector) def __init__(self): super(ZenossDataCallHomeCollector, self).__init__(IZenossData) @@ -89,22 +99,22 @@ def __init__(self): self._needsDmd = True +@implementer(ICallHomeCollector) class HostDataCallHomeCollector(CallHomeCollector): """ Gathers data from all IHostData utilities registered """ - implements(ICallHomeCollector) def __init__(self): super(HostDataCallHomeCollector, self).__init__(IHostData) self._key = "Host Data" +@implementer(IMasterCallHomeCollector) class ZenossEnvDataCallHomeCollector(CallHomeCollector): """ Gathers data from all IZenossEnvData utilities registered """ - implements(IMasterCallHomeCollector) def __init__(self): super(ZenossEnvDataCallHomeCollector, self).__init__(IZenossEnvData) @@ -123,8 +133,9 @@ def getExistingVersionHistories(self): metricsString = self._dmd.callHome.metrics if metricsString and metricsString.strip(): metricsObj = json.loads(metricsString) - versionHistories = metricsObj.get(VERSION_HISTORIES_KEY, - {}) + versionHistories = metricsObj.get( + VERSION_HISTORIES_KEY, {} + ) except AttributeError: pass return {VERSION_HISTORIES_KEY: versionHistories} @@ -144,12 +155,15 @@ def getData(self): data.update(chData) except Exception as e: errorObject = dict( - callhome_collector=utilClass.__name__, - name=name, - exception=str(e)) - log.warn("Caught exception while generating callhome data " - "%(callhome_collector)s:%(name)s : %(exception)s", - errorObject) + callhome_collector=utilClass.__name__, + name=name, + exception=str(e), + ) + log.warn( + "Caught exception while generating callhome data " + "%(callhome_collector)s:%(name)s : %(exception)s", + errorObject, + ) errors.append(errorObject) if self._master: for name, utilClass in getUtilitiesFor(IMasterCallHomeCollector): @@ -162,26 +176,35 @@ def getData(self): data.update(chData) except Exception as e: errorObject = dict( - callhome_collector=utilClass.__name__, - name=name, - exception=str(e)) - log.warn("Caught exception while generating callhome " - "data %(callhome_collector)s:%(name)s : " - "%(exception)s", errorObject) + callhome_collector=utilClass.__name__, + name=name, + exception=str(e), + ) + log.warn( + "Caught exception while generating callhome " + "data %(callhome_collector)s:%(name)s : " + "%(exception)s", + errorObject, + ) errors.append(errorObject) if self._dmd: for name, utilClass in getUtilitiesFor( - IVersionHistoryCallHomeCollector): + IVersionHistoryCallHomeCollector + ): try: utilClass().addVersionHistory(self._dmd, data) except Exception as e: errorObject = dict( - callhome_collector=utilClass.__name__, - name=name, - exception=str(e)) - log.warn("Caught exception while adding version " - "history: %(callhome_collector)s:%(name)s : " - "%(exception)s", errorObject) + callhome_collector=utilClass.__name__, + name=name, + exception=str(e), + ) + log.warn( + "Caught exception while adding version " + "history: %(callhome_collector)s:%(name)s : " + "%(exception)s", + errorObject, + ) errors.append(errorObject) if errors: data[EXTERNAL_ERROR_KEY] = errors @@ -189,13 +212,16 @@ def getData(self): class Main(ZenScriptBase): - def run(self): if self.options.status: chs = CallHomeStatus() - print 'Status:\t Description:\t Error:\t' + print("Status:\t Description:\t Error:\t") for i in chs.status(): - print '{0}\t {1}\t {2}'.format(i.get('status'), i.get('description'), i.get('error')) + print( + "{0}\t {1}\t {2}".format( + i.get("status"), i.get("description"), i.get("error") + ) + ) return if self.options.master: @@ -211,40 +237,55 @@ def run(self): data = chd.getData() if self.options.pretty: from pprint import pprint + pprint(data) else: sort = False if self.options.jsonIndent: sort = True - print(json.dumps(data, indent=self.options.jsonIndent, - sort_keys=sort)) + print( + json.dumps( + data, indent=self.options.jsonIndent, sort_keys=sort + ) + ) chs.stage(chs.COLLECT_CALLHOME, "FINISHED") def buildOptions(self): """basic options setup sub classes can add more options here""" ZenScriptBase.buildOptions(self) - self.parser.add_option('-M', '--master', - dest='master', - default=False, - action='store_true', - help='Gather zenoss master data') - self.parser.add_option('-p', - dest='pretty', - default=False, - action='store_true', - help='pretty print the output') - self.parser.add_option('-i', '--json_indent', - dest='jsonIndent', - help='indent setting for json output', - default=None, - type='int') - self.parser.add_option('-s', '--status', - action='store_true', - dest='status', - help='show detail status information', - default=False) - - -if __name__ == '__main__': + self.parser.add_option( + "-M", + "--master", + dest="master", + default=False, + action="store_true", + help="Gather zenoss master data", + ) + self.parser.add_option( + "-p", + dest="pretty", + default=False, + action="store_true", + help="pretty print the output", + ) + self.parser.add_option( + "-i", + "--json_indent", + dest="jsonIndent", + help="indent setting for json output", + default=None, + type="int", + ) + self.parser.add_option( + "-s", + "--status", + action="store_true", + dest="status", + help="show detail status information", + default=False, + ) + + +if __name__ == "__main__": main = Main(connect=False) main.run() diff --git a/Products/ZenCallHome/tests/testCallHomeGeneration.py b/Products/ZenCallHome/tests/testCallHomeGeneration.py index 40e4813e73..f0facae9e8 100644 --- a/Products/ZenCallHome/tests/testCallHomeGeneration.py +++ b/Products/ZenCallHome/tests/testCallHomeGeneration.py @@ -7,33 +7,37 @@ # ############################################################################## - -import time import json +import time from datetime import datetime - -from zope.interface import Interface, implements +from zope.interface import Interface, implementer +from Zope2.App import zcml from Products.ZenTestCase.BaseTestCase import BaseTestCase -from Zope2.App import zcml import Products.ZenCallHome + from Products.ZenCallHome import ICallHomeCollector -from Products.ZenCallHome.callhome import (CallHomeCollector, CallHomeData, - EXTERNAL_ERROR_KEY, - REPORT_DATE_KEY, - VERSION_HISTORIES_KEY) +from Products.ZenCallHome.callhome import ( + CallHomeCollector, + CallHomeData, + EXTERNAL_ERROR_KEY, + REPORT_DATE_KEY, + VERSION_HISTORIES_KEY, +) from Products.ZenCallHome.VersionHistory import ( - VERSION_START_KEY, - KeyedVersionHistoryCallHomeCollector) + KeyedVersionHistoryCallHomeCollector, + VERSION_START_KEY, +) from Products.ZenCallHome.transport import ( - CallHome, - CallHomeData as PersistentCallHomeData) + CallHome, + CallHomeData as PersistentCallHomeData, +) -DATETIME_ISOFORMAT = '%Y-%m-%dT%H:%M:%S.%f' +DATETIME_ISOFORMAT = "%Y-%m-%dT%H:%M:%S.%f" TEST_DATA = """ -""" # noqa E501 +""" # noqa E501 FAILING_TEST_DATA = """ -""" # noqa E501 +""" # noqa E501 SIMPLE_SUCCESS_COLLECTOR = """ -""" # noqa E501 +""" # noqa E501 SIMPLE_SUCCESS_KEY = "simplesuccess" @@ -71,7 +75,7 @@ provides="Products.ZenCallHome.ICallHomeCollector" name="fastfail"/> -""" # noqa E501 +""" # noqa E501 FAST_FAIL_KEY = "fastfail" @@ -85,20 +89,18 @@ provides="Products.ZenCallHome.IVersionHistoryCallHomeCollector" name="testversionhistory"/> -""" # noqa E501 +""" # noqa E501 class ITestCallHomeData(Interface): - """ - """ + """ """ + def callHomeData(self): - """ - """ + """ """ +@implementer(ITestCallHomeData) class TestCallHomeData(object): - implements(ITestCallHomeData) - def callHomeData(self): yield "test", "test" @@ -107,18 +109,15 @@ class FailingTestDataException(Exception): pass +@implementer(ITestCallHomeData) class FailingTestCallHomeData(object): - implements(ITestCallHomeData) - def callHomeData(self): raise FailingTestDataException(FAILING_DATA_ERROR_MESSAGE) +@implementer(ICallHomeCollector) class SimpleSuccessCollector(CallHomeCollector): - """ - Default success collector as a control variable - """ - implements(ICallHomeCollector) + """Default success collector as a control variable.""" def __init__(self): super(SimpleSuccessCollector, self).__init__(ITestCallHomeData) @@ -129,11 +128,9 @@ class FastFailTestException(Exception): pass +@implementer(ICallHomeCollector) class FastFailCollector(CallHomeCollector): - """ - Default success collector as a control variable - """ - implements(ICallHomeCollector) + """Default success collector as a control variable.""" def __init__(self): super(FastFailCollector, self).__init__(ITestCallHomeData) @@ -142,6 +139,7 @@ def __init__(self): def generateData(self): raise FastFailTestException(FAST_FAIL_ERROR_MESSAGE) + TEST_VERSION_HISTORY_ENTITY = "testentity" TEST_VERSION_1 = "testversion1" TEST_VERSION_2 = "testversion2" @@ -153,22 +151,22 @@ def returnHistory(): class TestVersionHistoryCollector(KeyedVersionHistoryCallHomeCollector): - """ - """ + """ """ + def __init__(self): super(TestVersionHistoryCollector, self).__init__( - TEST_VERSION_HISTORY_ENTITY, {}) + TEST_VERSION_HISTORY_ENTITY, {} + ) def getCurrentVersion(self, dmd, callHomeData): return returnHistory() class testCallHomeGeneration(BaseTestCase): - def afterSetUp(self): super(testCallHomeGeneration, self).afterSetUp() - zcml.load_config('meta.zcml', Products.ZenCallHome) - zcml.load_config('configure.zcml', Products.ZenCallHome) + zcml.load_config("meta.zcml", Products.ZenCallHome) + zcml.load_config("configure.zcml", Products.ZenCallHome) def beforeTearDown(self): super(testCallHomeGeneration, self).beforeTearDown() @@ -199,8 +197,9 @@ def testCallHomeCollectorFailure(self): self.assertTrue(FAST_FAIL_KEY not in data) self.assertTrue("Zenoss App Data" in data) self.assertTrue(EXTERNAL_ERROR_KEY in data) - self.assertEquals(FAST_FAIL_ERROR_MESSAGE, - data[EXTERNAL_ERROR_KEY][0]['exception']) + self.assertEquals( + FAST_FAIL_ERROR_MESSAGE, data[EXTERNAL_ERROR_KEY][0]["exception"] + ) def testConstituentDataFailure(self): # check current version of report (should be empty?) @@ -224,8 +223,10 @@ def testConstituentDataFailure(self): successData = data[SIMPLE_SUCCESS_KEY] self.assertTrue("test" in successData) self.assertTrue(EXTERNAL_ERROR_KEY in data) - self.assertEquals(FAILING_DATA_ERROR_MESSAGE, - data[EXTERNAL_ERROR_KEY][0]['exception']) + self.assertEquals( + FAILING_DATA_ERROR_MESSAGE, + data[EXTERNAL_ERROR_KEY][0]["exception"], + ) def testPayloadGeneration(self): # check current version of report (should be empty) @@ -262,24 +263,28 @@ def testPayloadGeneration(self): payloadObj = json.loads(payload) # make sure payload has the required fields - self.assertTrue('product' in payloadObj) - self.assertTrue('uuid' in payloadObj) - self.assertTrue('symkey' in payloadObj) - self.assertTrue('metrics' in payloadObj) + self.assertTrue("product" in payloadObj) + self.assertTrue("uuid" in payloadObj) + self.assertTrue("symkey" in payloadObj) + self.assertTrue("metrics" in payloadObj) # reconstitute metrics obj & make sure send date is present # and has a valid time - metricsObj = json.loads(payloadObj['metrics']) - self.assertTrue('Send Date' in metricsObj) - sendDateDT = datetime.strptime(metricsObj['Send Date'], - DATETIME_ISOFORMAT) - reportDateDT = datetime.strptime(metricsObj['Report Date'], - DATETIME_ISOFORMAT) + metricsObj = json.loads(payloadObj["metrics"]) + self.assertTrue("Send Date" in metricsObj) + sendDateDT = datetime.strptime( + metricsObj["Send Date"], DATETIME_ISOFORMAT + ) + reportDateDT = datetime.strptime( + metricsObj["Report Date"], DATETIME_ISOFORMAT + ) self.assertTrue(reportDateDT < sendDateDT) - self.assertTrue(beforeReportGeneration <= reportDateDT - <= afterReportGeneration) - self.assertTrue(beforePayloadGeneration <= sendDateDT - <= afterPayloadGeneration) + self.assertTrue( + beforeReportGeneration <= reportDateDT <= afterReportGeneration + ) + self.assertTrue( + beforePayloadGeneration <= sendDateDT <= afterPayloadGeneration + ) def testZenossVersionHistory(self): # check current version of report (should be empty?) @@ -289,13 +294,13 @@ def testZenossVersionHistory(self): chd = CallHomeData(self.dmd, True) data = chd.getData() reportDate = data[REPORT_DATE_KEY] - zenossVersion = data['Zenoss App Data']['Zenoss'] + zenossVersion = data["Zenoss App Data"]["Zenoss"] # make sure report has Zenoss version history record self.assertTrue(VERSION_HISTORIES_KEY in data) versionHistories = data[VERSION_HISTORIES_KEY] - self.assertTrue('Zenoss' in versionHistories) - versionHistory = versionHistories['Zenoss'] + self.assertTrue("Zenoss" in versionHistories) + versionHistory = versionHistories["Zenoss"] self.assertTrue(zenossVersion in versionHistory) historyRecord = versionHistory[zenossVersion] self.assertTrue(VERSION_START_KEY in historyRecord) @@ -384,23 +389,24 @@ def testSendMethod(self): # reconstitute metrics obj & make sure send date is present # and has a valid time - metricsObj = json.loads(payloadObj['metrics']) - self.assertTrue('Send Method' in metricsObj) - self.assertEquals('directpost', metricsObj['Send Method']) + metricsObj = json.loads(payloadObj["metrics"]) + self.assertTrue("Send Method" in metricsObj) + self.assertEquals("directpost", metricsObj["Send Method"]) # Fetch the payload the browserjs way payloadGenerator = CallHome(self.dmd) - payload = payloadGenerator.get_payload(method='browserjs', - doEncrypt=False) + payload = payloadGenerator.get_payload( + method="browserjs", doEncrypt=False + ) # reconstitute object payloadObj = json.loads(payload) # reconstitute metrics obj & make sure send date is present # and has a valid time - metricsObj = json.loads(payloadObj['metrics']) - self.assertTrue('Send Method' in metricsObj) - self.assertEquals('browserjs', metricsObj['Send Method']) + metricsObj = json.loads(payloadObj["metrics"]) + self.assertTrue("Send Method" in metricsObj) + self.assertEquals("browserjs", metricsObj["Send Method"]) def testGenerateReportWithEmptyMetricsField(self): # Make sure that an empty metrics field @@ -412,21 +418,21 @@ def testGenerateReportWithEmptyMetricsField(self): # call callhome scripting chd = CallHomeData(self.dmd, True) - data = chd.getData() # noqa F841 + data = chd.getData() # noqa F841 # Then handle empty string value self.dmd.callHome.metrics = "" # call callhome scripting chd = CallHomeData(self.dmd, True) - data = chd.getData() # noqa F841 + data = chd.getData() # noqa F841 # Then handle whitespace-only string value self.dmd.callHome.metrics = " " # call callhome scripting chd = CallHomeData(self.dmd, True) - data = chd.getData() # noqa F841 + data = chd.getData() # noqa F841 # # UNFORTUNATELY CANNOT EASILY UNIT TEST TIMEOUTS BECAUSE @@ -438,6 +444,7 @@ def testGenerateReportWithEmptyMetricsField(self): def test_suite(): from unittest import TestSuite, makeSuite + suite = TestSuite() suite.addTest(makeSuite(testCallHomeGeneration)) return suite diff --git a/Products/ZenCallHome/tests/testVersionHistory.py b/Products/ZenCallHome/tests/testVersionHistory.py index ff4cdbdcfe..084344abb7 100644 --- a/Products/ZenCallHome/tests/testVersionHistory.py +++ b/Products/ZenCallHome/tests/testVersionHistory.py @@ -7,16 +7,16 @@ # ############################################################################## - from datetime import datetime, timedelta - from Products.ZenTestCase.BaseTestCase import BaseTestCase + from Products.ZenCallHome.callhome import REPORT_DATE_KEY from Products.ZenCallHome.VersionHistory import ( - VERSION_START_KEY, - VERSION_HISTORIES_KEY, - KeyedVersionHistoryCallHomeCollector) + VERSION_START_KEY, + VERSION_HISTORIES_KEY, + KeyedVersionHistoryCallHomeCollector, +) TEST_ENTITY = "testentity" @@ -34,8 +34,8 @@ TEST_VERSION_VALUE_1 = "versionstring_1" TEST_VERSION_VALUE_2 = "versionstring_2" REPORT_DATE_VALUE_1 = datetime.utcnow() -REPORT_DATE_VALUE_2 = (REPORT_DATE_VALUE_1 + timedelta(days=1)) -REPORT_DATE_VALUE_3 = (REPORT_DATE_VALUE_2 + timedelta(days=1)) +REPORT_DATE_VALUE_2 = REPORT_DATE_VALUE_1 + timedelta(days=1) +REPORT_DATE_VALUE_3 = REPORT_DATE_VALUE_2 + timedelta(days=1) REPORT_DATE_VALUE_1 = REPORT_DATE_VALUE_1.isoformat() REPORT_DATE_VALUE_2 = REPORT_DATE_VALUE_2.isoformat() REPORT_DATE_VALUE_3 = REPORT_DATE_VALUE_3.isoformat() @@ -46,30 +46,31 @@ def createTestCallHomeData(): "histprop1": "testvalue1", "app": { "histprop2": "testvalue2", - "testversion": TEST_VERSION_VALUE_1 - }, - REPORT_DATE_KEY: REPORT_DATE_VALUE_1 - } + "testversion": TEST_VERSION_VALUE_1, + }, + REPORT_DATE_KEY: REPORT_DATE_VALUE_1, + } + TEST_KEY_MAP = { HISTPROP1_KEY: PROP1_TARGET_KEY, - HISTPROP2_KEY: PROP2_TARGET_KEY + HISTPROP2_KEY: PROP2_TARGET_KEY, } class TestVersionHistoryCollector(KeyedVersionHistoryCallHomeCollector): - """ - """ + """ """ + def __init__(self): - super(TestVersionHistoryCollector, self).__init__(TEST_ENTITY, - TEST_KEY_MAP) + super(TestVersionHistoryCollector, self).__init__( + TEST_ENTITY, TEST_KEY_MAP + ) def getCurrentVersion(self, dmd, callHomeData): return self.getKeyedValue(TEST_VERSION_KEY, callHomeData) class testVersionHistory(BaseTestCase): - def afterSetUp(self): super(testVersionHistory, self).afterSetUp() # zcml.load_config('meta.zcml', Products.ZenCallHome) @@ -101,8 +102,9 @@ def testVersionHistory(self): self.assertTrue(TEST_VERSION_VALUE_1 in versionHistory) historyRecord = versionHistory[TEST_VERSION_VALUE_1] self.assertTrue(VERSION_START_KEY in historyRecord) - self.assertEquals(REPORT_DATE_VALUE_1, - historyRecord[VERSION_START_KEY]) + self.assertEquals( + REPORT_DATE_VALUE_1, historyRecord[VERSION_START_KEY] + ) self.assertTrue(PROP1_TARGET_KEY in historyRecord) self.assertEquals(HISTPROP1_VALUE, historyRecord[PROP1_TARGET_KEY]) self.assertTrue(PROP2_TARGET_KEY in historyRecord) @@ -128,8 +130,9 @@ def testVersionHistory(self): self.assertTrue(TEST_VERSION_VALUE_1 in versionHistory) historyRecord = versionHistory[TEST_VERSION_VALUE_1] self.assertTrue(VERSION_START_KEY in historyRecord) - self.assertEquals(REPORT_DATE_VALUE_1, - historyRecord[VERSION_START_KEY]) + self.assertEquals( + REPORT_DATE_VALUE_1, historyRecord[VERSION_START_KEY] + ) self.assertTrue(PROP1_TARGET_KEY in historyRecord) self.assertEquals(HISTPROP1_VALUE, historyRecord[PROP1_TARGET_KEY]) self.assertTrue(PROP2_TARGET_KEY in historyRecord) @@ -137,7 +140,7 @@ def testVersionHistory(self): # Update the version and report date. testCallHomeData[REPORT_DATE_KEY] = REPORT_DATE_VALUE_3 - testCallHomeData['app']['testversion'] = TEST_VERSION_VALUE_2 + testCallHomeData["app"]["testversion"] = TEST_VERSION_VALUE_2 # Update the version history collector.addVersionHistory(self.dmd, testCallHomeData) @@ -153,17 +156,20 @@ def testVersionHistory(self): self.assertTrue(TEST_VERSION_VALUE_2 in versionHistory) historyRecord = versionHistory[TEST_VERSION_VALUE_2] self.assertTrue(VERSION_START_KEY in historyRecord) - self.assertEquals(REPORT_DATE_VALUE_3, - historyRecord[VERSION_START_KEY]) + self.assertEquals( + REPORT_DATE_VALUE_3, historyRecord[VERSION_START_KEY] + ) self.assertTrue(PROP1_TARGET_KEY in historyRecord) - self.assertEquals(HISTPROP1_SECONDVALUE, - historyRecord[PROP1_TARGET_KEY]) + self.assertEquals( + HISTPROP1_SECONDVALUE, historyRecord[PROP1_TARGET_KEY] + ) self.assertTrue(PROP2_TARGET_KEY in historyRecord) self.assertEquals(HISTPROP2_VALUE, historyRecord[PROP2_TARGET_KEY]) def test_suite(): from unittest import TestSuite, makeSuite + suite = TestSuite() suite.addTest(makeSuite(testVersionHistory)) return suite diff --git a/Products/ZenCallHome/transport/__init__.py b/Products/ZenCallHome/transport/__init__.py index ce13baf8ca..de1771dd32 100644 --- a/Products/ZenCallHome/transport/__init__.py +++ b/Products/ZenCallHome/transport/__init__.py @@ -7,7 +7,6 @@ # ############################################################################## - import base64 import json import logging @@ -15,36 +14,39 @@ import string import time import zlib + from datetime import datetime from persistent.dict import PersistentDict +from Persistence import Persistent +from zenoss.protocols.services.zep import ZepConnectionError from zope.component import getUtilitiesFor -from Persistence import Persistent -from Products.ZenCallHome.transport.crypt import encrypt, decrypt -from Products.ZenCallHome.transport.interfaces import IReturnPayloadProcessor from Products.ZenUtils.Version import Version from Products.Zuul import getFacade -from zenoss.protocols.services.zep import ZepConnectionError -from Products.ZenCallHome.CallHomeStatus import CallHomeStatus -__doc__ = ("Callhome mechanism. Reports anonymous statistics " + - "back to Zenoss, Inc.") +from ..CallHomeStatus import CallHomeStatus +from .crypt import encrypt, decrypt +from .interfaces import IReturnPayloadProcessor + +__doc__ = ( + "Callhome mechanism. Reports anonymous statistics " + "back to Zenoss, Inc." +) # number of seconds between successful checkins -CHECKIN_WAIT = 60*60*24 +CHECKIN_WAIT = 60 * 60 * 24 # number of seconds between checkin attempts (per method) -CHECKIN_ATTEMPT_WAIT = 60*60*2 +CHECKIN_ATTEMPT_WAIT = 60 * 60 * 2 -logger = logging.getLogger('zen.callhome') +logger = logging.getLogger("zen.callhome") def is_callhome_disabled(dmd): - return not getattr(dmd, 'versionCheckOptIn', True) + return not getattr(dmd, "versionCheckOptIn", True) class CallHome(object): - def __init__(self, dmd): self.dmd = dmd try: @@ -55,58 +57,54 @@ def __init__(self, dmd): self.chs = CallHomeStatus() def attempt(self, method): - ''' + """ Decide whether or not to attempt a callhome. This is computed from the time elapsed from last successful callhome, or time elapsed from the last attempt via the method passed in with the 'method' param. - ''' - if (is_callhome_disabled(self.dmd) and - not self.callHome.requestCallhome): + """ + if ( + is_callhome_disabled(self.dmd) + and not self.callHome.requestCallhome + ): return False - now = long(time.time()) + now = int(time.time()) # If we have waited long enough between checkings or attempts (or one # has been requested), and we have metrics to send and are not # currently updating them, then attempt a callhome if ( - ( - now - self.callHome.lastAttempt[method] > CHECKIN_ATTEMPT_WAIT - and - now - self.callHome.lastSuccess > CHECKIN_WAIT - ) - or - self.callHome.requestCallhome - ): + now - self.callHome.lastAttempt[method] > CHECKIN_ATTEMPT_WAIT + and now - self.callHome.lastSuccess > CHECKIN_WAIT + ) or self.callHome.requestCallhome: if ( self.callHome.metrics - and - not self.callHome.requestMetricsGather - ): + and not self.callHome.requestMetricsGather + ): self.callHome.lastAttempt[method] = now self.callHome.requestCallhome = False return True return False - def get_payload(self, method='directpost', doEncrypt=True): - ''' + def get_payload(self, method="directpost", doEncrypt=True): + """ Retrieve the current callhome payload to send. This is the call that occurs at send/request time (as opposed to the time that the report was generated). - ''' + """ payload = {} # product info - payload['product'] = self.dmd.getProductName() - payload['uuid'] = self.dmd.uuid or "NOT ACTIVATED" - payload['symkey'] = self.callHome.symmetricKey + payload["product"] = self.dmd.getProductName() + payload["uuid"] = self.dmd.uuid or "NOT ACTIVATED" + payload["symkey"] = self.callHome.symmetricKey metrics = self.callHome.metrics metricsObj = json.loads(metrics) - metricsObj['Send Date'] = datetime.utcnow().isoformat() - metricsObj['Send Method'] = method + metricsObj["Send Date"] = datetime.utcnow().isoformat() + metricsObj["Send Method"] = method - payload['metrics'] = json.dumps(metricsObj) + payload["metrics"] = json.dumps(metricsObj) payloadString = json.dumps(payload) if doEncrypt: @@ -115,62 +113,73 @@ def get_payload(self, method='directpost', doEncrypt=True): return payloadString def save_return_payload(self, returnPayload): - ''' + """ Process and save the data returned from the callhome server. This always includes versioning and crypto key changes, and may include other data to be processed by plugins to the IReturnPayloadProcessor interface. - ''' + """ try: - returnPayload = zlib.decompress(base64.urlsafe_b64decode( - returnPayload)) + returnPayload = zlib.decompress( + base64.urlsafe_b64decode(returnPayload) + ) returnPayload = json.loads(returnPayload) except Exception: - logger.debug('Error decoding return payload from server') + logger.debug("Error decoding return payload from server") return - if all(x in returnPayload for x in ('currentPublicKey', - 'revocationList')): + if all( + x in returnPayload for x in ("currentPublicKey", "revocationList") + ): # TODO: VERIFY revocation list, and apply - newPubkey = returnPayload.get('currentPublicKey') + newPubkey = returnPayload.get("currentPublicKey") if self.callHome.publicKey != newPubkey: self.callHome.publicKey = newPubkey - if 'encrypted' in returnPayload: - base64data = base64.urlsafe_b64decode(str(returnPayload.get( - 'encrypted'))) + if "encrypted" in returnPayload: + base64data = base64.urlsafe_b64decode( + str(returnPayload.get("encrypted")) + ) data = json.loads(decrypt(base64data, self.callHome.symmetricKey)) - if 'compliancereport' in data: - data['compliancereport']['pdf'] = base64.urlsafe_b64decode(str( - data['compliancereport']['pdf'])) + if "compliancereport" in data: + data["compliancereport"]["pdf"] = base64.urlsafe_b64decode( + str(data["compliancereport"]["pdf"]) + ) - if 'latestVersion' in data: + if "latestVersion" in data: # Save the latest version, and send a # message if new version available - self.dmd.lastVersionCheck = long(time.time()) - available = Version.parse('Zenoss ' + data['latestVersion']) - if (getattr(self.dmd, 'availableVersion', '') - != available.short()): + self.dmd.lastVersionCheck = int(time.time()) + available = Version.parse("Zenoss " + data["latestVersion"]) + if ( + getattr(self.dmd, "availableVersion", "") + != available.short() + ): self.dmd.availableVersion = available.short() if self.dmd.About.getZenossVersion() < available: try: import socket - zep = getFacade('zep') - summary = ('A new version of Zenoss (%s)' + - 'has been released') % available.short() - zep.create(summary, 'Info', socket.getfqdn()) + + zep = getFacade("zep") + summary = ( + "A new version of Zenoss (%s)" + + "has been released" + ) % available.short() + zep.create(summary, "Info", socket.getfqdn()) except ZepConnectionError: - logger.warning("ZEP not running - can't send " + - "new version event") + logger.warning( + "ZEP not running - can't send " + + "new version event" + ) # Go through other data in the return payload, and process for name, utility in getUtilitiesFor(IReturnPayloadProcessor): if name in data: utility.process(self.dmd, data[name]) - self.callHome.lastSuccess = long(time.time()) - self.chs.updateStat('lastSuccess', long(time.time())) + self.callHome.lastSuccess = int(time.time()) + self.chs.updateStat("lastSuccess", int(time.time())) return @@ -180,13 +189,15 @@ def __init__(self): self.requestCallhome = False self.lastAttempt = PersistentDict() - self.lastAttempt['browserjs'] = 0 - self.lastAttempt['directpost'] = 0 - - self.publicKey = 'EC7EFA98' - keyParts = ((random.choice(string.ascii_letters + string.digits) - for x in range(64))) - self.symmetricKey = ''.join(keyParts) + self.lastAttempt["browserjs"] = 0 + self.lastAttempt["directpost"] = 0 + + self.publicKey = "EC7EFA98" + keyParts = ( + random.choice(string.ascii_letters + string.digits) + for x in range(64) + ) + self.symmetricKey = "".join(keyParts) self.metrics = None self.lastMetricsGather = 0 diff --git a/Products/ZenCallHome/transport/crypt/__init__.py b/Products/ZenCallHome/transport/crypt/__init__.py index 70388d7123..02ccf91e38 100644 --- a/Products/ZenCallHome/transport/crypt/__init__.py +++ b/Products/ZenCallHome/transport/crypt/__init__.py @@ -7,49 +7,61 @@ # ############################################################################## - import logging import os import subprocess from Products.ZenUtils.Utils import zenPath -logger = logging.getLogger('zen.callhome') +logger = logging.getLogger("zen.callhome") -CRYPTPATH = zenPath('Products', 'ZenCallHome', 'transport', 'crypt') -GPGCMD = 'gpg --batch --no-tty --quiet --no-auto-check-trustdb ' +CRYPTPATH = zenPath("Products", "ZenCallHome", "transport", "crypt") +GPGCMD = "gpg --batch --no-tty --quiet --no-auto-check-trustdb " def _getEnv(): env = os.environ.copy() - env.pop('GPG_AGENT_INFO', None) + env.pop("GPG_AGENT_INFO", None) return env def encrypt(stringToEncrypt, publicKey): - cmd = (GPGCMD + '--keyring %s --trustdb-name %s -e -r %s' % - (CRYPTPATH + '/pubring.gpg', - CRYPTPATH + '/trustdb.gpg', - publicKey)) + cmd = GPGCMD + "--keyring %s --trustdb-name %s -e -r %s" % ( + CRYPTPATH + "/pubring.gpg", + CRYPTPATH + "/trustdb.gpg", + publicKey, + ) - p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, env=_getEnv(), - stdout=subprocess.PIPE, stderr=open(os.devnull)) + p = subprocess.Popen( + cmd, + shell=True, + stdin=subprocess.PIPE, + env=_getEnv(), + stdout=subprocess.PIPE, + stderr=open(os.devnull), + ) out = p.communicate(input=stringToEncrypt)[0] if p.returncode != 0: - logger.warn('Unable to encrypt payload -- is GPG installed?') + logger.warn("Unable to encrypt payload -- is GPG installed?") return None return out def decrypt(stringToDecrypt, symKey): - cmd = GPGCMD + '--passphrase %s -d' % symKey + cmd = GPGCMD + "--passphrase %s -d" % symKey - p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, env=_getEnv(), - stdout=subprocess.PIPE, stderr=open(os.devnull)) + p = subprocess.Popen( + cmd, + shell=True, + stdin=subprocess.PIPE, + env=_getEnv(), + stdout=subprocess.PIPE, + stderr=open(os.devnull), + ) out = p.communicate(input=stringToDecrypt)[0] if p.returncode != 0: - logger.warn('Unable to decrypt payload -- is GPG installed?') + logger.warn("Unable to decrypt payload -- is GPG installed?") return None return out diff --git a/Products/ZenCallHome/transport/cycler.py b/Products/ZenCallHome/transport/cycler.py index f6d8bf335a..f4518435dc 100644 --- a/Products/ZenCallHome/transport/cycler.py +++ b/Products/ZenCallHome/transport/cycler.py @@ -7,32 +7,33 @@ # ############################################################################## - import logging import os import time import transaction + +from twisted.internet import reactor from twisted.internet.protocol import ProcessProtocol from twisted.internet.task import LoopingCall -from twisted.internet import reactor -from Products.ZenCallHome.transport import CallHome -from Products.ZenCallHome.transport.methods.directpost import direct_post from Products.ZenUtils.Utils import zenPath from Products.Zuul.utils import safe_hasattr -from Products.ZenCallHome.CallHomeStatus import CallHomeStatus + +from ..CallHomeStatus import CallHomeStatus +from . import CallHome +from .methods.directpost import direct_post # number of seconds between metrics updates -GATHER_METRICS_INTERVAL = 60*60*24*30 +GATHER_METRICS_INTERVAL = 60 * 60 * 24 * 30 -logger = logging.getLogger('zen.callhome') +logger = logging.getLogger("zen.callhome") class CallHomeCycler(object): def __init__(self, dmd): self.dmd = dmd - if not safe_hasattr(dmd, 'callHome') or dmd.callHome is None: + if not safe_hasattr(dmd, "callHome") or dmd.callHome is None: dmd._p_jar.sync() CallHome(dmd).callHome transaction.commit() @@ -46,34 +47,33 @@ def run(self): chs = CallHomeStatus() chs.stage(chs.START_CALLHOME) try: - now = long(time.time()) + now = int(time.time()) self.dmd._p_jar.sync() # Start metrics gather if needed if ( - ( - now - self.callhome.lastMetricsGather > - GATHER_METRICS_INTERVAL - or - self.callhome.requestMetricsGather - ) - and not - self.gatherProtocol - ): + now - self.callhome.lastMetricsGather > GATHER_METRICS_INTERVAL + or self.callhome.requestMetricsGather + ) and not self.gatherProtocol: self.gatherProtocol = GatherMetricsProtocol() self.callhome.requestMetricsGather = True # Update metrics if run complete - if self.gatherProtocol and (self.gatherProtocol.data or - self.gatherProtocol.failed): + if self.gatherProtocol and ( + self.gatherProtocol.data or self.gatherProtocol.failed + ): chs.stage(chs.GPROTOCOL) if not self.gatherProtocol.failed: self.callhome.metrics = self.gatherProtocol.data try: chs.stage(chs.GPROTOCOL, "FINISHED") chs.stage(chs.UPDATE_REPORT, "FINISHED") - chs.updateStat('lastTook', int(time.time()) - chs.getStat('startedAt')) + chs.updateStat( + "lastTook", int(time.time()) - chs.getStat("startedAt") + ) except Exception as e: - logger.warning("Callhome cycle status update failed: '%r'", e) + logger.warning( + "Callhome cycle status update failed: '%r'", e + ) self.callhome.lastMetricsGather = now self.callhome.requestMetricsGather = False self.gatherProtocol = None @@ -88,15 +88,15 @@ def run(self): class GatherMetricsProtocol(ProcessProtocol): - def __init__(self): self.data = None self.failed = False self.output = [] self.error = [] - chPath = zenPath('Products', 'ZenCallHome', 'callhome.py') - reactor.spawnProcess(self, 'python', args=['python', chPath, '-M'], - env=os.environ) + chPath = zenPath("Products", "ZenCallHome", "callhome.py") + reactor.spawnProcess( + self, "python", args=["python", chPath, "-M"], env=os.environ + ) def outReceived(self, data): self.output.append(data) @@ -105,11 +105,17 @@ def errReceived(self, data): self.error.append(data) def processEnded(self, reason): - out = ''.join(self.output) - err = ''.join(self.error) + out = "".join(self.output) + err = "".join(self.error) if reason.value.exitCode != 0: self.failed = True - logger.warning(('Callhome metrics gathering failed: ' + - 'stdout: %s, stderr: %s'), out, err) + logger.warning( + ( + "Callhome metrics gathering failed: " + + "stdout: %s, stderr: %s" + ), + out, + err, + ) else: self.data = out diff --git a/Products/ZenCallHome/transport/methods/browserjs.py b/Products/ZenCallHome/transport/methods/browserjs.py index c7f5fedba8..ebfda11334 100644 --- a/Products/ZenCallHome/transport/methods/browserjs.py +++ b/Products/ZenCallHome/transport/methods/browserjs.py @@ -7,7 +7,6 @@ # ############################################################################## - import base64 import json import logging @@ -23,52 +22,60 @@ from Products.ZenCallHome.transport import CallHome -JS_CALLHOME_URL = 'https://callhome.zenoss.com/callhome/v2/js' +JS_CALLHOME_URL = "https://callhome.zenoss.com/callhome/v2/js" MAX_GET_SIZE = 768 -logger = logging.getLogger('zen.callhome') +logger = logging.getLogger("zen.callhome") def split_to_range(strToSplit, maxSize): - return ([strToSplit[i:i+maxSize] - for i in range(0, len(strToSplit), maxSize)]) + return [ + strToSplit[i : i + maxSize] for i in range(0, len(strToSplit), maxSize) + ] def encode_for_js(toEnc): base64ToEnc = base64.urlsafe_b64encode(toEnc) - randToken = (''.join(random.choice(string.ascii_letters + string.digits) - for x in range(8))) + randToken = "".join( + random.choice(string.ascii_letters + string.digits) for x in range(8) + ) encPackets = split_to_range(base64ToEnc, MAX_GET_SIZE) - encPackets = [json.dumps({ - 'idx': x, - 'tot': len(encPackets), - 'rnd': randToken, - 'dat': encPackets[x]}) for x in range(len(encPackets))] + encPackets = [ + json.dumps( + { + "idx": x, + "tot": len(encPackets), + "rnd": randToken, + "dat": encPackets[x], + } + ) + for x in range(len(encPackets)) + ] return [base64.urlsafe_b64encode(zlib.compress(x)) for x in encPackets] +@interface.implementer(IHeadExtraManager) class ScriptTag(viewlet.ViewletBase): """ JS script tag injector for browser-based checkins """ - interface.implements(IHeadExtraManager) def render(self): dmd = self.context.dmd # if not logged in, inject nothing if not dmd.ZenUsers.getUserSettings(): - return '' + return "" callhome = CallHome(dmd) # if we've checked in or attempted to check in recently, inject nothing - if not callhome.attempt('browserjs'): - return '' + if not callhome.attempt("browserjs"): + return "" - payload = callhome.get_payload(method='browserjs') + payload = callhome.get_payload(method="browserjs") if not payload: - logger.warning('Error getting or encrypting payload for browserjs') - return '' + logger.warning("Error getting or encrypting payload for browserjs") + return "" # Output the checkin data to a js snippet, wait a few seconds in the # browser, and inject script tags to the checkin url to the body tag. @@ -89,8 +96,10 @@ def render(self): }; var task = new Ext.util.DelayedTask(Zenoss.Callhome_next); task.delay(5000); - """ % (json.dumps(encode_for_js(payload)), - JS_CALLHOME_URL) + """ % ( + json.dumps(encode_for_js(payload)), + JS_CALLHOME_URL, + ) class CallhomeRouter(DirectRouter): @@ -98,4 +107,4 @@ def checkin(self, returnPayload): # record successful check in callhome = CallHome(self.context.dmd) callhome.save_return_payload(returnPayload) - return '' + return "" diff --git a/Products/ZenCallHome/transport/methods/directpost.py b/Products/ZenCallHome/transport/methods/directpost.py index 2c8a7317db..5e3cf73ff8 100644 --- a/Products/ZenCallHome/transport/methods/directpost.py +++ b/Products/ZenCallHome/transport/methods/directpost.py @@ -7,34 +7,34 @@ # ############################################################################## - import base64 import logging -from urllib import urlencode import urllib2 -from Products.ZenCallHome.transport import CallHome +from urllib import urlencode + from Products.ZenCallHome.CallHomeStatus import CallHomeStatus +from Products.ZenCallHome.transport import CallHome -POST_CHECKIN_URL = 'https://callhome.zenoss.com/callhome/v2/post' +POST_CHECKIN_URL = "https://callhome.zenoss.com/callhome/v2/post" _URL_TIMEOUT = 5 -logger = logging.getLogger('zen.callhome') +logger = logging.getLogger("zen.callhome") def direct_post(dmd): callhome = CallHome(dmd) chs = CallHomeStatus() - if not callhome.attempt('directpost'): + if not callhome.attempt("directpost"): return payload = callhome.get_payload() if not payload: - logger.warning('Error getting or encrypting payload for direct-post') + logger.warning("Error getting or encrypting payload for direct-post") return payload = base64.urlsafe_b64encode(payload) - params = urlencode({'enc': payload}) + params = urlencode({"enc": payload}) chs.stage(chs.REQUEST_CALLHOME) try: @@ -42,9 +42,7 @@ def direct_post(dmd): returnPayload = httpreq.read() except Exception as e: chs.stage(chs.REQUEST_CALLHOME, "FAILED", str(e)) - logger.warning('Error retrieving data from callhome server %s', e) + logger.warning("Error retrieving data from callhome server %s", e) else: chs.stage(chs.REQUEST_CALLHOME, "FINISHED") callhome.save_return_payload(returnPayload) - - return diff --git a/Products/ZenCallHome/transport/methods/versioncheck.py b/Products/ZenCallHome/transport/methods/versioncheck.py index 3813d9552c..a6683a27a7 100644 --- a/Products/ZenCallHome/transport/methods/versioncheck.py +++ b/Products/ZenCallHome/transport/methods/versioncheck.py @@ -7,30 +7,31 @@ # ############################################################################## - import json import logging import time -from urllib import urlencode + import urllib2 +from urllib import urlencode + from Products.ZenUtils.Version import Version -VERSION_CHECK_URL = 'https://callhome.zenoss.com/callhome/v2/versioncheck' +VERSION_CHECK_URL = "https://callhome.zenoss.com/callhome/v2/versioncheck" _URL_TIMEOUT = 5 -logger = logging.getLogger('zen.callhome') +logger = logging.getLogger("zen.callhome") def version_check(dmd): - params = urlencode({'product': dmd.getProductName()}) + params = urlencode({"product": dmd.getProductName()}) try: httpreq = urllib2.urlopen(VERSION_CHECK_URL, params, _URL_TIMEOUT) returnPayload = json.loads(httpreq.read()) except Exception as e: - logger.warning('Error retrieving version from callhome server: %s', e) + logger.warning("Error retrieving version from callhome server: %s", e) else: - available = Version.parse('Zenoss ' + returnPayload['latest']) + available = Version.parse("Zenoss " + returnPayload["latest"]) version = available.short() - dmd.lastVersionCheck = long(time.time()) - if getattr(dmd, 'availableVersion', '') != version: + dmd.lastVersionCheck = int(time.time()) + if getattr(dmd, "availableVersion", "") != version: dmd.availableVersion = version From e19bb71aebd74cd7f09fa98723de33b598b65397 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 9 May 2023 14:05:45 -0500 Subject: [PATCH 005/147] Minor changes in ZenEvent to match changes in develop branch --- Products/ZenEvents/TrapFilter.py | 8 ++++---- Products/ZenEvents/skins/zenevents/zepConfig.pt | 2 +- Products/ZenEvents/zeneventd.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Products/ZenEvents/TrapFilter.py b/Products/ZenEvents/TrapFilter.py index a93f68cae1..2ca404cbac 100644 --- a/Products/ZenEvents/TrapFilter.py +++ b/Products/ZenEvents/TrapFilter.py @@ -337,16 +337,16 @@ def _readFilters(self): errorMessage = self._parseFilterDefinition(line, lineNumber) if errorMessage: - errorMessage = "Failed to parse filter definition file %s at line %d: %s" % (format(path), lineNumber, errorMessage) + errorMessage = "Failed to parse filter definition file %s at line %d: %s" % (path, lineNumber, errorMessage) raise TrapFilterError(errorMessage) self._filtersDefined = 0 != (len(self._v1Traps) + len(self._v1Filters) + len(self._v2Filters)) if self._filtersDefined: - log.info("Finished reading filter definition file %s", format(path)) + log.info("Finished reading filter definition file %s", path) else: - log.warn("No zentrap filters found in %s", format(path)) + log.warn("No zentrap filters found in %s", path) else: - errorMessage = "Could find filter definition file %s" % format(path) + errorMessage = "Could find filter definition file %s" % (path,) raise TrapFilterError(errorMessage) def initialize(self): diff --git a/Products/ZenEvents/skins/zenevents/zepConfig.pt b/Products/ZenEvents/skins/zenevents/zepConfig.pt index 1f6f3f8334..2b681dd0b1 100644 --- a/Products/ZenEvents/skins/zenevents/zepConfig.pt +++ b/Products/ZenEvents/skins/zenevents/zepConfig.pt @@ -11,7 +11,7 @@ } + href="/++resource++zenui/css/xtheme-zenoss.css" /> diff --git a/Products/ZenEvents/zeneventd.py b/Products/ZenEvents/zeneventd.py index f012507d59..a3c5dbb8e8 100644 --- a/Products/ZenEvents/zeneventd.py +++ b/Products/ZenEvents/zeneventd.py @@ -260,8 +260,12 @@ def processMessage(self, message): if log.isEnabledFor(logging.DEBUG): # assume to_dict() is expensive. log.debug("Publishing event: %s", to_dict(zepRawEvent)) - yield self.queueConsumer.publishMessage(EXCHANGE_ZEP_ZEN_EVENTS, - self._routing_key(zepRawEvent), zepRawEvent, declareExchange=False) + yield self.queueConsumer.publishMessage( + EXCHANGE_ZEP_ZEN_EVENTS, + self._routing_key(zepRawEvent), + zepRawEvent, + declareExchange=False + ) yield self.queueConsumer.acknowledge(message) except DropEvent as e: if log.isEnabledFor(logging.DEBUG): @@ -282,7 +286,9 @@ def __init__(self, dmd): super(EventDTwistedWorker, self).__init__() self._amqpConnectionInfo = getUtility(IAMQPConnectionInfo) self._queueSchema = getUtility(IQueueSchema) - self._consumer_task = TwistedQueueConsumerTask(EventPipelineProcessor(dmd)) + self._consumer_task = TwistedQueueConsumerTask( + EventPipelineProcessor(dmd) + ) self._consumer = QueueConsumer(self._consumer_task, dmd) def run(self): From a4a969907ba2de914c5f9d3bf8eb0061ccbb46eb Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 9 May 2023 15:39:02 -0500 Subject: [PATCH 006/147] rejoin separated strings --- Products/ZenCallHome/transport/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Products/ZenCallHome/transport/__init__.py b/Products/ZenCallHome/transport/__init__.py index de1771dd32..7ccdbf1010 100644 --- a/Products/ZenCallHome/transport/__init__.py +++ b/Products/ZenCallHome/transport/__init__.py @@ -30,8 +30,7 @@ from .interfaces import IReturnPayloadProcessor __doc__ = ( - "Callhome mechanism. Reports anonymous statistics " - "back to Zenoss, Inc." + "Callhome mechanism. Reports anonymous statistics back to Zenoss, Inc." ) # number of seconds between successful checkins From 225b27abe98bb9ec46fcec777920f77b6ff6e8d1 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 9 May 2023 16:31:21 -0500 Subject: [PATCH 007/147] Fixed a unit test --- .../ApplyDataMap/tests/test_datamaputils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py b/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py index 04e90b2f17..d91a36106f 100644 --- a/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py +++ b/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py @@ -316,22 +316,22 @@ def test_encoding(t): ), "utf-8": ObjectMap( { - "a": six.text_type("\xe0").encode("utf-8"), - "b": six.text_type("\xe0").encode("utf-8"), - "c": six.text_type("\xe0").encode("utf-8"), + "a": six.u("\xe0").encode("utf-8"), + "b": six.u("\xe0").encode("utf-8"), + "c": six.u("\xe0").encode("utf-8"), } ), "latin-1": ObjectMap( { - "a": six.text_type("\xe0").encode("latin-1"), - "b": six.text_type("\xe0").encode("latin-1"), - "c": six.text_type("\xe0").encode("latin-1"), + "a": six.u("\xe0").encode("latin-1"), + "b": six.u("\xe0").encode("latin-1"), + "c": six.u("\xe0").encode("latin-1"), } ), "utf-16": ObjectMap( { - "a": six.text_type("\xff\xfeabcdef").encode("utf-16"), - "b": six.text_type("\xff\xfexyzwow").encode("utf-16"), + "a": six.u("\xff\xfeabcdef").encode("utf-16"), + "b": six.u("\xff\xfexyzwow").encode("utf-16"), # (water, z, G clef), UTF-16 encoded, # little-endian with BOM "c": r"\xff\xfe\x34\x6c\x7a\x00\x34\xd8\x13\xdd", From 8485698e8d0bacae16c8fe86e48d337ea64888d0 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 12 May 2023 15:53:47 -0500 Subject: [PATCH 008/147] Back out some uses of six library. Minor formatting changes in DataCollector, Jobber, ZenHub, ZenMessaging, ZenRelations, ZenReports, and ZenTestCase packages. --- .../ApplyDataMap/tests/test_datamaputils.py | 18 +- Products/Jobber/interfaces.py | 54 +++--- Products/Jobber/metadirectives.py | 20 +-- Products/ZenCollector/scheduler.py | 4 +- Products/ZenHub/metricpublisher/utils.py | 2 +- Products/ZenHub/server/priority.py | 2 +- .../ZenMessaging/queuemessaging/adapters.py | 5 +- Products/ZenRelations/tests/testEvents.py | 1 - Products/ZenReports/plugins/interface.py | 6 +- .../ZenReports/tests/test_report_loader.py | 170 ++++++++++-------- Products/ZenTestCase/BaseTestCase.py | 163 +++++++++++------ Products/ZenTestCase/__init__.py | 9 +- 12 files changed, 252 insertions(+), 202 deletions(-) diff --git a/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py b/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py index d91a36106f..e82783fd4f 100644 --- a/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py +++ b/Products/DataCollector/ApplyDataMap/tests/test_datamaputils.py @@ -9,8 +9,6 @@ from base64 import b64encode -import six - from mock import Mock, sentinel, patch from Products.DataCollector.plugins.DataMaps import ObjectMap @@ -316,22 +314,22 @@ def test_encoding(t): ), "utf-8": ObjectMap( { - "a": six.u("\xe0").encode("utf-8"), - "b": six.u("\xe0").encode("utf-8"), - "c": six.u("\xe0").encode("utf-8"), + "a": u"\xe0".encode("utf-8"), + "b": u"\xe0".encode("utf-8"), + "c": u"\xe0".encode("utf-8"), } ), "latin-1": ObjectMap( { - "a": six.u("\xe0").encode("latin-1"), - "b": six.u("\xe0").encode("latin-1"), - "c": six.u("\xe0").encode("latin-1"), + "a": u"\xe0".encode("latin-1"), + "b": u"\xe0".encode("latin-1"), + "c": u"\xe0".encode("latin-1"), } ), "utf-16": ObjectMap( { - "a": six.u("\xff\xfeabcdef").encode("utf-16"), - "b": six.u("\xff\xfexyzwow").encode("utf-16"), + "a": u"\xff\xfeabcdef".encode("utf-16"), + "b": u"\xff\xfexyzwow".encode("utf-16"), # (water, z, G clef), UTF-16 encoded, # little-endian with BOM "c": r"\xff\xfe\x34\x6c\x7a\x00\x34\xd8\x13\xdd", diff --git a/Products/Jobber/interfaces.py b/Products/Jobber/interfaces.py index 2e1beee9ad..42dbc366ee 100644 --- a/Products/Jobber/interfaces.py +++ b/Products/Jobber/interfaces.py @@ -7,9 +7,7 @@ # ############################################################################## -from __future__ import absolute_import - -import six +from __future__ import absolute_import, unicode_literals from celery import states from zope.interface import Interface @@ -21,66 +19,64 @@ class IJobRecord(Interface): """ """ jobid = TextLine( - title=six.text_type("Job ID"), - description=six.text_type("The Job's unique identifier"), + title="Job ID", + description="The Job's unique identifier", ) name = TextLine( - title=six.text_type("Name"), - description=six.text_type("The full class name of the job"), + title="Name", + description="The full class name of the job", ) summary = TextLine( - title=six.text_type("Summary"), - description=six.text_type( - "A brief and general summary of the job's function" - ), + title="Summary", + description="A brief and general summary of the job's function", ) description = TextLine( - title=six.text_type("Description"), - description=six.text_type("A description of what this job will do"), + title="Description", + description="A description of what this job will do", ) userid = TextLine( - title=six.text_type("User ID"), - description=six.text_type("The user that created the job"), + title="User ID", + description="The user that created the job", ) logfile = TextLine( - title=six.text_type("Logfile"), - description=six.text_type("Path to this job's log file."), + title="Logfile", + description="Path to this job's log file.", ) status = Choice( - title=six.text_type("Status"), - description=six.text_type("The current status of the job"), + title="Status", + description="The current status of the job", vocabulary=SimpleVocabulary.fromValues(states.ALL_STATES), ) created = Datetime( - title=six.text_type("Created"), - description=six.text_type("When the job was created"), + title="Created", + description="When the job was created", ) started = Datetime( - title=six.text_type("Started"), - description=six.text_type("When the job began executing"), + title="Started", + description="When the job began executing", ) finished = Datetime( - title=six.text_type("Finished"), - description=six.text_type("When the job finished executing"), + title="Finished", + description="When the job finished executing", ) duration = Timedelta( - title=six.text_type("Duration"), - description=six.text_type("How long the job has run"), + title="Duration", + description="How long the job has run", ) complete = Bool( - title=six.text_type("Complete"), - description=six.text_type("True if the job has finished running"), + title="Complete", + description="True if the job has finished running", ) def abort(): diff --git a/Products/Jobber/metadirectives.py b/Products/Jobber/metadirectives.py index accb3b0e51..a4eae6f39b 100644 --- a/Products/Jobber/metadirectives.py +++ b/Products/Jobber/metadirectives.py @@ -9,8 +9,6 @@ from __future__ import absolute_import, unicode_literals -import six - from zope.configuration.fields import GlobalObject from zope.interface import Interface from zope.schema import TextLine @@ -20,13 +18,13 @@ class IJob(Interface): """Registers a ZenJobs Job class.""" class_ = GlobalObject( - title=six.text_type("Job Class"), - description=six.text_type("The class of the job to register"), + title="Job Class", + description="The class of the job to register", ) name = TextLine( - title=six.text_type("Name"), - description=six.text_type("Optional name of the job"), + title="Name", + description="Optional name of the job", required=False, ) @@ -35,13 +33,11 @@ class ICelerySignal(Interface): """Registers a Celery signal handler.""" name = TextLine( - title=six.text_type("Name"), - description=six.text_type("The signal receiving a handler"), + title="Name", + description="The signal receiving a handler", ) handler = TextLine( - title=six.text_type("Handler"), - description=six.text_type( - "Classpath to the function handling the signal" - ), + title="Handler", + description="Classpath to the function handling the signal", ) diff --git a/Products/ZenCollector/scheduler.py b/Products/ZenCollector/scheduler.py index cbd562f61c..c1daac26a7 100644 --- a/Products/ZenCollector/scheduler.py +++ b/Products/ZenCollector/scheduler.py @@ -52,7 +52,7 @@ def __init__(self, state): def addCall(self, elapsedTime): self.totalElapsedTime += elapsedTime - self.totalElapsedTimeSquared += elapsedTime**2 + self.totalElapsedTimeSquared += elapsedTime ** 2 self.totalCalls += 1 if self.totalCalls == 1: @@ -82,7 +82,7 @@ def stddev(self): return math.sqrt( ( self.totalElapsedTimeSquared - - self.totalElapsedTime**2 / self.totalCalls + - self.totalElapsedTime ** 2 / self.totalCalls ) / (self.totalCalls - 1) ) diff --git a/Products/ZenHub/metricpublisher/utils.py b/Products/ZenHub/metricpublisher/utils.py index 3356a72e47..93d41a5a85 100644 --- a/Products/ZenHub/metricpublisher/utils.py +++ b/Products/ZenHub/metricpublisher/utils.py @@ -30,7 +30,7 @@ def inner(*args, **kwargs): return f(*args, **kwargs) except exception: failures += 1 - slots = ((2**failures) - 1) / 2.0 + slots = ((2 ** failures) - 1) / 2.0 mdelay = min(max(slots * mdelay, mdelay), maxdelay) sleepfunc(mdelay) diff --git a/Products/ZenHub/server/priority.py b/Products/ZenHub/server/priority.py index 67c8f99b07..56b72c3429 100644 --- a/Products/ZenHub/server/priority.py +++ b/Products/ZenHub/server/priority.py @@ -150,7 +150,7 @@ def _build_weighted_list(data): # Generate a series of weights. The first element should have the # highest weight. - weights = [(2**n) - 1 for n in range(len(elements), 0, -1)] + weights = [(2 ** n) - 1 for n in range(len(elements), 0, -1)] # Build a list of element lists where each element list has a length # matching their weight. E.g. given elements ('a', 'b') and weights diff --git a/Products/ZenMessaging/queuemessaging/adapters.py b/Products/ZenMessaging/queuemessaging/adapters.py index 405cfffeb6..a90f2ef661 100644 --- a/Products/ZenMessaging/queuemessaging/adapters.py +++ b/Products/ZenMessaging/queuemessaging/adapters.py @@ -380,10 +380,11 @@ def __init__(self, obj): ObjectProtobuf.__init__(self, obj) def addDetail(self, proto, name, value): - isIterable = lambda x : hasattr(x, '__iter__') detail = proto.details.add() detail.name = name - if isIterable(value): + # Test whether 'value' is iterable. + # Avoids strings because strings don't have an __iter__ method. + if hasattr(value, "__iter__"): for v in value: detail.value.append(_safestr(v)) else: diff --git a/Products/ZenRelations/tests/testEvents.py b/Products/ZenRelations/tests/testEvents.py index 5c4c176de4..c5d459be70 100644 --- a/Products/ZenRelations/tests/testEvents.py +++ b/Products/ZenRelations/tests/testEvents.py @@ -54,7 +54,6 @@ class ITestItem(interface.Interface): @interface.implementer(ITestItem, IItem) class TestItem(RelationshipManager): - def __init__(self, id): self.id = id self.buildRelations() diff --git a/Products/ZenReports/plugins/interface.py b/Products/ZenReports/plugins/interface.py index 8934f144f3..053512c1b3 100644 --- a/Products/ZenReports/plugins/interface.py +++ b/Products/ZenReports/plugins/interface.py @@ -18,7 +18,7 @@ class interface(AliasPlugin): - "The interface usage report" + """The interface usage report""" def getComponentPath(self): return "os/interfaces" @@ -52,7 +52,7 @@ def getColumns(self): Column( "tmp_ipAddress", PythonColumnHandler( - 'component.ipaddresses()[0].id if ' + "component.ipaddresses()[0].id if " 'len(component.ipaddresses()) == 1 else ""' ), ), @@ -96,7 +96,7 @@ def getCompositeColumns(self): Column( "totalBits", PythonColumnHandler( - '(input + output) * 8 if input is not None and ' + "(input + output) * 8 if input is not None and " 'output is not None else "N/A"' ), ), diff --git a/Products/ZenReports/tests/test_report_loader.py b/Products/ZenReports/tests/test_report_loader.py index 6a5488016b..ffb5a31b24 100644 --- a/Products/ZenReports/tests/test_report_loader.py +++ b/Products/ZenReports/tests/test_report_loader.py @@ -9,99 +9,112 @@ from Products.ZenTestCase.BaseTestCase import BaseTestCase -from mock import Mock, MagicMock, create_autospec, patch +from mock import Mock, create_autospec, patch from Products.ZenReports.ReportLoader import ( zenPath, - transaction, ReportLoader, - Report + Report, ) class ReportLoaderTest(BaseTestCase): - def setUp(self): - self.rp_load = ReportLoader() + self.rp_load = ReportLoader() def test_loadDatabase(self): - self.rp_load.loadAllReports = create_autospec(self.rp_load.loadAllReports) + self.rp_load.loadAllReports = create_autospec( + self.rp_load.loadAllReports + ) self.rp_load.loadDatabase() self.rp_load.loadAllReports.assert_called_once_with() @patch( - 'Products.ZenReports.ReportLoader.transaction.commit', - autospec=True, spec_set=True + "Products.ZenReports.ReportLoader.transaction.commit", + autospec=True, + spec_set=True, ) def test_loadAllReports(self, commit): - repdir = zenPath('Products/ZenReports', self.rp_load.options.dir) - self.rp_load.loadDirectory = create_autospec(self.rp_load.loadDirectory) + repdir = zenPath("Products/ZenReports", self.rp_load.options.dir) + self.rp_load.loadDirectory = create_autospec( + self.rp_load.loadDirectory + ) self.rp_load.loadAllReports() self.rp_load.loadDirectory.assert_called_once_with(repdir) commit.assert_called_once_with() - def test_loadAllReports_zp(self): + def test_loadAllReports_zp(self): self.rp_load.options.zenpack = True - self.rp_load.getZenPackDirs = create_autospec(self.rp_load.getZenPackDirs) + self.rp_load.getZenPackDirs = create_autospec( + self.rp_load.getZenPackDirs + ) self.rp_load.loadAllReports() - self.rp_load.getZenPackDirs.assert_called_once_with(self.rp_load.options.zenpack) + self.rp_load.getZenPackDirs.assert_called_once_with( + self.rp_load.options.zenpack + ) def test_getZenPackDirs(self): - zp_name = 'test_zp' - zp_path = '/path/to/test_zp' - zp_obj = Mock(id='test_zp') + zp_name = "test_zp" + zp_path = "/path/to/test_zp" + zp_obj = Mock(id="test_zp") zp_obj.path = Mock(return_value=zp_path) self.rp_load.dmd.ZenPackManager.packs = create_autospec( - self.rp_load.dmd.ZenPackManager.packs, - return_value=[zp_obj] + self.rp_load.dmd.ZenPackManager.packs, return_value=[zp_obj] ) - self.rp_load.options.dir = 'reports' - zp_dir_result = ['/path/to/test_zp/reports'] + self.rp_load.options.dir = "reports" + zp_dir_result = ["/path/to/test_zp/reports"] result = self.rp_load.getZenPackDirs(name=zp_name) self.assertEqual(result, zp_dir_result) self.assertIsInstance(result, list) self.assertEqual(len(result), 1) def test_getZenPackDirs_error(self): - zp_name = 'noname_zp' - zp_path = '/path/to/test_zp' - zp_obj = Mock(id='test_zp') + zp_name = "noname_zp" + zp_path = "/path/to/test_zp" + zp_obj = Mock(id="test_zp") zp_obj.path = Mock(return_value=zp_path) self.rp_load.dmd.ZenPackManager.packs = create_autospec( - self.rp_load.dmd.ZenPackManager.packs, - return_value=[zp_obj] + self.rp_load.dmd.ZenPackManager.packs, return_value=[zp_obj] ) - self.rp_load.options.dir = 'reports' - #set loglevel to 50(CRITICAL) this will remove error log - self.rp_load.log.setLevel('CRITICAL') + self.rp_load.options.dir = "reports" + # set loglevel to 50(CRITICAL) this will remove error log + self.rp_load.log.setLevel("CRITICAL") with self.assertRaises(SystemExit) as exc: self.rp_load.getZenPackDirs(name=zp_name) self.assertEqual(exc.exception.code, 1) @patch( - 'Products.ZenReports.ReportLoader.os.walk', - autospec=True, spec_set=True + "Products.ZenReports.ReportLoader.os.walk", + autospec=True, + spec_set=True, ) def test_reports(self, walk): - rp_dir = '/path/to/test_zp/reports/SomeReports' - os_walk_data = [(rp_dir, [], ['reportName.rpt'])] + rp_dir = "/path/to/test_zp/reports/SomeReports" + os_walk_data = [(rp_dir, [], ["reportName.rpt"])] walk.return_value = os_walk_data - ret_data = [('/SomeReports', - 'reportName', '/path/to/test_zp/reports/SomeReports/reportName.rpt' - )] + ret_data = [ + ( + "/SomeReports", + "reportName", + "/path/to/test_zp/reports/SomeReports/reportName.rpt", + ) + ] result = self.rp_load.reports(rp_dir) self.assertEqual(result, ret_data) - def test_unloadDirectory(self): - rp_dir = '/path/to/test_zp/reports/SomeReports' - orgpath = '/SomeReports' - rp_id = 'reportName' - report_data = [(orgpath, - rp_id, '/path/to/test_zp/Reports/SomeReports/reportName.rpt' - )] + rp_dir = "/path/to/test_zp/reports/SomeReports" + orgpath = "/SomeReports" + rp_id = "reportName" + report_data = [ + ( + orgpath, + rp_id, + "/path/to/test_zp/Reports/SomeReports/reportName.rpt", + ) + ] rorg = Mock(id=rp_id) - rorg_parent = Mock(id='Reports') + rorg_parent = Mock(id="Reports") setattr(rorg, rp_id, True) rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) @@ -111,23 +124,27 @@ def test_unloadDirectory(self): self.rp_load.unloadDirectory(repdir=rp_dir) - self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with(orgpath) + self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with( + orgpath + ) rorg._delObject.assert_called_with(rp_id) rorg.objectValues.assert_called_once_with() rorg.getPrimaryParent.assert_called_once_with() - def test_unloadDirectory_false(self): - '''test that _delObject method was not called - ''' - rp_dir = '/path/to/test_zp/reports/SomeReports' - orgpath = '/SomeReports' - rp_id = 'reportName' - report_data = [(orgpath, - rp_id, '/path/to/test_zp/Reports/SomeReports/reportName.rpt' - )] - rorg = Mock(id='Reports') - rorg_parent = Mock(id='Reports') + """test that _delObject method was not called""" + rp_dir = "/path/to/test_zp/reports/SomeReports" + orgpath = "/SomeReports" + rp_id = "reportName" + report_data = [ + ( + orgpath, + rp_id, + "/path/to/test_zp/Reports/SomeReports/reportName.rpt", + ) + ] + rorg = Mock(id="Reports") + rorg_parent = Mock(id="Reports") rorg._delObject = Mock() rorg.objectValues = Mock(return_value=False) rorg.getPrimaryParent = Mock(return_value=rorg_parent) @@ -139,17 +156,16 @@ def test_unloadDirectory_false(self): rorg._delObject.assert_not_called() - def test_loadDirectory_force(self): - full_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' - rp_dir = '/path/to/test_zp/reports/SomeReports' - orgpath = '/SomeReports' - rp_id = 'reportName' + full_path = "/path/to/test_zp/Reports/SomeReports/reportName.rpt" + rp_dir = "/path/to/test_zp/reports/SomeReports" + orgpath = "/SomeReports" + rp_id = "reportName" report_data = [(orgpath, rp_id, full_path)] self.rp_load.options.force = True rorg = Mock() report = Mock() - #set that this report is not from zenpack + # set that this report is not from zenpack report.pack = Mock(return_value=False) setattr(rorg, rp_id, report) rorg._delObject = Mock() @@ -160,20 +176,22 @@ def test_loadDirectory_force(self): self.rp_load.loadDirectory(rp_dir) - self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with(orgpath) + self.rp_load.dmd.Reports.createOrganizer.assert_called_once_with( + orgpath + ) rorg._delObject.assert_called_once_with(rp_id) self.rp_load.loadFile.assert_called_with(rorg, rp_id, full_path) - def test_loadDirectory(self): - '''test that _delObject method was not called and we didn't overwrite reports - ''' - full_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' - rp_dir = '/path/to/test_zp/reports/SomeReports' - orgpath = '/SomeReports' - rp_id = 'reportName' + """ + Test that _delObject method was not called reports wasn't overwritten. + """ + full_path = "/path/to/test_zp/Reports/SomeReports/reportName.rpt" + rp_dir = "/path/to/test_zp/reports/SomeReports" + orgpath = "/SomeReports" + rp_id = "reportName" report_data = [(orgpath, rp_id, full_path)] - #force option is False by default, this is for better clarity + # force option is False by default, this is for better clarity self.rp_load.options.force = False rorg = Mock() setattr(rorg, rp_id, True) @@ -188,15 +206,12 @@ def test_loadDirectory(self): rorg._delObject.assert_not_called() self.rp_load.loadFile.assert_not_called() - @patch( - '__builtin__.file', - autospec=True, spec_set=True - ) + @patch("__builtin__.file", autospec=True, spec_set=True) def test_loadFile(self, file_mock): - rp_name = 'reportName' - full_rp_path = '/path/to/test_zp/Reports/SomeReports/reportName.rpt' + rp_name = "reportName" + full_rp_path = "/path/to/test_zp/Reports/SomeReports/reportName.rpt" report_txt = "some report data" - #mock build in file method and its instance read method + # mock build in file method and its instance read method file_read = Mock() file_read.read = Mock(return_value=report_txt) file_mock.return_value = file_read @@ -206,4 +221,3 @@ def test_loadFile(self, file_mock): self.assertIsInstance(rp, Report) self.assertEqual(rp.id, rp_name) root._setObject.assert_called_once_with(rp_name, rp) - \ No newline at end of file diff --git a/Products/ZenTestCase/BaseTestCase.py b/Products/ZenTestCase/BaseTestCase.py index bf43eb6317..450950d892 100644 --- a/Products/ZenTestCase/BaseTestCase.py +++ b/Products/ZenTestCase/BaseTestCase.py @@ -1,51 +1,47 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - import logging import zope.component -import zenoss.modelindex.api -from zope.traversing.adapters import DefaultTraversable -from transaction._transaction import Transaction from Testing import ZopeTestCase from Testing.ZopeTestCase.layer import ZopeLite - +from transaction._transaction import Transaction +from zenoss.modelindex.model_index import SearchParams from Zope2.App import zcml +from zope.testing.cleanup import cleanUp +from zope.traversing.adapters import DefaultTraversable -from Products.ZenModel.DmdBuilder import DmdBuilder -from Products.ZenModel.ZentinelPortal import PortalGenerator from Products.ZenEvents.EventManagerBase import EventManagerBase +from Products.ZenEvents.MySqlEventManager import log from Products.ZenEvents.MySqlSendEvent import MySqlSendEventMixin +from Products.ZenModel.DmdBuilder import DmdBuilder +from Products.ZenModel.ZentinelPortal import PortalGenerator from Products.ZenRelations.ZenPropertyManager import setDescriptors -from Products.ZenEvents.MySqlEventManager import log from Products.ZenUtils.Utils import unused, load_config_override -from zope.testing.cleanup import cleanUp - from Products.Zuul.catalog.model_catalog import get_solr_config -from zenoss.modelindex.model_index import SearchParams log.warn = lambda *args, **kwds: None # setup the Products needed for the Zenoss test instance -ZopeTestCase.installProduct('ZenModel', 1) -ZopeTestCase.installProduct('ZCatalog', 1) -ZopeTestCase.installProduct('OFolder', 1) -ZopeTestCase.installProduct('ManagableIndex', 1) -ZopeTestCase.installProduct('AdvancedQuery', 1) -ZopeTestCase.installProduct('ZCTextIndex', 1) -ZopeTestCase.installProduct('CMFCore', 1) -ZopeTestCase.installProduct('CMFDefault', 1) -ZopeTestCase.installProduct('MailHost', 1) -ZopeTestCase.installProduct('Transience', 1) -ZopeTestCase.installProduct('ZenRelations', 1) +ZopeTestCase.installProduct("ZenModel", 1) +ZopeTestCase.installProduct("ZCatalog", 1) +ZopeTestCase.installProduct("OFolder", 1) +ZopeTestCase.installProduct("ManagableIndex", 1) +ZopeTestCase.installProduct("AdvancedQuery", 1) +ZopeTestCase.installProduct("ZCTextIndex", 1) +ZopeTestCase.installProduct("CMFCore", 1) +ZopeTestCase.installProduct("CMFDefault", 1) +ZopeTestCase.installProduct("MailHost", 1) +ZopeTestCase.installProduct("Transience", 1) +ZopeTestCase.installProduct("ZenRelations", 1) def manage_addDummyManager(context, id): @@ -56,34 +52,63 @@ def manage_addDummyManager(context, id): class DummyCursor(object): - def __init__(self, *args, **kwds): pass - def execute(self, *args, **kwds): pass + def __init__(self, *args, **kwds): + pass + + def execute(self, *args, **kwds): + pass class DummyConnection(object): - def __init__(self, *args, **kwds): pass + def __init__(self, *args, **kwds): + pass + def cursor(self): return DummyCursor() - def close(self): pass + + def close(self): + pass class DummyManager(MySqlSendEventMixin, EventManagerBase): - __pychecker__ = 'no-override' def __init__(self, *args, **kwds): EventManagerBase.__init__(self, *args, **kwds) + def connect(self, *args, **kwds): unused(args, kwds) return DummyConnection() - def sendEvent(self, *args, **kwds): unused(args, kwds) - def sendEvents(self, *args, **kwds): unused(args, kwds) - def doSendEvent(self, *args, **kwds): unused(args, kwds) - def getEventSummary(self, *args, **kwds): unused(args, kwds) - def getEventDetail(self, *args, **kwds): unused(args, kwds) - def getDeviceIssues(self, *args, **kwds): unused(args, kwds) - def getHeartbeat(self, *args, **kwds): unused(args, kwds) - def getEventList(self, *args, **kwds): unused(args, kwds); return [] - def applyEventContext(self, evt): return evt - def applyDeviceContext(self, dev, evt): unused(dev); return evt + + def sendEvent(self, *args, **kwds): + unused(args, kwds) + + def sendEvents(self, *args, **kwds): + unused(args, kwds) + + def doSendEvent(self, *args, **kwds): + unused(args, kwds) + + def getEventSummary(self, *args, **kwds): + unused(args, kwds) + + def getEventDetail(self, *args, **kwds): + unused(args, kwds) + + def getDeviceIssues(self, *args, **kwds): + unused(args, kwds) + + def getHeartbeat(self, *args, **kwds): + unused(args, kwds) + + def getEventList(self, *args, **kwds): + unused(args, kwds) + return [] + + def applyEventContext(self, evt): + return evt + + def applyDeviceContext(self, dev, evt): + unused(dev) + return evt def reset_model_catalog(): @@ -91,13 +116,22 @@ def reset_model_catalog(): Deletes temporary documents from previous tests. They should be cleaned by abort() but just in case """ - model_index = zope.component.createObject('ModelIndex', get_solr_config(test=True)) + model_index = zope.component.createObject( + "ModelIndex", get_solr_config(test=True) + ) model_index.unindex_search(SearchParams(query="NOT tx_state:0")) def init_model_catalog_for_tests(): - from Products.Zuul.catalog.model_catalog import register_model_catalog, register_data_manager_factory - from zenoss.modelindex.api import _register_factories, reregister_subscriptions + from Products.Zuul.catalog.model_catalog import ( + register_model_catalog, + register_data_manager_factory, + ) + from zenoss.modelindex.api import ( + _register_factories, + reregister_subscriptions, + ) + _register_factories() register_model_catalog(test=True) register_data_manager_factory(test=True) @@ -106,31 +140,36 @@ def init_model_catalog_for_tests(): class ZenossTestCaseLayer(ZopeLite): - @classmethod def testSetUp(cls): import Products zope.component.testing.setUp(cls) zope.component.provideAdapter(DefaultTraversable, (None,)) - zcml.load_config('testing.zcml', Products.ZenTestCase) + zcml.load_config("testing.zcml", Products.ZenTestCase) import Products.ZenMessaging.queuemessaging - load_config_override('nopublisher.zcml', Products.ZenMessaging.queuemessaging) + + load_config_override( + "nopublisher.zcml", Products.ZenMessaging.queuemessaging + ) # Have to force registering these as they are torn down between tests from zenoss.protocols.adapters import registerAdapters + registerAdapters() # Register Model Catalog related stuff init_model_catalog_for_tests() from twisted.python.runtime import platform + platform.supportsThreads_orig = platform.supportsThreads - platform.supportsThreads = lambda : None + platform.supportsThreads = lambda: None @classmethod def testTearDown(cls): from twisted.python.runtime import platform + platform.supportsThreads = platform.supportsThreads_orig cleanUp() @@ -148,32 +187,42 @@ def afterSetUp(self): logging.disable(logging.CRITICAL) gen = PortalGenerator() - if hasattr( self.app, 'zport' ): - self.app._delObject( 'zport', suppress_events=True) + if hasattr(self.app, "zport"): + self.app._delObject("zport", suppress_events=True) - gen.create(self.app, 'zport', True) + gen.create(self.app, "zport", True) # builder params: # portal, cvthost, evtuser, evtpass, evtdb, # smtphost, smtpport, pagecommand - builder = DmdBuilder(self.app.zport, 'localhost', 'zenoss', 'zenoss', - 'events', 3306, 'localhost', '25', '') + builder = DmdBuilder( + self.app.zport, + "localhost", + "zenoss", + "zenoss", + "events", + 3306, + "localhost", + "25", + "", + ) builder.build() self.dmd = builder.dmd - self.dmd.ZenUsers.manage_addUser('tester', roles=('Manager',)) - user = self.app.zport.acl_users.getUserById('tester') + self.dmd.ZenUsers.manage_addUser("tester", roles=("Manager",)) + user = self.app.zport.acl_users.getUserById("tester") from AccessControl.SecurityManagement import newSecurityManager + newSecurityManager(None, user) # Let's hide transaction.commit() so that tests don't fubar # each other self._transaction_commit = Transaction.commit - Transaction.commit=lambda *x: None + Transaction.commit = lambda *x: None setDescriptors(self.dmd) def beforeTearDown(self): - if hasattr( self, '_transaction_commit' ): - Transaction.commit=self._transaction_commit + if hasattr(self, "_transaction_commit"): + Transaction.commit = self._transaction_commit self.app = None self.dmd = None diff --git a/Products/ZenTestCase/__init__.py b/Products/ZenTestCase/__init__.py index de5b4971fc..8f3a86088f 100644 --- a/Products/ZenTestCase/__init__.py +++ b/Products/ZenTestCase/__init__.py @@ -1,11 +1,8 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - - - From f57dda99399109a54dd247eddf32af3982cc20bc Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 15 May 2023 08:38:06 -0500 Subject: [PATCH 009/147] Removed ununnecessary use of 'six' package. --- Products/DataCollector/ApplyDataMap/datamaputils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Products/DataCollector/ApplyDataMap/datamaputils.py b/Products/DataCollector/ApplyDataMap/datamaputils.py index 232c6bc844..d3f8a52ab7 100644 --- a/Products/DataCollector/ApplyDataMap/datamaputils.py +++ b/Products/DataCollector/ApplyDataMap/datamaputils.py @@ -10,7 +10,6 @@ import logging import sys -from six import string_types from zope.event import notify from Products.DataCollector.plugins.DataMaps import MultiArgs @@ -155,7 +154,7 @@ def _get_attr_value(obj, attr): def _sanitize_value(value, obj): - if isinstance(value, string_types): + if isinstance(value, basestring): try: return _decode_value(value, obj) except UnicodeDecodeError: From 774ff0bfb8a03a34b6b51e49d5fbc0dd77b53fc3 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 16 May 2023 11:56:28 -0500 Subject: [PATCH 010/147] KeyedSet objects must use has_key for index key lookups --- Products/ZenCollector/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenCollector/scheduler.py b/Products/ZenCollector/scheduler.py index c1daac26a7..114af272f8 100644 --- a/Products/ZenCollector/scheduler.py +++ b/Products/ZenCollector/scheduler.py @@ -437,7 +437,7 @@ def _startTask( if task_name in self._loopingCalls: loopingCall = self._loopingCalls[task_name] if not loopingCall.running: - if configId in self._tasksToCleanup: + if self._tasksToCleanup.has_key(configId): # noqa W601 delay = random.randint(0, int(interval / 2)) delayed = delayed + delay if attempts > Scheduler.ATTEMPTS: From 6082d25909e36b3a912f0360b49bda64609560b5 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 15 Jun 2023 15:41:27 -0500 Subject: [PATCH 011/147] Reformatted ZenWidgets using the black formatting tool. --- Products/ZenWidgets/PersistentMessage.py | 53 +-- Products/ZenWidgets/Portlet.py | 95 +++-- Products/ZenWidgets/PortletManager.py | 149 +++++--- Products/ZenWidgets/ZenTableManager.py | 326 ++++++++++-------- Products/ZenWidgets/ZenTableState.py | 147 ++++---- .../ZenossPortlets/ZenossPortlets.py | 85 ++--- .../ZenWidgets/ZenossPortlets/__init__.py | 9 +- Products/ZenWidgets/__init__.py | 59 ++-- Products/ZenWidgets/browser/Portlets.py | 203 ++++++----- Products/ZenWidgets/browser/__init__.py | 9 +- Products/ZenWidgets/browser/messaging.py | 38 +- .../ZenWidgets/browser/quickstart/__init__.py | 9 +- .../browser/quickstart/userViews.py | 71 ++-- .../ZenWidgets/browser/quickstart/views.py | 273 +++++++++------ Products/ZenWidgets/interfaces.py | 64 ++-- Products/ZenWidgets/messaging.py | 136 ++++---- Products/ZenWidgets/tests/__init__.py | 9 +- Products/ZenWidgets/tests/test_Portlets.py | 37 +- Products/ZenWidgets/tests/test_messaging.py | 61 ++-- 19 files changed, 1024 insertions(+), 809 deletions(-) diff --git a/Products/ZenWidgets/PersistentMessage.py b/Products/ZenWidgets/PersistentMessage.py index 073f4f5d76..8ace7de7b8 100644 --- a/Products/ZenWidgets/PersistentMessage.py +++ b/Products/ZenWidgets/PersistentMessage.py @@ -1,35 +1,45 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -__doc__ = """ +""" This is in a separate module to prevent recursive import. """ + import cgi import time -from zope.interface import implements + +from zope.interface import implementer from Products.ZenModel.ZenModelRM import ZenModelRM -from Products.ZenRelations.RelSchema import * -from Products.ZenWidgets.interfaces import IMessage -from Products.ZenWidgets.messaging import INFO +from Products.ZenRelations.RelSchema import ToManyCont, ToOne +from .interfaces import IMessage +from .messaging import INFO + + +@implementer(IMessage) class PersistentMessage(ZenModelRM): + """A single message. + + Messages are stored as relations on UserSettings and in the session object. """ - A single message. Messages are stored as relations on UserSettings and in - the session object. - """ - implements(IMessage) - _relations = (("messageQueue", ToOne( - ToManyCont, "Products.ZenModel.UserSettings.UserSettings", "messages") - ),) + _relations = ( + ( + "messageQueue", + ToOne( + ToManyCont, + "Products.ZenModel.UserSettings.UserSettings", + "messages", + ), + ), + ) title = None body = None @@ -38,8 +48,7 @@ class PersistentMessage(ZenModelRM): _read = False def __init__(self, id, title, body, priority=INFO, image=None): - """ - Initialization method. + """Initialize an PersistentMessage instance. @param title: The message title @type title: str @@ -58,13 +67,9 @@ def __init__(self, id, title, body, priority=INFO, image=None): self.timestamp = time.time() def mark_as_read(self): - """ - Mark this message as read. - """ + """Mark this message as read.""" self._read = True def delete(self): - """ - Delete this message from the system. - """ + """Delete this message from the system.""" self.__primary_parent__._delObject(self.id) diff --git a/Products/ZenWidgets/Portlet.py b/Products/ZenWidgets/Portlet.py index 49f1402586..956e0e0fbf 100644 --- a/Products/ZenWidgets/Portlet.py +++ b/Products/ZenWidgets/Portlet.py @@ -1,30 +1,30 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - import logging -log = logging.getLogger('zen.Portlet') +from os.path import basename, exists from string import Template +from AccessControl.class_init import InitializeClass + +from Products.ZenModel.ZenModelRM import ZenModelRM from Products.ZenModel.ZenossSecurity import ZEN_COMMON -from os.path import basename, exists from Products.ZenRelations.RelSchema import ToManyCont, ToOne -from Products.ZenModel.ZenModelRM import ZenModelRM -from AccessControl.class_init import InitializeClass from Products.ZenUtils.Utils import zenPath +log = logging.getLogger("zen.Portlet") + + def manage_addPortlet(self, context, REQUEST=None): - """ - Add a portlet. - """ - pass + """Add a portlet.""" + class Portlet(ZenModelRM): """ @@ -34,29 +34,42 @@ class Portlet(ZenModelRM): Portlets should not be instantiated directly. They should only be created by a PortletManager object. """ - source = '' + + source = "" height = 200 - portal_type = meta_type = 'Portlet' + portal_type = meta_type = "Portlet" _relations = ( - ("portletManager", ToOne( - ToManyCont, "Products.ZenWidgets.PortletManager", "portlets")), + ( + "portletManager", + ToOne( + ToManyCont, "Products.ZenWidgets.PortletManager", "portlets" + ), + ), ) _properties = ( - {'id':'title','type':'string','mode':'w'}, - {'id':'description', 'type':'string', 'mode':'w'}, - {'id':'permission', 'type':'string', 'mode':'w'}, - {'id':'sourcepath', 'type':'string', 'mode':'w'}, - {'id':'preview', 'type':'string', 'mode':'w'}, - {'id':'height', 'type':'int', 'mode':'w'}, + {"id": "title", "type": "string", "mode": "w"}, + {"id": "description", "type": "string", "mode": "w"}, + {"id": "permission", "type": "string", "mode": "w"}, + {"id": "sourcepath", "type": "string", "mode": "w"}, + {"id": "preview", "type": "string", "mode": "w"}, + {"id": "height", "type": "int", "mode": "w"}, ) - - def __init__(self, sourcepath, id='', title='', description='', - preview='', height=200, permission=ZEN_COMMON): - if not id: id = basename(sourcepath).split('.')[0] + def __init__( + self, + sourcepath, + id="", + title="", + description="", + preview="", + height=200, + permission=ZEN_COMMON, + ): + if not id: + id = basename(sourcepath).split(".")[0] self.id = id ZenModelRM.__init__(self, id) self.title = title @@ -75,30 +88,40 @@ def check(self): def _read_source(self): try: - path = self.sourcepath if exists(self.sourcepath) else self._getSourcePath() + path = ( + self.sourcepath + if exists(self.sourcepath) + else self._getSourcePath() + ) f = file(path) except IOError as ex: log.error("Unable to load portlet from '%s': %s", path, ex) return else: - tvars = {'portletId': self.id, - 'portletTitle': self.title, - 'portletHeight': self.height} + tvars = { + "portletId": self.id, + "portletTitle": self.title, + "portletHeight": self.height, + } self.source = Template(f.read()).safe_substitute(tvars) f.close() - def getPrimaryPath(self,fromNode=None): + def getPrimaryPath(self, fromNode=None): """ Override the default, which doesn't account for things on zport """ - return ('', 'zport') + super(Portlet, self).getPrimaryPath(fromNode) + return ("", "zport") + super(Portlet, self).getPrimaryPath(fromNode) def get_source(self, debug_mode=False): - if debug_mode: self._read_source() + if debug_mode: + self._read_source() src = [] src.append(self.source) - src.append("YAHOO.zenoss.portlet.register_portlet('%s', '%s');" % ( - self.id, self.title)) - return '\n'.join(src) + src.append( + "YAHOO.zenoss.portlet.register_portlet('%s', '%s');" + % (self.id, self.title) + ) + return "\n".join(src) + InitializeClass(Portlet) diff --git a/Products/ZenWidgets/PortletManager.py b/Products/ZenWidgets/PortletManager.py index 042d504d92..eb1e491081 100644 --- a/Products/ZenWidgets/PortletManager.py +++ b/Products/ZenWidgets/PortletManager.py @@ -1,37 +1,41 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -import os import md5 +import os from Globals import DevelopmentMode from AccessControl import getSecurityManager from AccessControl.class_init import InitializeClass -from Products.ZenRelations.RelSchema import ToManyCont, ToOne -from Products.ZenModel.ZenModelRM import ZenModelRM + from Products.ZenMessaging.audit import audit +from Products.ZenModel.ZenModelRM import ZenModelRM from Products.ZenModel.ZenossSecurity import ZEN_COMMON -from Products.ZenWidgets import messaging +from Products.ZenRelations.RelSchema import ToManyCont, ToOne from Products.Zuul.utils import ZuulMessageFactory as _t -from Portlet import Portlet +from . import messaging +from .Portlet import Portlet + + +def getuid(): + return md5.md5(os.urandom(10)).hexdigest()[:8] -getuid = lambda:md5.md5(os.urandom(10)).hexdigest()[:8] -class DuplicatePortletRegistration(Exception): pass +class DuplicatePortletRegistration(Exception): + pass + def manage_addPortletManager(context, id="", REQUEST=None): - """ - Create a portlet manager under context. - """ - if not id: id = "ZenPortletManager" + """Create a portlet manager under context.""" + if not id: + id = "ZenPortletManager" zpm = PortletManager(id) context._setObject(id, zpm) zpm = context._getOb(id) @@ -39,41 +43,74 @@ def manage_addPortletManager(context, id="", REQUEST=None): class PortletManager(ZenModelRM): - """ - A registry for portlet source and metadata. Provides access functions and - handles portlet permissions. + """A registry for portlet source and metadata. + + Provides access functions and handles portlet permissions. """ - portal_type = meta_type = 'PortletManager' + portal_type = meta_type = "PortletManager" _relations = ( - ("portlets", ToManyCont(ToOne, "Products.ZenWidgets.Portlet", - "portletManager")), + ( + "portlets", + ToManyCont(ToOne, "Products.ZenWidgets.Portlet", "portletManager"), + ), ) - - def register_extjsPortlet(self, id, title, height=200, permission=ZEN_COMMON): + + def register_extjsPortlet( + self, id, title, height=200, permission=ZEN_COMMON + ): """ Registers an ExtJS portlet """ - ppath = os.path.join('Products','ZenWidgets','ZenossPortlets','ExtPortlet.js') - self.register_portlet(ppath, id=id, title=_t(title), height=height, - permission=permission) - - def register_portlet(self, sourcepath, id='', title='', description='', - preview='', height=200, permission=ZEN_COMMON): + ppath = os.path.join( + "Products", "ZenWidgets", "ZenossPortlets", "ExtPortlet.js" + ) + self.register_portlet( + ppath, id=id, title=_t(title), height=height, permission=permission + ) + + def register_portlet( + self, + sourcepath, + id="", + title="", + description="", + preview="", + height=200, + permission=ZEN_COMMON, + ): """ Registers a new source file and creates an associated Portlet to store the metadata and provide access methods. """ p = self.find(id, sourcepath) if p: - old_values = (p.sourcepath, p.id, p.title, p.description, p.preview, p.height, p.permission) - new_values = (sourcepath, id, _t(title), description, preview, height, permission) + old_values = ( + p.sourcepath, + p.id, + p.title, + p.description, + p.preview, + p.height, + p.permission, + ) + new_values = ( + sourcepath, + id, + _t(title), + description, + preview, + height, + permission, + ) if old_values == new_values: # Portlet unchanged - don't re-register return self.unregister_portlet(p.id) - p = Portlet(sourcepath, id, _t(title), description, preview, height, permission) + p = Portlet( + sourcepath, id, _t(title), description, preview, height, permission + ) self.portlets._setObject(id, p) def unregister_portlet(self, id): @@ -87,22 +124,23 @@ def get_portlets(self): user = getSecurityManager().getUser() dmd = self.dmd.primaryAq() return filter( - lambda x:user.has_permission(x.permission, dmd) and x.check(), - self.portlets()) + lambda x: user.has_permission(x.permission, dmd) and x.check(), + self.portlets(), + ) - def find(self, id='', sourcepath=''): - """ - Look for a registered portlet with an id or source path. - """ + def find(self, id="", sourcepath=""): + """Look for a registered portlet with an id or source path.""" for portlet in self.portlets(): # special case for ExtJs portlets which will all have the same sourcepath - if portlet.id==id or (portlet.sourcepath==sourcepath and not 'ExtPortlet' in sourcepath): return portlet + if portlet.id == id or ( + portlet.sourcepath == sourcepath + and not "ExtPortlet" in sourcepath + ): + return portlet return None def update_source(self, REQUEST=None): - """ - Reread the source files for all portlets. - """ + """Reread the source files for all portlets.""" for portlet in self.portlets(): portlet._read_source() @@ -112,27 +150,28 @@ def get_source(self, REQUEST=None): javascript file. """ srcs = [x.get_source(DevelopmentMode) for x in self.get_portlets()] - srcs.append('YAHOO.register("portletsource", YAHOO.zenoss.portlet, {})') + srcs.append( + 'YAHOO.register("portletsource", YAHOO.zenoss.portlet, {})' + ) if REQUEST: - REQUEST.response.headers['Content-Type'] = 'text/javascript' - return '\n'.join(srcs) + REQUEST.response.headers["Content-Type"] = "text/javascript" + return "\n".join(srcs) def edit_portlet_perms(self, REQUEST=None): - """ - Update the portlet permissions - """ + """Update the portlet permissions.""" for portlet in REQUEST.form: - if not portlet.endswith('_permission'): continue - portname = portlet.split('_')[0] + if not portlet.endswith("_permission"): + continue + portname = portlet.split("_")[0] p = self.find(id=portname) p.permission = REQUEST.form[portlet] - if REQUEST: + if REQUEST: messaging.IMessageSender(self).sendToBrowser( - 'Permissions Saved', - "Saved At: %s" % self.getCurrentUserNowString() + "Permissions Saved", + "Saved At: %s" % self.getCurrentUserNowString(), ) - REQUEST['RESPONSE'].redirect('/zport/dmd/editPortletPerms') - audit('UI.Portlet.Edit', data_=REQUEST.form) + REQUEST["RESPONSE"].redirect("/zport/dmd/editPortletPerms") + audit("UI.Portlet.Edit", data_=REQUEST.form) InitializeClass(PortletManager) diff --git a/Products/ZenWidgets/ZenTableManager.py b/Products/ZenWidgets/ZenTableManager.py index c0352603d7..93ac63b121 100644 --- a/Products/ZenWidgets/ZenTableManager.py +++ b/Products/ZenWidgets/ZenTableManager.py @@ -7,74 +7,75 @@ # ############################################################################## - """ZenTableManager -ZenTableManager is a Zope Product that helps manage and display -large sets of tabular data. It allows for column sorting, -break down of the set into pages, and filtering of elements -in the table. It also allows users to store their own default -page size (but publishes a hook to get this values from -a different location). +ZenTableManager is a Zope Product that helps manage and display large sets of +tabular data. It allows for column sorting, break down of the set into pages, +and filtering of elements in the table. It also allows users to store their +own default page size (but publishes a hook to get this values from a +different location). """ import logging import math import re -import ZTUtils import urllib -from AccessControl.class_init import InitializeClass + +import ZTUtils + from Acquisition import aq_base -from OFS.SimpleItem import SimpleItem -from OFS.PropertyManager import PropertyManager +from AccessControl.class_init import InitializeClass from DocumentTemplate.sequence.SortEx import sort +from OFS.PropertyManager import PropertyManager +from OFS.SimpleItem import SimpleItem from persistent.dict import PersistentDict -from ZenTableState import ZenTableState from Products.ZenUtils.Utils import getTranslation -log = logging.getLogger('zen.ZenTableManager') +from .ZenTableState import ZenTableState + +log = logging.getLogger("zen.ZenTableManager") + -class TableStateNotFound(Exception): pass +class TableStateNotFound(Exception): + pass -def convert(x): - return 0.0 if isinstance(x, float) and math.isnan(x) else x +def convert(x): + return 0.0 if isinstance(x, float) and math.isnan(x) else x -def zencmp(o1, o2): - return cmp(convert(o1), convert(o2)) +def zencmp(o1, o2): + return cmp(convert(o1), convert(o2)) -def manage_addZenTableManager(context, id="", REQUEST = None): - """make a CVDeviceLoader""" - if not id: id = "ZenTableManager" + +def manage_addZenTableManager(context, id="", REQUEST=None): + """Make a CVDeviceLoader.""" + if not id: + id = "ZenTableManager" ztm = ZenTableManager(id) context._setObject(id, ztm) ztm = context._getOb(id) ztm.initTableManagerSkins() if REQUEST is not None: - REQUEST.RESPONSE.redirect(context.absolute_url_path() - +'/manage_main') + REQUEST.RESPONSE.redirect(context.absolute_url_path() + "/manage_main") + class ZenTableManager(SimpleItem, PropertyManager): - """ZenTableManager manages display of tabular data""" + """ZenTableManager manages display of tabular data.""" - portal_type = meta_type = 'ZenTableManager' + portal_type = meta_type = "ZenTableManager" _properties = ( - {'id':'defaultBatchSize', 'type':'int','mode':'w'}, - {'id':'abbrStartLabel', 'type':'int','mode':'w'}, - {'id':'abbrEndLabel', 'type':'int','mode':'w'}, - {'id':'abbrPadding', 'type':'int','mode':'w'}, - {'id':'abbrSeparator', 'type':'string','mode':'w'}, + {"id": "defaultBatchSize", "type": "int", "mode": "w"}, + {"id": "abbrStartLabel", "type": "int", "mode": "w"}, + {"id": "abbrEndLabel", "type": "int", "mode": "w"}, + {"id": "abbrPadding", "type": "int", "mode": "w"}, + {"id": "abbrSeparator", "type": "string", "mode": "w"}, ) - manage_options = ( - PropertyManager.manage_options + - SimpleItem.manage_options - ) - + manage_options = PropertyManager.manage_options + SimpleItem.manage_options def __init__(self, id): self.id = id @@ -83,9 +84,9 @@ def __init__(self, id): self.abbrEndLabel = 5 self.abbrPadding = 5 self.abbrSeparator = ".." - self.abbrThresh = self.abbrStartLabel + \ - self.abbrEndLabel + self.abbrPadding - + self.abbrThresh = ( + self.abbrStartLabel + self.abbrEndLabel + self.abbrPadding + ) def getDefaultBatchSize(self): dbs = self.defaultBatchSize @@ -94,80 +95,80 @@ def getDefaultBatchSize(self): dbs = zu.getUserSettings().defaultPageSize return dbs - def setupTableState(self, tableName, **keys): - """initialize or setup the session variable to track table state""" + """Initialize or setup the session variable to track table state.""" tableState = self.getTableState(tableName, **keys) request = self.REQUEST tableState.updateFromRequest(request) return tableState - def getTableState(self, tableName, attrname=None, default=None, **keys): - """return an existing table state or a single value from the state""" + """Return an existing table state or a single value from the state.""" from Products.ZenUtils.Utils import unused + unused(default) request = self.REQUEST tableStates = self.getTableStates() tableState = tableStates.get(tableName, None) if not tableState: dbs = self.getDefaultBatchSize() - tableStates[tableName] = ZenTableState(request,tableName,dbs,**keys) + tableStates[tableName] = ZenTableState( + request, tableName, dbs, **keys + ) tableState = tableStates[tableName] - if attrname == None: + if attrname is None: return tableStates[tableName] return getattr(tableState, attrname, None) - def getReqTableState(self, tableName, attrname): """ Return attrname from request if present if not return from tableState. """ request = self.REQUEST - if request.has_key(attrname): + if request.has_key(attrname): # noqa W601 return request[attrname] return self.getTableState(tableName, attrname) - def setTableState(self, tableName, attrname, value): """Set the value of a table state attribute and return it.""" tableState = self.getTableState(tableName) return tableState.setTableState(attrname, value) - def setReqTableState(self, tableName, attrname, default=None, reset=False): - """set the a value in the table state from the request""" + """Set the a value in the table state from the request.""" tableState = self.getTableState(tableName) value = self.REQUEST.get(attrname, None) tableState = self.getTableState(tableName) - return tableState.setTableState(attrname, value, - default=default, reset=reset) - + return tableState.setTableState( + attrname, value, default=default, reset=reset + ) def deleteTableState(self, tableName): - """delete an existing table state""" + """Delete an existing table state.""" tableStates = self.getTableStates() if tableName in tableStates: del tableStates[tableName] - def getBatch(self, tableName, objects, **keys): - """Filter, sort and batch objects and pass return set. - """ + """Filter, sort and batch objects and pass return set.""" if log.isEnabledFor(logging.DEBUG): import os - fmt = 'getBatch pid=%s, tableName=%s, %s objects' + + fmt = "getBatch pid=%s, tableName=%s, %s objects" pid = os.getpid() log.debug(fmt, pid, tableName, len(objects)) if not objects: objects = [] tableState = self.setupTableState(tableName, **keys) if tableState.onlyMonitored and objects: - objects = [o for o in objects if getattr(o, 'isMonitored', o.monitored)()] + objects = [ + o for o in objects if getattr(o, "isMonitored", o.monitored)() + ] if tableState.filter and objects: - objects = self.filterObjects(objects, tableState.filter, - tableState.filterFields) + objects = self.filterObjects( + objects, tableState.filter, tableState.filterFields + ) # objects is frequently a generator. Need a list in order to sort if not isinstance(objects, list): objects = list(objects) @@ -175,53 +176,57 @@ def getBatch(self, tableName, objects, **keys): objects = self.sortObjects(objects, tableState) tableState.totalobjs = len(objects) tableState.buildPageNavigation(objects) - if not hasattr(self.REQUEST, 'doExport'): - objects = ZTUtils.Batch(objects, - tableState.batchSize or len(objects), - start=tableState.start, orphan=0) + if not hasattr(self.REQUEST, "doExport"): + objects = ZTUtils.Batch( + objects, + tableState.batchSize or len(objects), + start=tableState.start, + orphan=0, + ) return objects - def getBatchForm(self, objects, request): - """Create batch based on objects no sorting for filter applied. - """ - batchSize = request.get('batchSize',self.defaultBatchSize) - if batchSize in ['', '0']: + """Create batch based on objects no sorting for filter applied.""" + batchSize = request.get("batchSize", self.defaultBatchSize) + if batchSize in ["", "0"]: batchSize = 0 else: batchSize = int(batchSize) - start = int(request.get('start',0)) - resetStart = int(request.get('resetStart',0)) - lastindex = request.get('lastindex',0) - navbutton = request.get('navbutton',None) + start = int(request.get("start", 0)) + resetStart = int(request.get("resetStart", 0)) + lastindex = request.get("lastindex", 0) + navbutton = request.get("navbutton", None) if navbutton == "first" or resetStart: start = 0 elif navbutton == "last": - start=lastindex + start = lastindex elif navbutton == "next": start = start + batchSize - if start > lastindex: start = lastindex + if start > lastindex: + start = lastindex elif navbutton == "prev": start = start - batchSize - elif request.has_key("nextstart"): + elif request.has_key("nextstart"): # noqa W601 start = request.nextstart - if 0 < start > len(objects): start = 0 + if 0 < start > len(objects): + start = 0 request.start = start - objects = ZTUtils.Batch(objects, batchSize or len(objects), - start=request.start, orphan=0) + objects = ZTUtils.Batch( + objects, batchSize or len(objects), start=request.start, orphan=0 + ) return objects - def filterObjects(self, objects, regex, filterFields): - """filter objects base on a regex in regex and list of fields + """Filter objects base on a regex in regex and list of fields in filterFields.""" - if self.REQUEST.SESSION.has_key('message'): - self.REQUEST.SESSION.delete('message') + if self.REQUEST.SESSION.has_key("message"): # noqa W601 + self.REQUEST.SESSION.delete("message") if not regex: return objects - try: search = re.compile(regex,re.I).search + try: + search = re.compile(regex, re.I).search except re.error: - self.REQUEST.SESSION['message'] = "Invalid regular expression." + self.REQUEST.SESSION["message"] = "Invalid regular expression." return objects filteredObjects = [] for obj in objects: @@ -237,61 +242,78 @@ def filterObjects(self, objects, regex, filterFields): value = str(value) target.append(value) targetstring = " ".join(target) - if search(targetstring): filteredObjects.append(obj) + if search(targetstring): + filteredObjects.append(obj) return filteredObjects - def evaluateTales(self, expression, dev): log.warning("evaluating %s", dev.__dict__) - variables_and_funcs = { - 'device':dev, 'dev':dev - } - expression = expression.replace('python:', 'attr=') + variables_and_funcs = {"device": dev, "dev": dev} + expression = expression.replace("python:", "attr=") try: - exec(expression, variables_and_funcs) - attr = variables_and_funcs['attr'] + exec (expression, variables_and_funcs) + attr = variables_and_funcs["attr"] log.warning("attr is %s", attr) except Exception as ex: attr = str(ex) return attr def sortObjects(self, objects, request): - """Sort objects. - """ + """Sort objects.""" + def dictAwareSort(objects, field, rule, sence): if not objects: return objects + class Wrapper: def __init__(self, field, cargo): - if callable(field): field = field() - #make sorting case-insensitive - if isinstance(field, basestring): field = field.lower() + if callable(field): + field = field() + # make sorting case-insensitive + if isinstance(field, basestring): + field = field.lower() self.field = field self.cargo = cargo + if field.startswith("python:"): - objects = [Wrapper(self.evaluateTales(field, o), o) for o in objects] + objects = [ + Wrapper(self.evaluateTales(field, o), o) for o in objects + ] else: if isinstance(objects[0], dict): - objects = [Wrapper(o.get(field, ''), o) for o in objects] + objects = [Wrapper(o.get(field, ""), o) for o in objects] else: - objects = [Wrapper(getattr(o, field, ''), o) for o in objects] - objects = sort(objects, (('field', rule, sence),), {'zencmp': zencmp}) + objects = [ + Wrapper(getattr(o, field, ""), o) for o in objects + ] + objects = sort( + objects, (("field", rule, sence),), {"zencmp": zencmp} + ) return [w.cargo for w in objects] - if (getattr(aq_base(request), 'sortedHeader', False) - and getattr(aq_base(request),"sortedSence", False)): + if getattr(aq_base(request), "sortedHeader", False) and getattr( + aq_base(request), "sortedSence", False + ): sortedHeader = request.sortedHeader sortedSence = request.sortedSence sortRule = getattr(aq_base(request), "sortRule", "cmp") - objects = dictAwareSort(objects, sortedHeader, sortRule, sortedSence) + objects = dictAwareSort( + objects, sortedHeader, sortRule, sortedSence + ) return objects - - def getTableHeader(self, tableName, fieldName, fieldTitle, - sortRule='cmp', style='tableheader',attributes="", - i18n_domain='zenoss'): - """generate a tag that allows column sorting""" + def getTableHeader( + self, + tableName, + fieldName, + fieldTitle, + sortRule="cmp", + style="tableheader", + attributes="", + i18n_domain="zenoss", + ): + """Generate a tag that allows column sorting.""" href = self.getTableHeaderHref(tableName, fieldName, sortRule) style = self.getTableHeaderStyle(tableName, fieldName, style) tag = """""" % (style, attributes) @@ -299,58 +321,60 @@ def getTableHeader(self, tableName, fieldName, fieldTitle, # Owwwwwwwwwww from Products.Zuul.utils import ZuulMessageFactory as _t + msg = getTranslation(_t(fieldTitle), self.REQUEST, domain=i18n_domain) tag += msg + "\n" return tag - - def getTableHeaderHref(self, tableName, fieldName, - sortRule='cmp',params=""): - """build the href attribute for the table headers""" - + def getTableHeaderHref( + self, tableName, fieldName, sortRule="cmp", params="" + ): + """Build the href attribute for the table headers.""" tableState = self.getTableState(tableName) sortedHeader = tableState.sortedHeader sortedSence = tableState.sortedSence if sortedHeader == fieldName: - if sortedSence == 'asc': - sortedSence = 'desc' - elif sortedSence == 'desc': - sortedSence = 'asc' + if sortedSence == "asc": + sortedSence = "desc" + elif sortedSence == "desc": + sortedSence = "asc" else: - sortedSence = 'asc' + sortedSence = "asc" href = "%s?tableName=%s&sortedHeader=%s&" % ( - self.REQUEST.URL, tableName, urllib.quote_plus(fieldName)) - href += "sortedSence=%s&sortRule=%s%s\">" % ( - sortedSence, sortRule, params) + self.REQUEST.URL, + tableName, + urllib.quote_plus(fieldName), + ) + href += 'sortedSence=%s&sortRule=%s%s">' % ( + sortedSence, + sortRule, + params, + ) tableState.addFilterField(fieldName) return href - def getTableHeaderStyle(self, tableName, fieldName, style): """apends "selected" onto the CSS style if this field is selected""" if self.getTableState(tableName, "sortedHeader") == fieldName: style = style + "selected" return style - def getTableStates(self): session = self.REQUEST.SESSION try: - return session['zentablestates'] + return session["zentablestates"] except KeyError: init = PersistentDict() - session['zentablestates'] = init + session["zentablestates"] = init return init - def tableStatesHasTable(self, tableName): - return self.getTableStates().has_key(tableName) - + return tableName in self.getTableStates() def getNavData(self, objects, batchSize, sortedHeader): pagenav = [] - if batchSize in ['', '0']: + if batchSize in ["", "0"]: batchSize = 0 else: batchSize = int(batchSize) @@ -358,55 +382,59 @@ def getNavData(self, objects, batchSize, sortedHeader): if sortedHeader: label = self._buildTextLabel(objects[index], sortedHeader) elif batchSize: - label = str(1+index/batchSize) + label = str(1 + index / batchSize) else: - label = '1' - pagenav.append({ 'label': label, 'index': index }) + label = "1" + pagenav.append({"label": label, "index": index}) return pagenav - def _buildTextLabel(self, item, sortedHeader): endAbbr = "" attr = getattr(item, sortedHeader, "") - if callable(attr): attr = attr() + if callable(attr): + attr = attr() label = str(attr) if len(label) > self.abbrThresh: - startAbbr = label[:self.abbrStartLabel] + startAbbr = label[: self.abbrStartLabel] if self.abbrEndLabel > 0: - endAbbr = label[-self.abbrEndLabel:] + endAbbr = label[-self.abbrEndLabel :] label = "".join((startAbbr, self.abbrSeparator, endAbbr)) return label - def initTableManagerSkins(self): """setup the skins that come with ZenTableManager""" - layers = ('zentablemanager','zenui') + layers = ("zentablemanager", "zenui") try: import string from Products.CMFCore.utils import getToolByName from Products.CMFCore.DirectoryView import addDirectoryViews - skinstool = getToolByName(self, 'portal_skins') + + skinstool = getToolByName(self, "portal_skins") for layer in layers: if layer not in skinstool.objectIds(): - addDirectoryViews(skinstool, 'skins', globals()) + addDirectoryViews(skinstool, "skins", globals()) skins = skinstool.getSkinSelections() for skin in skins: path = skinstool.getSkinPath(skin) - path = map(string.strip, string.split(path,',')) + path = map(string.strip, string.split(path, ",")) for layer in layers: if layer not in path: try: - path.insert(path.index('custom')+1, layer) + path.insert(path.index("custom") + 1, layer) except ValueError: path.append(layer) - path = ','.join(path) + path = ",".join(path) skinstool.addSkinSelection(skin, path) except ImportError as e: - if "Products.CMFCore.utils" in e.args: pass - else: raise + if "Products.CMFCore.utils" in e.args: + pass + else: + raise except AttributeError as e: - if "portal_skin" in e.args: pass - else: raise + if "portal_skin" in e.args: + pass + else: + raise InitializeClass(ZenTableManager) diff --git a/Products/ZenWidgets/ZenTableState.py b/Products/ZenWidgets/ZenTableState.py index 42e2a7bd0c..fbf8745c10 100644 --- a/Products/ZenWidgets/ZenTableState.py +++ b/Products/ZenWidgets/ZenTableState.py @@ -7,32 +7,29 @@ # ############################################################################## - -__doc__="""ZenTableState +"""ZenTableState Track the state of a given table. -$Id: ZenTableState.py,v 1.3 2004/01/17 04:56:13 edahl Exp $""" - -__revision__ = "$Revision: 1.3 $"[11:-2] +""" -from AccessControl.class_init import InitializeClass from AccessControl import ClassSecurityInfo +from AccessControl.class_init import InitializeClass from DateTime.DateTime import DateTime from persistent.dict import PersistentDict + class ZenTableState: - defaultValue = "" # So that we don't have to clear the session + defaultValue = "" # So that we don't have to clear the session changesThatResetStart = [ "batchSize", "filter", "sortedHeader", "sortedSence", - "defaultValue" - "onlyMonitored" - ] + "defaultValue" "onlyMonitored", + ] requestAtts = [ "batchSize", @@ -45,18 +42,18 @@ class ZenTableState: "URL", "defaultValue", "onlyMonitored", - "generate" - ] + "generate", + ] security = ClassSecurityInfo() - #this session info isn't anything worth protecting - security.setDefaultAccess('allow') + # this session info isn't anything worth protecting + security.setDefaultAccess("allow") def __init__(self, request, tableName, defaultBatchSize, **keys): self.URL = request.URL self.tableName = tableName self.sortedHeader = "primarySortKey" - self.sortedSence="asc" + self.sortedSence = "asc" self.sortRule = "cmp" self.onlyMonitored = 0 self.defaultBatchSize = defaultBatchSize @@ -71,8 +68,9 @@ def __init__(self, request, tableName, defaultBatchSize, **keys): self.abbrEndLabel = 5 self.abbrPadding = 5 self.abbrSeparator = ".." - self.abbrThresh = self.abbrStartLabel + \ - self.abbrEndLabel + self.abbrPadding + self.abbrThresh = ( + self.abbrStartLabel + self.abbrEndLabel + self.abbrPadding + ) self.tableClass = "tableheader" self.resetStart = False self.showAll = False @@ -102,81 +100,97 @@ def setTableStateFromKeys(self, keys): if key not in self.requestAtts: self.requestAtts.append(key) - def updateFromRequest(self, request): """update table state based on request""" - states = request.SESSION['zentablestates'] + states = request.SESSION["zentablestates"] if not isinstance(states, PersistentDict): - request.SESSION['zentablestates'] = PersistentDict(states) - request.SESSION['zentablestates']._p_changed = True + request.SESSION["zentablestates"] = PersistentDict(states) + request.SESSION["zentablestates"]._p_changed = True if self.URL != request.URL: self.batchSize = self.defaultBatchSize - self.start=0 - self.filter = '' - - # 'tableName' will be empty on GET requests, therefore we check for the 'showAll' option here - if request.get("showAll", False) or "showAll=true" in request.get("QUERY_STRING") or request.get("adapt", False or "adapt=false" in request.get("QUERY_STRING")): - if not request.get('tableName', None): + self.start = 0 + self.filter = "" + + # 'tableName' will be empty on GET requests, therefore we check for + # the 'showAll' option here. + if ( + request.get("showAll", False) + or "showAll=true" in request.get("QUERY_STRING") + or request.get( + "adapt", False or "adapt=false" in request.get("QUERY_STRING") + ) + ): + if not request.get("tableName", None): self.showAll = True self.start = 0 self.batchSize = 0 - # the batch size needs to be set to the total object/result count. - # we don't have the objects here, so we will set the batchSize - # where we do have the objects -- see buildPageNavigation() below. + # The batch size needs to be set to the total object/result + # count. We don't have the objects here, so we will set the + # batchSize where we do have the objects -- see + # buildPageNavigation() below. - if request.get('tableName', None) != self.tableName: + if request.get("tableName", None) != self.tableName: return for attname in self.requestAtts: - if request.has_key(attname): - self.setTableState(attname, int(request[attname]) if attname == 'start' else request[attname], request=request) - if request.get("showAll", False) or "showAll=true" in request.get("QUERY_STRING"): + if request.has_key(attname): # noqa W601 + self.setTableState( + attname, + int(request[attname]) + if attname == "start" + else request[attname], + request=request, + ) + if request.get("showAll", False) or "showAll=true" in request.get( + "QUERY_STRING" + ): self.showAll = True self.start = 0 self.batchSize = 0 - if not request.has_key('onlyMonitored'): - self.setTableState('onlyMonitored', 0) - if request.get("first",False): + if not request.has_key("onlyMonitored"): # noqa W601 + self.setTableState("onlyMonitored", 0) + if request.get("first", False): self.resetStart = True elif request.get("last", False): - self.start=self.lastindex + self.start = self.lastindex elif request.get("next", False): np = self.start + self.batchSize - if np > self.lastindex: self.start = self.lastindex - else: self.start = np + if np > self.lastindex: + self.start = self.lastindex + else: + self.start = np elif request.get("prev", False): pp = self.start - self.batchSize - if pp < 0: self.start = 0 - else: self.start = pp - ourl = "/".join((request.URL,request.get("zenScreenName",""))) + if pp < 0: + self.start = 0 + else: + self.start = pp + ourl = "/".join((request.URL, request.get("zenScreenName", ""))) if self.resetStart or (self.URL != request.URL and self.URL != ourl): self.start = 0 self.resetStart = False - def getPageNavigation(self): return self.pagenav - def buildPageNavigation(self, objects): self.pagenav = [] # this conditional is for setting the batchSize on a "showAll" - #if self.showAll: + # if self.showAll: # self.batchSize = len(objects) # self.start = 0 # self.showAll = False if self.batchSize == 0: return self.pagenav - lastindex=0 + lastindex = 0 for index in range(0, self.totalobjs, self.batchSize): pg = {} - pg['label'] = self._pageLabel(objects, index) - pg['index'] = index + pg["label"] = self._pageLabel(objects, index) + pg["index"] = index self.pagenav.append(pg) - lastindex=index + lastindex = index self.lastindex = lastindex - def _pageLabel(self, objects, index): """make label for page navigation if field isn't sorted use page #""" pageLabel = "" @@ -185,17 +199,17 @@ def _pageLabel(self, objects, index): if self.sortedHeader: pageLabel = self._buildTextLabel(objects[index]) elif self.batchSize: - pageLabel = str(1+index/self.batchSize) + pageLabel = str(1 + index / self.batchSize) else: - pageLabel = '1' + pageLabel = "1" return pageLabel - def _buildTextLabel(self, item): startAbbr = "" endAbbr = "" attr = getattr(item, self.sortedHeader, self.defaultValue) - if callable(attr): attr = attr() + if callable(attr): + attr = attr() if isinstance(attr, DateTime) and not attr.millis(): label = self.defaultValue else: @@ -205,16 +219,17 @@ def _buildTextLabel(self, item): # ensuring that we are always working with a string label = str(label) if len(label) > self.abbrThresh: - startAbbr = label[:self.abbrStartLabel] + startAbbr = label[: self.abbrStartLabel] if self.abbrEndLabel > 0: - endAbbr = label[-self.abbrEndLabel:] + endAbbr = label[-self.abbrEndLabel :] label = "".join((startAbbr, self.abbrSeparator, endAbbr)) return label - - def setTableState(self, attname, value, default=None, reset=False, request=None): - if attname == 'batchSize': - if value in ['', '0']: + def setTableState( + self, attname, value, default=None, reset=False, request=None + ): + if attname == "batchSize": + if value in ["", "0"]: value = 0 else: # If given parameter is not numeric this will catch it @@ -226,20 +241,20 @@ def setTableState(self, attname, value, default=None, reset=False, request=None) # Restore whatever was the previous value value = getattr(self, attname, None) if request is not None: - # Set attribute in request to previous value so it gets stored properly + # Set attribute in request to previous value so it + # gets stored properly. request[attname] = value - if not hasattr(self, attname) and default != None: + if not hasattr(self, attname) and default is not None: setattr(self, attname, default) if reset and attname not in self.changesThatResetStart: self.changesThatResetStart.append(attname) if attname not in self.requestAtts: self.requestAtts.append(attname) - if value != None and getattr(self,attname, None) != value: + if value is not None and getattr(self, attname, None) != value: setattr(self, attname, value) if attname in self.changesThatResetStart: self.resetStart = True - return getattr(self,attname) - + return getattr(self, attname) def addFilterField(self, fieldName): """make sure we only add non-dup filterfields""" diff --git a/Products/ZenWidgets/ZenossPortlets/ZenossPortlets.py b/Products/ZenWidgets/ZenossPortlets/ZenossPortlets.py index 240e512537..610b008fc0 100644 --- a/Products/ZenWidgets/ZenossPortlets/ZenossPortlets.py +++ b/Products/ZenWidgets/ZenossPortlets/ZenossPortlets.py @@ -1,75 +1,82 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - import os -from Products.ZenModel.ZenossSecurity import * + +from Products.ZenModel.ZenossSecurity import ( + ZEN_COMMON, + ZEN_MANAGE_DMD, + ZEN_VIEW, +) + def _portletpath(*args): """ Shortcut, since these all live in the same directory. Portlet needs a path relative to $ZENHOME. """ - return os.path.join('Products','ZenWidgets','ZenossPortlets', *args) + return os.path.join("Products", "ZenWidgets", "ZenossPortlets", *args) + portlets = [ { - 'sourcepath': _portletpath('HeartbeatsPortlet.js'), - 'id': 'HeartbeatsPortlet', - 'title': 'Daemon Processes Down', - 'permission': ZEN_MANAGE_DMD + "sourcepath": _portletpath("HeartbeatsPortlet.js"), + "id": "HeartbeatsPortlet", + "title": "Daemon Processes Down", + "permission": ZEN_MANAGE_DMD, }, { - 'sourcepath': _portletpath('GoogleMapsPortlet.js'), - 'id': 'GoogleMapsPortlet', - 'title': 'Google Maps', - 'permission': ZEN_VIEW + "sourcepath": _portletpath("GoogleMapsPortlet.js"), + "id": "GoogleMapsPortlet", + "title": "Google Maps", + "permission": ZEN_VIEW, }, { - 'sourcepath': _portletpath('SiteWindowPortlet.js'), - 'id': 'SiteWindowPortlet', - 'title': 'Site Window', - 'permission': ZEN_VIEW + "sourcepath": _portletpath("SiteWindowPortlet.js"), + "id": "SiteWindowPortlet", + "title": "Site Window", + "permission": ZEN_VIEW, }, { - 'sourcepath': _portletpath('DeviceIssuesPortlet.js'), - 'id': 'DeviceIssuesPortlet', - 'title': 'Device Issues', - 'permission': ZEN_COMMON + "sourcepath": _portletpath("DeviceIssuesPortlet.js"), + "id": "DeviceIssuesPortlet", + "title": "Device Issues", + "permission": ZEN_COMMON, }, { - 'sourcepath': _portletpath('TopLevelOrgsPortlet.js'), - 'id': 'TopLevelOrgsPortlet', - 'title': 'Top Level Organizers', - 'permission': ZEN_VIEW + "sourcepath": _portletpath("TopLevelOrgsPortlet.js"), + "id": "TopLevelOrgsPortlet", + "title": "Top Level Organizers", + "permission": ZEN_VIEW, }, { - 'sourcepath': _portletpath('WatchListPortlet.js'), - 'id': 'WatchListPortlet', - 'title': 'Watch List', - 'permission': ZEN_COMMON + "sourcepath": _portletpath("WatchListPortlet.js"), + "id": "WatchListPortlet", + "title": "Watch List", + "permission": ZEN_COMMON, }, { - 'sourcepath': _portletpath('productionStatePortlet.js'), - 'id': 'ProdStatePortlet', - 'title': 'Production States', - 'permission': ZEN_COMMON + "sourcepath": _portletpath("productionStatePortlet.js"), + "id": "ProdStatePortlet", + "title": "Production States", + "permission": ZEN_COMMON, }, { - 'sourcepath': _portletpath('userMessagesPortlet.js'), - 'id': 'UserMsgsPortlet', - 'title': 'Messages', - 'permission': ZEN_COMMON + "sourcepath": _portletpath("userMessagesPortlet.js"), + "id": "UserMsgsPortlet", + "title": "Messages", + "permission": ZEN_COMMON, }, ] + def register_default_portlets(portletmanager): for portlet in portlets: - if portletmanager.find(portlet['id']) is None: + if portletmanager.find(portlet["id"]) is None: portletmanager.register_portlet(**portlet) diff --git a/Products/ZenWidgets/ZenossPortlets/__init__.py b/Products/ZenWidgets/ZenossPortlets/__init__.py index de5b4971fc..8f3a86088f 100644 --- a/Products/ZenWidgets/ZenossPortlets/__init__.py +++ b/Products/ZenWidgets/ZenossPortlets/__init__.py @@ -1,11 +1,8 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - - - diff --git a/Products/ZenWidgets/__init__.py b/Products/ZenWidgets/__init__.py index 9d42478b9f..d917e8cffb 100644 --- a/Products/ZenWidgets/__init__.py +++ b/Products/ZenWidgets/__init__.py @@ -1,60 +1,55 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## +try: + from Products.CMFCore.DirectoryView import registerDirectory -"""__init__ - -Initializer for ZenTableManager - -$Id: __init__.py,v 1.3 2004/04/04 23:56:49 edahl Exp $""" - -__version__ = 0.5 -__revision__ = "$Revision: 1.3 $"[11:-2] - + registerDirectory("skins", globals()) +except ImportError: + pass from Products.Five.browser import BrowserView -from ZenTableManager import ZenTableManager -from ZenTableManager import manage_addZenTableManager -try: - from Products.CMFCore.DirectoryView import registerDirectory - registerDirectory('skins', globals()) -except ImportError: pass +from .ZenossPortlets.ZenossPortlets import register_default_portlets +from .ZenTableManager import manage_addZenTableManager, ZenTableManager -from ZenossPortlets.ZenossPortlets import register_default_portlets def update_portlets(app): + """Reread in portlet source on startup. + + If this is the initial load, and objects don't exist yet, don't do + anything. """ - Reread in portlet source on startup. If this is the initial load, and - objects don't exist yet, don't do anything. - """ - if hasattr(app, 'zport') and hasattr(app.zport, 'ZenPortletManager'): + if hasattr(app, "zport") and hasattr(app.zport, "ZenPortletManager"): register_default_portlets(app.zport.ZenPortletManager) for pack in app.zport.dmd.ZenPackManager.packs(): - for portlet in getattr(pack, 'register_portlets', lambda *x:())(): - if app.zport.ZenPortletManager.find(portlet['id']) is None: - app.zport.ZenPortletManager.register_extjsPortlet(**portlet) + for portlet in getattr(pack, "register_portlets", lambda *x: ())(): + if app.zport.ZenPortletManager.find(portlet["id"]) is None: + app.zport.ZenPortletManager.register_extjsPortlet( + **portlet + ) + def initialize(registrar): registrar.registerClass( ZenTableManager, permission="Add ZenTableManager", - constructors = (manage_addZenTableManager,), - icon = "ZenTableManager_icon.gif" + constructors=(manage_addZenTableManager,), + icon="ZenTableManager_icon.gif", ) + def registerPortlets(event): - """ - Handler for IZopeApplicationOpenedEvent which registers portlets. - """ + """Handler for IZopeApplicationOpenedEvent which registers portlets.""" update_portlets(event.app) + class ExtJSShortcut(BrowserView): def __getitem__(self, name): - return self.context.unrestrictedTraverse('++resource++extjs')[name] + return self.context.unrestrictedTraverse("++resource++extjs")[name] diff --git a/Products/ZenWidgets/browser/Portlets.py b/Products/ZenWidgets/browser/Portlets.py index 8428961b26..f8910a1ee6 100644 --- a/Products/ZenWidgets/browser/Portlets.py +++ b/Products/ZenWidgets/browser/Portlets.py @@ -1,38 +1,36 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -import re -import json +import logging from Products.Five.browser import BrowserView from Products.AdvancedQuery import Eq, In, And - -from Products.ZenUtils.Utils import relative_time -from Products.Zuul import getFacade -from Products.ZenEvents.HeartbeatUtils import getHeartbeatObjects from zenoss.protocols.services import ServiceException from zenoss.protocols.services.zep import ZepConnectionError + +from Products.ZenEvents.browser.EventPillsAndSummaries import ( + getDashboardObjectsEventSummary, + getEventPillME, + ObjectsEventSummary, +) +from Products.ZenEvents.HeartbeatUtils import getHeartbeatObjects +from Products.ZenModel.Device import Device +from Products.ZenModel.ZenossSecurity import ZEN_VIEW from Products.ZenUtils.guid.interfaces import IGUIDManager from Products.ZenUtils.jsonutils import json -from Products.ZenUtils.Utils import nocache, formreq, extractPostContent -from Products.ZenWidgets import messaging -from Products.ZenModel.Device import Device -from Products.ZenModel.ZenossSecurity import * -from Products.ZenEvents.browser.EventPillsAndSummaries import \ - getDashboardObjectsEventSummary, \ - ObjectsEventSummary, \ - getEventPillME +from Products.ZenUtils.Utils import nocache, formreq, relative_time +from Products.Zuul import getFacade from Products.Zuul.catalog.interfaces import IModelCatalogTool -import logging -log = logging.getLogger('zen.portlets') +from .. import messaging + +log = logging.getLogger("zen.portlets") def zepConnectionError(retval=None): @@ -40,21 +38,28 @@ def outer(func): def inner(self, *args, **kwargs): try: return func(self, *args, **kwargs) - except ZepConnectionError as e: - msg = 'Connection refused. Check zeneventserver status on Services' - messaging.IMessageSender(self.context).sendToBrowser("ZEP connection error", - msg, - priority=messaging.CRITICAL, - sticky=True) + except ZepConnectionError: + msg = ( + "Connection refused. Check zeneventserver status on " + 'Services' + ) + messaging.IMessageSender(self.context).sendToBrowser( + "ZEP connection error", + msg, + priority=messaging.CRITICAL, + sticky=True, + ) log.warn("Could not connect to ZEP") return retval + return inner + return outer + class TopLevelOrganizerPortletView(ObjectsEventSummary): - """ - Return JSON event summaries for a root organizer. - """ + """Return JSON event summaries for a root organizer.""" + @nocache @formreq def __call__(self, dataRoot): @@ -70,13 +75,14 @@ class ProductionStatePortletView(BrowserView): Return a map of device to production state in a format suitable for a YUI data table. """ + @nocache @formreq def __call__(self, *args, **kwargs): return self.getDevProdStateJSON(*args, **kwargs) @json - def getDevProdStateJSON(self, prodStates=['Maintenance']): + def getDevProdStateJSON(self, prodStates=["Maintenance"]): """ Return a map of device to production state in a format suitable for a YUI data table. @@ -101,20 +107,27 @@ def getProdStateInt(prodStateString): numericProdStates = [getProdStateInt(p) for p in prodStates] catalog = IModelCatalogTool(self.context.getPhysicalRoot().zport.dmd) - query = In('productionState', numericProdStates) - - query = And(query, Eq('objectImplements', 'Products.ZenModel.Device.Device')) - objects = list(catalog.search(query=query, orderby='id', fields="uuid")) + query = In("productionState", numericProdStates) + + query = And( + query, Eq("objectImplements", "Products.ZenModel.Device.Device") + ) + objects = list( + catalog.search(query=query, orderby="id", fields="uuid") + ) devs = (x.getObject() for x in objects) - mydict = {'columns':['Device', 'Prod State'], 'data':[]} + mydict = {"columns": ["Device", "Prod State"], "data": []} for dev in devs: - if not self.context.checkRemotePerm(ZEN_VIEW, dev): continue - mydict['data'].append({ - 'Device' : dev.getPrettyLink(), - 'Prod State' : dev.getProdState() - }) - if len(mydict['data'])>=100: + if not self.context.checkRemotePerm(ZEN_VIEW, dev): + continue + mydict["data"].append( + { + "Device": dev.getPrettyLink(), + "Prod State": dev.getProdState(), + } + ) + if len(mydict["data"]) >= 100: break return mydict @@ -132,6 +145,7 @@ class WatchListPortletView(BrowserView): of the table @rtype: string """ + @nocache @formreq def __call__(self, *args, **kwargs): @@ -143,24 +157,27 @@ def getEntityListEventSummary(self, entities=None): entities = [] elif isinstance(entities, basestring): entities = [entities] + def getob(e): e = str(e) try: - if not e.startswith('/zport/dmd'): - bigdev = '/zport/dmd' + e + if not e.startswith("/zport/dmd"): + bigdev = "/zport/dmd" + e obj = self.context.dmd.unrestrictedTraverse(bigdev) except (AttributeError, KeyError): obj = self.context.dmd.Devices.findDevice(e) - if self.context.has_permission("View", obj): return obj - entities = filter(lambda x:x is not None, map(getob, entities)) + if self.context.has_permission("View", obj): + return obj + + entities = filter(lambda x: x is not None, map(getob, entities)) return getDashboardObjectsEventSummary( - self.context.dmd.ZenEventManager, entities) + self.context.dmd.ZenEventManager, entities + ) class DeviceIssuesPortletView(BrowserView): - """ - A list of devices with issues. - """ + """A list of devices with issues.""" + @nocache def __call__(self): return self.getDeviceIssuesJSON() @@ -179,18 +196,17 @@ def getDeviceIssuesJSON(self): {'Device':'', 'Events':'
'}, ]}" """ - mydict = {'columns':[], 'data':[]} - mydict['columns'] = ['Device', 'Events'] + mydict = {"columns": [], "data": []} + mydict["columns"] = ["Device", "Events"] deviceinfo = self.getDeviceDashboard() for alink, pill in deviceinfo: - mydict['data'].append({'Device':alink, - 'Events':pill}) + mydict["data"].append({"Device": alink, "Events": pill}) return mydict @zepConnectionError([]) def getDeviceDashboard(self): """return device info for bad device to dashboard""" - zep = getFacade('zep') + zep = getFacade("zep") manager = IGUIDManager(self.context.dmd) deviceSeverities = zep.getDeviceIssuesDict() zem = self.context.dmd.ZenEventManager @@ -199,49 +215,62 @@ def getDeviceDashboard(self): for uuid in deviceSeverities.keys(): uuid_data = {} - uuid_data['uuid'] = uuid + uuid_data["uuid"] = uuid severities = deviceSeverities[uuid] try: - uuid_data['severities'] = dict((zep.getSeverityName(sev).lower(), counts) for (sev, counts) in severities.iteritems()) + uuid_data["severities"] = dict( + (zep.getSeverityName(sev).lower(), counts) + for (sev, counts) in severities.iteritems() + ) except ServiceException: continue bulk_data.append(uuid_data) - bulk_data.sort(key=lambda x:(x['severities']['critical'], x['severities']['error'], x['severities']['warning']), reverse=True) + bulk_data.sort( + key=lambda x: ( + x["severities"]["critical"], + x["severities"]["error"], + x["severities"]["warning"], + ), + reverse=True, + ) devices_found = 0 MAX_DEVICES = 100 devdata = [] for data in bulk_data: - uuid = data['uuid'] - severities = data['severities'] + uuid = data["uuid"] + severities = data["severities"] dev = manager.getObject(uuid) if dev and isinstance(dev, Device): - if (not zem.checkRemotePerm(ZEN_VIEW, dev) + if ( + not zem.checkRemotePerm(ZEN_VIEW, dev) or dev.getProductionState() < zem.prodStateDashboardThresh - or dev.priority < zem.priorityDashboardThresh): + or dev.priority < zem.priorityDashboardThresh + ): continue alink = dev.getPrettyLink() pill = getEventPillME(dev, severities=severities) - evts = [alink,pill] + evts = [alink, pill] devdata.append(evts) devices_found = devices_found + 1 if devices_found >= MAX_DEVICES: break return devdata -heartbeat_columns = ['Host', 'Daemon Process', 'Seconds Down'] + +heartbeat_columns = ["Host", "Daemon Process", "Seconds Down"] + class HeartbeatPortletView(BrowserView): - """ - Heartbeat issues in YUI table form, for the dashboard portlet - """ + """Heartbeat issues in YUI table form, for the dashboard portlet.""" + @nocache def __call__(self): return self.getHeartbeatIssuesJSON() - @zepConnectionError({'columns': heartbeat_columns, 'data':[]}) + @zepConnectionError({"columns": heartbeat_columns, "data": []}) @json def getHeartbeatIssuesJSON(self): """ @@ -254,15 +283,15 @@ def getHeartbeatIssuesJSON(self): {'Device':'', 'Daemon':'zenhub', 'Seconds':10} ]}" """ - data = getHeartbeatObjects(deviceRoot=self.context.dmd.Devices, - keys=heartbeat_columns) - return {'columns': heartbeat_columns, 'data': data} + data = getHeartbeatObjects( + deviceRoot=self.context.dmd.Devices, keys=heartbeat_columns + ) + return {"columns": heartbeat_columns, "data": data} class UserMessagesPortletView(BrowserView): - """ - User messages in YUI table form, for the dashboard portlet. - """ + """User messages in YUI table form, for the dashboard portlet.""" + @nocache @json def __call__(self): @@ -276,20 +305,24 @@ def __call__(self): {'Device':'', 'Daemon':'zenhub', 'Seconds':10} ]}" """ - ICONS = ['/zport/dmd/img/agt_action_success-32.png', - '/zport/dmd/img/messagebox_warning-32.png', - '/zport/dmd/img/agt_stop-32.png'] + ICONS = [ + "/zport/dmd/img/agt_action_success-32.png", + "/zport/dmd/img/messagebox_warning-32.png", + "/zport/dmd/img/agt_stop-32.png", + ] msgbox = messaging.IUserMessages(self.context) msgs = msgbox.get_messages() - cols = ['Message'] + cols = ["Message"] res = [] for msg in msgs: - res.append(dict( - title = msg.title, - imgpath = ICONS[msg.priority], - body = msg.body, - ago = relative_time(msg.timestamp), - deletelink = msg.absolute_url_path() + '/delMsg' - )) + res.append( + dict( + title=msg.title, + imgpath=ICONS[msg.priority], + body=msg.body, + ago=relative_time(msg.timestamp), + deletelink=msg.absolute_url_path() + "/delMsg", + ) + ) res.reverse() - return { 'columns': cols, 'data': res } + return {"columns": cols, "data": res} diff --git a/Products/ZenWidgets/browser/__init__.py b/Products/ZenWidgets/browser/__init__.py index de5b4971fc..8f3a86088f 100644 --- a/Products/ZenWidgets/browser/__init__.py +++ b/Products/ZenWidgets/browser/__init__.py @@ -1,11 +1,8 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - - - diff --git a/Products/ZenWidgets/browser/messaging.py b/Products/ZenWidgets/browser/messaging.py index bf5a324690..c12ccf328f 100644 --- a/Products/ZenWidgets/browser/messaging.py +++ b/Products/ZenWidgets/browser/messaging.py @@ -1,42 +1,46 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - from Products.Five.browser import BrowserView from Products.ZenUtils.jsonutils import json -from Products.ZenModel.ZenossSecurity import * -from Products.ZenWidgets.interfaces import IUserMessages, IBrowserMessages -from Products.ZenWidgets import messaging + +from .. import messaging +from ..interfaces import IBrowserMessages, IUserMessages + class UserMessages(BrowserView): """ Delivers up user messages for the current user to the client-side YAHOO.zenoss.Messenger. """ + @json def __call__(self): messages = IUserMessages(self.context).get_unread() messages.extend(IBrowserMessages(self.context).get_unread()) - messages.sort(key=lambda x:x.timestamp) + messages.sort(key=lambda x: x.timestamp) result = [] for message in messages: - result.append(dict( - sticky=message.priority>=messaging.CRITICAL and True or False, - image=message.image, - title=message.title, - body=message.body, - priority=message.priority - )) + result.append( + dict( + sticky=message.priority >= messaging.CRITICAL + and True + or False, + image=message.image, + title=message.title, + body=message.body, + priority=message.priority, + ) + ) message.mark_as_read() - result = {'totalRecords':len(result), - 'messages':result} + result = {"totalRecords": len(result), "messages": result} return result diff --git a/Products/ZenWidgets/browser/quickstart/__init__.py b/Products/ZenWidgets/browser/quickstart/__init__.py index de5b4971fc..8f3a86088f 100644 --- a/Products/ZenWidgets/browser/quickstart/__init__.py +++ b/Products/ZenWidgets/browser/quickstart/__init__.py @@ -1,11 +1,8 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - - - diff --git a/Products/ZenWidgets/browser/quickstart/userViews.py b/Products/ZenWidgets/browser/quickstart/userViews.py index 4a5cbd3a50..e5dea600ee 100644 --- a/Products/ZenWidgets/browser/quickstart/userViews.py +++ b/Products/ZenWidgets/browser/quickstart/userViews.py @@ -1,32 +1,33 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2009, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - import logging -log = logging.getLogger("zen.widgets.userviews") +from Products.CMFCore.utils import getToolByName from Products.Five.browser import BrowserView from Products.Five.browser.pagetemplatefile import ZopeTwoPageTemplateFile + +from Products.ZenModel.Quickstart import getTopQuickstartStep from Products.ZenUtils import Ext from Products.ZenUtils.csrf import check_csrf_token -from Products.CMFCore.utils import getToolByName -from Products.ZenModel.Quickstart import getTopQuickstartStep + +log = logging.getLogger("zen.widgets.userviews") class SetAdminPasswordException(Exception): - """There was a problem setting the admin password""" + """There was a problem setting the admin password.""" + class CreateUserView(BrowserView): - """ - Creates the initial user and sets the admin password. - """ - __call__ = ZopeTwoPageTemplateFile('templates/createuser.pt') + """Creates the initial user and sets the admin password.""" + + __call__ = ZopeTwoPageTemplateFile("templates/createuser.pt") @Ext.form_action def createUser(self): @@ -43,35 +44,47 @@ def createUser(self): userPassword = self.request.form.get("password1") emailAddress = self.request.form.get("emailAddress") - zenUsers = getToolByName(self.context, 'ZenUsers') + zenUsers = getToolByName(self.context, "ZenUsers") # Set admin password try: - admin = zenUsers.getUserSettings('admin') - admin.manage_editUserSettings(password=adminPassword, - sndpassword=adminPassword, - roles=('ZenManager', 'Manager'), - oldpassword='zenoss') + admin = zenUsers.getUserSettings("admin") + admin.manage_editUserSettings( + password=adminPassword, + sndpassword=adminPassword, + roles=("ZenManager", "Manager"), + oldpassword="zenoss", + ) except Exception: log.exception("Failed to set admin password") - response.error('admin-password1', - "There was a problem setting the admin password.") + response.error( + "admin-password1", + "There was a problem setting the admin password.", + ) - if not zenUsers.checkValidId(userName) == True: - response.error('username', 'That username already exists.') + if not zenUsers.checkValidId(userName) is True: + response.error("username", "That username already exists.") else: - ret = zenUsers.manage_addUser(userName, userPassword, - ('Manager',), REQUEST=None, email=emailAddress) + ret = zenUsers.manage_addUser( + userName, + userPassword, + ("Manager",), + REQUEST=None, + email=emailAddress, + ) if ret is None: - response.error('username', - 'We were unable to add a user at this time.' - ' Check your installation.') + response.error( + "username", + "We were unable to add a user at this time." + " Check your installation.", + ) if not response.has_errors(): # Log out, so the form can log us in as the new user - acl_users = self.context.getPhysicalRoot().acl_users + _ = self.context.getPhysicalRoot().acl_users self.context.acl_users.resetCredentials( - self.request, self.request.response) + self.request, self.request.response + ) # Don't run the quickstart next time self.context.dmd._rq = True diff --git a/Products/ZenWidgets/browser/quickstart/views.py b/Products/ZenWidgets/browser/quickstart/views.py index 6faa14c13c..c6ec4a7a4f 100644 --- a/Products/ZenWidgets/browser/quickstart/views.py +++ b/Products/ZenWidgets/browser/quickstart/views.py @@ -7,55 +7,61 @@ # ############################################################################## - -import re -import logging import cgi +import logging +import re + from Acquisition import aq_base from Products.Five.browser import BrowserView from Products.Five.browser.pagetemplatefile import ZopeTwoPageTemplateFile + +from Products.ZenMessaging.audit import audit from Products.ZenModel.IpNetwork import AutoDiscoveryJob -from Products.ZenWidgets.messaging import IMessageSender from Products.ZenUtils import Ext from Products.ZenUtils.jsonutils import json -from Products.ZenMessaging.audit import audit -_is_network = lambda x: bool(re.compile(r'^(\d+\.){3}\d+\/\d+$').search(x)) -_is_range = lambda x: bool(re.compile(r'^(\d+\.){3}\d+\-\d+$').search(x)) +from ...messaging import IMessageSender + log = logging.getLogger("zen.quickstart") + +def _is_network(x): + return bool(re.compile(r"^(\d+\.){3}\d+\/\d+$").search(x)) + + +def _is_range(x): + return bool(re.compile(r"^(\d+\.){3}\d+\-\d+$").search(x)) + + class QuickstartBase(BrowserView): - """ - Standard macros for the quickstart. - """ - template = ZopeTwoPageTemplateFile('templates/quickstart_macros.pt') + """Standard macros for the quickstart.""" + + template = ZopeTwoPageTemplateFile("templates/quickstart_macros.pt") def __getitem__(self, key): return self.template.macros[key] class OutlineView(BrowserView): - """ - Displays the steps the user will soon be completing. The anticipation! - """ - __call__ = ZopeTwoPageTemplateFile('templates/outline.pt') + """Displays the steps the user will soon be completing.""" + + __call__ = ZopeTwoPageTemplateFile("templates/outline.pt") class CreateUserView(BrowserView): - """ - Creates the initial user and sets the admin password. - """ - __call__ = ZopeTwoPageTemplateFile('templates/createuser.pt') + """Creates the initial user and sets the admin password.""" + + __call__ = ZopeTwoPageTemplateFile("templates/createuser.pt") class DeviceAddView(BrowserView): - """ - Specify devices to be added. - """ + """Specify devices to be added.""" + @property def hasLDAPInstalled(self): try: - import ZenPacks.zenoss.LDAPAuthenticator + import ZenPacks.zenoss.LDAPAuthenticator # noqa F401 + # return javascript true/false return "true" except ImportError: @@ -67,19 +73,19 @@ def default_communities(self): Format the value of Devices.Discovered.zSnmpCommunities for a textarea """ devclass = self.context.dmd.Devices.Discovered.primaryAq() - return '\n'.join(devclass.zSnmpCommunities) + return "\n".join(devclass.zSnmpCommunities) def _assemble_types_list(self): """ Walks all device classes building a list of description/protocol pairs. """ - ALLOWED_PROTOCOLS = ('SSH', 'SNMP', 'WMI', 'WinRM') + ALLOWED_PROTOCOLS = ("SSH", "SNMP", "WMI", "WinRM") devclass = self.context.dmd.Devices orgs = devclass.getSubOrganizers() types = [] for org in orgs: # Skip it if it doesn't have types registered - if not hasattr(aq_base(org), 'devtypes') or not org.devtypes: + if not hasattr(aq_base(org), "devtypes") or not org.devtypes: continue for t in org.devtypes: try: @@ -93,16 +99,23 @@ def _assemble_types_list(self): # special case for migrating from WMI to WinRM so we # can allow the zenpack to be backwards compatible - if org.getOrganizerName() == '/Server/Microsoft/Windows' and ptcl == 'WMI': + if ( + org.getOrganizerName() == "/Server/Microsoft/Windows" + and ptcl == "WMI" + ): ptcl = "WinRM" # We only care about orgs with acceptable protocols - if ptcl not in ALLOWED_PROTOCOLS: continue + if ptcl not in ALLOWED_PROTOCOLS: + continue types.append((org.getOrganizerName(), desc, ptcl)) return types @json def collectors(self): - return [[name] for name in self.context.dmd.Monitors.getPerformanceMonitorNames()] + return [ + [name] + for name in self.context.dmd.Monitors.getPerformanceMonitorNames() + ] @json def device_types(self): @@ -115,12 +128,16 @@ def device_types(self): appropriate ZenPack installed?). """ # Turn them into the dictionary format expected - types = {'win':[], 'ssh':[], 'snmp':[], 'winrm': []} + types = {"win": [], "ssh": [], "snmp": [], "winrm": []} for t in self._assemble_types_list(): - if t[2]=='WMI': types['win'].append(t) - elif t[2]=='SNMP': types['snmp'].append(t) - elif t[2]=='SSH': types['ssh'].append(t) - elif t[2]=='WinRM': types['win'].append(t) + if t[2] == "WMI": + types["win"].append(t) + elif t[2] == "SNMP": + types["snmp"].append(t) + elif t[2] == "SSH": + types["ssh"].append(t) + elif t[2] == "WinRM": + types["win"].append(t) def dev_class_exists(path): """ @@ -128,8 +145,7 @@ def dev_class_exists(path): exists. """ try: - self.context.unrestrictedTraverse( - '/zport/dmd/Devices' + path) + self.context.unrestrictedTraverse("/zport/dmd/Devices" + path) except AttributeError: return False else: @@ -140,10 +156,13 @@ def format_type(credtype, classpath, description, protocol): Turn information representing a device class into a dictionary of the format our ComboBox expects. """ - value = '%s_%s' % (classpath, credtype) - return dict(value=value, - shortdesc="%s (%s)" % (description, protocol), - description=description, protocol=protocol) + value = "%s_%s" % (classpath, credtype) + return dict( + value=value, + shortdesc="%s (%s)" % (description, protocol), + description=description, + protocol=protocol, + ) # Iterate over all types response = [] @@ -155,148 +174,184 @@ def format_type(credtype, classpath, description, protocol): response.append(format_type(credtype, *devtype)) # Sort alphabetically by description - response.sort(key=lambda x:x['description']) + response.sort(key=lambda x: x["description"]) # Final response needs an object under a defined root, in this case # "types" return dict(types=response) - @Ext.form_action def autodiscovery(self): response = Ext.FormResponse() - submitted = self.request.form.get('network', []) + submitted = self.request.form.get("network", []) if isinstance(submitted, basestring): submitted = [submitted] zProperties = { - 'zCommandUsername': self.request.form.get('sshusername'), - 'zCommandPassword': self.request.form.get('sshpass'), - 'zWinRMUser': self.request.form.get('winusername'), - 'zWinRMPassword': self.request.form.get('winpass'), - 'zSnmpCommunities': self.request.form.get('snmpcommunities').splitlines() + "zCommandUsername": self.request.form.get("sshusername"), + "zCommandPassword": self.request.form.get("sshpass"), + "zWinRMUser": self.request.form.get("winusername"), + "zWinRMPassword": self.request.form.get("winpass"), + "zSnmpCommunities": self.request.form.get( + "snmpcommunities" + ).splitlines(), } - collector = self.request.form.get('autodiscovery_collector', 'localhost') + collector = self.request.form.get( + "autodiscovery_collector", "localhost" + ) # Split rows into networks and ranges nets = [] ranges = [] for row in submitted: - if _is_network(row): nets.append(row) - elif _is_range(row): ranges.append(row) + if _is_network(row): + nets.append(row) + elif _is_range(row): + ranges.append(row) if not nets and not ranges: - response.error('network', - 'You must enter at least one network or IP range.') + response.error( + "network", "You must enter at least one network or IP range." + ) if nets: for net in nets: # Make the network if it doesn't exist, so zendisc has # something to discover - _n = self.context.dmd.Networks.createNet(net) + _ = self.context.dmd.Networks.createNet(net) try: - netdesc = ("network %s" % nets[0] if len(nets)==1 - else "%s networks" % len(nets)) + netdesc = ( + "network %s" % nets[0] + if len(nets) == 1 + else "%s networks" % len(nets) + ) self.context.JobManager.addJob( AutoDiscoveryJob, description="Discover %s" % netdesc, kwargs=dict( - nets=nets, - zProperties=zProperties, - collector=collector - ) + nets=nets, zProperties=zProperties, collector=collector + ), ) except Exception as e: log.exception(e) - response.error('network', 'There was an error scheduling this ' - 'job. Please check your installation and try ' - 'again.') + response.error( + "network", + "There was an error scheduling this job. " + "Please check your installation and try again.", + ) else: IMessageSender(self.context).sendToUser( - 'Autodiscovery Task Created', - 'Discovery of the following networks is in progress: %s' % ( - ', '.join(nets)) + "Autodiscovery Task Created", + "Discovery of the following networks is in progress: %s" + % (", ".join(nets)), ) if ranges: # Ranges can just be sent to zendisc, as they are merely sets of # IPs try: - rangedesc = ("IP range %s" % ranges[0] - if len(ranges)==1 - else "%s IP ranges" % len(ranges)) + rangedesc = ( + "IP range %s" % ranges[0] + if len(ranges) == 1 + else "%s IP ranges" % len(ranges) + ) self.context.JobManager.addJob( AutoDiscoveryJob, description="Discover %s" % rangedesc, kwargs=dict( ranges=ranges, zProperties=zProperties, - collector=collector - ) + collector=collector, + ), ) except Exception as e: log.exception(e) - response.error('network', 'There was an error scheduling this ' - 'job. Please check your installation and try ' - 'again.') + response.error( + "network", + "There was an error scheduling this job. " + "Please check your installation and try again.", + ) else: IMessageSender(self.context).sendToUser( - 'Autodiscovery Task Created', - 'Discovery of the following IP ranges is in progress: %s' % ( - ', '.join(ranges)) + "Autodiscovery Task Created", + "Discovery of the following IP ranges is in progress: %s" + % (", ".join(ranges)), ) - audit('UI.Device.Autodiscovery', networks=','.join(nets), ipRanges=','.join(ranges)) - response.redirect('/zport/dmd') + audit( + "UI.Device.Autodiscovery", + networks=",".join(nets), + ipRanges=",".join(ranges), + ) + response.redirect("/zport/dmd") return response - @Ext.form_action def manual(self): # Pull all the device name keys response = Ext.FormResponse() - devs = filter(lambda x:x.startswith('device_'), - self.request.form.keys()) + devs = filter( + lambda x: x.startswith("device_"), self.request.form.keys() + ) # Make sure we have at least one device name - devnames = filter(lambda x:bool(self.request.form.get(x)), devs) + devnames = filter(lambda x: bool(self.request.form.get(x)), devs) if not devnames: - response.error('device_0', - 'You must enter at least one hostname/IP.') + response.error( + "device_0", "You must enter at least one hostname/IP." + ) return response # Create jobs based on info passed for k in devs: # Ignore empty device names - if not self.request.form.get(k): continue - idx = k.split('_')[1] + if not self.request.form.get(k): + continue + idx = k.split("_")[1] devclass, type_ = self.request.form.get( - 'deviceclass_%s' % idx).split('_') - collector = self.request.form.get('collector_' + str(idx), 'localhost') + "deviceclass_%s" % idx + ).split("_") + collector = self.request.form.get( + "collector_" + str(idx), "localhost" + ) # Set zProps based on type - if type_=='ssh': + if type_ == "ssh": zProps = { - 'zCommandUsername': self.request.form.get('sshuser_%s' % idx), - 'zCommandPassword': self.request.form.get( - 'sshpass_%s' % idx), + "zCommandUsername": self.request.form.get( + "sshuser_%s" % idx + ), + "zCommandPassword": self.request.form.get( + "sshpass_%s" % idx + ), } - elif type_=='win': + elif type_ == "win": zProps = { - 'zWinRMUser': self.request.form.get('winuser_%s' % idx), - 'zWinRMPassword': self.request.form.get('winpass_%s' % idx), + "zWinRMUser": self.request.form.get("winuser_%s" % idx), + "zWinRMPassword": self.request.form.get( + "winpass_%s" % idx + ), } - elif type_=='snmp': + elif type_ == "snmp": zProps = { - 'zSnmpCommunities': self.request.form.get( - 'snmpcomm_%s' % idx + "zSnmpCommunities": self.request.form.get( + "snmpcomm_%s" % idx ).splitlines() } deviceName = self.request.form.get(k) perfConf = self.context.Monitors.getPerformanceMonitor(collector) - perfConf.addCreateDeviceJob(deviceName=deviceName, performanceMonitor=collector, - devicePath=devclass, zProperties=zProps, discoverProto='auto') - deviceClassUid = '/Devices' + devclass - deviceUid = '/'.join([deviceClassUid, 'devices', deviceName]) - audit('UI.Device.Add', deviceUid, deviceClass=deviceClassUid, model=True) + perfConf.addCreateDeviceJob( + deviceName=deviceName, + performanceMonitor=collector, + devicePath=devclass, + zProperties=zProps, + discoverProto="auto", + ) + deviceClassUid = "/Devices" + devclass + deviceUid = "/".join([deviceClassUid, "devices", deviceName]) + audit( + "UI.Device.Add", + deviceUid, + deviceClass=deviceClassUid, + model=True, + ) devnames = [self.request.form.get(dev) for dev in devs] IMessageSender(self.context).sendToUser( - 'Devices Added', - 'Modeling of the following devices has been scheduled: %s' % ( - cgi.escape(', '.join(filter(None, devnames))) - ) + "Devices Added", + "Modeling of the following devices has been scheduled: %s" + % (cgi.escape(", ".join(filter(None, devnames)))), ) - response.redirect('/zport/dmd') + response.redirect("/zport/dmd") return response diff --git a/Products/ZenWidgets/interfaces.py b/Products/ZenWidgets/interfaces.py index 7ef424fdc1..dc0935cffc 100644 --- a/Products/ZenWidgets/interfaces.py +++ b/Products/ZenWidgets/interfaces.py @@ -1,22 +1,23 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -from zope.interface import Interface, Attribute from zope.container.interfaces import IContained, IContainer +from zope.interface import Interface, Attribute class IMessage(IContained): + """A single message. + + Messages are stored in user-specific MessageQueue objects and in the + session object. """ - A single message. Messages are stored in user-specific MessageQueue objects - and in the session object. - """ + title = Attribute("Title of the message") body = Attribute("Body of the message") image = Attribute("Optional path to image to be displayed") @@ -25,28 +26,24 @@ class IMessage(IContained): sticky = Attribute("Explicitly designate stickiness") def delete(): - """ - Delete this message from any queues in which it exists. - """ + """Delete this message from any queues in which it exists.""" + def mark_as_read(): - """ - Mark this message as read. - """ + """Mark this message as read.""" class IMessageSender(Interface): - """ - Something able to send messages. - """ + """Something able to send messages.""" + def sendToBrowser(title, body, priority, image=None, sticky=None): - """ - Create a message and store it on the request object. - """ + """Create a message and store it on the request object.""" + def sendToUser(title, body, priority, image=None, user=None): """ Create a message and store it in the L{IMessageQueue} of the user specified. If no user is specified, use the queue of the current user. """ + def sendToAll(title, body, priority, image=None): """ For eash user in the system, create an identical message and store it @@ -55,33 +52,24 @@ def sendToAll(title, body, priority, image=None): class IMessageQueue(IContainer): - """ - Marker interface for a message container. - """ + """Marker interface for a message container.""" class IMessageBox(Interface): - """ - Something that can provide messages. - """ + """Something that can provide messages.""" + messagebox = Attribute("The source of IMessage objects.") + def get_messages(): - """ - Return all messages. - """ + """Return all messages.""" + def get_unread(): - """ - Return all messages that have not been marked as read. - """ + """Return all messages that have not been marked as read.""" class IUserMessages(IMessageBox): - """ - Object that is able to provide IMessage objects from a user queue. - """ + """Object that is able to provide IMessage objects from a user queue.""" class IBrowserMessages(IMessageBox): - """ - Object that is able to provide IMessage objects from the request. - """ + """Object that is able to provide IMessage objects from the request.""" diff --git a/Products/ZenWidgets/messaging.py b/Products/ZenWidgets/messaging.py index 9b7370b1fc..6e123595e3 100644 --- a/Products/ZenWidgets/messaging.py +++ b/Products/ZenWidgets/messaging.py @@ -1,32 +1,40 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## + import cgi import time -from zope.interface import implements from Products.CMFCore.utils import getToolByName -from Products.ZenRelations.utils import ZenRelationshipNameChooser -from Products.ZenWidgets.interfaces import * +from zope.interface import implementer +from Products.ZenRelations.utils import ZenRelationshipNameChooser +from Products.ZenWidgets.interfaces import ( + IBrowserMessages, + IMessage, + IMessageBox, + IMessageSender, + IUserMessages, +) # Constants representing priorities. # Parallel definitions exist in zenoss.js. -INFO = 0 -WARNING = 1 +INFO = 0 +WARNING = 1 CRITICAL = 2 + +@implementer(IMessage) class BrowserMessage(object): + """A single message. + + Messages are stored on UserSettings and in the session object. """ - A single message. Messages are stored on UserSettings and in the session - object. - """ - implements(IMessage) __parent__ = None title = None @@ -36,8 +44,7 @@ class BrowserMessage(object): _read = False def __init__(self, title, body, priority=INFO, image=None, sticky=None): - """ - Initialization method. + """Initialize a BrowserMessage instance. @param title: The message title @type title: str @@ -56,12 +63,13 @@ def __init__(self, title, body, priority=INFO, image=None, sticky=None): self.sticky = sticky def delete(self): - """ - Delete this message from the system. + """Delete this message from the system. """ self._read = True - try: self.__parent__.remove(self) - except (ValueError): pass + try: + self.__parent__.remove(self) + except (ValueError): + pass del self def mark_as_read(self): @@ -69,18 +77,17 @@ def mark_as_read(self): self.delete() +@implementer(IMessageBox) class MessageBox(object): + """Adapter for all persistent objects. + + Provides a method, L{get_messages}, that retrieves L{Message} objects. """ - Adapter for all persistent objects. Provides a method, L{get_messages}, - that retrieves L{Message} objects. - """ - implements(IMessageBox) messagebox = None def get_unread(self, min_priority=INFO): - """ - Retrieve unread messages. + """Retrieve unread messages. @param min_priority: Optional minimum priority of messages to be returned; one of INFO, WARNING, CRITICAL @@ -89,12 +96,11 @@ def get_unread(self, min_priority=INFO): @rtype: list """ msgs = self.get_messages(min_priority) - msgs = filter(lambda x:not x._read, msgs) + msgs = filter(lambda x: not x._read, msgs) return msgs def get_messages(self, min_priority=INFO): - """ - Retrieve messages from the current users's session object. + """Retrieve messages from the current users's session object. @param min_priority: Optional minimum priority of messages to be returned; one of INFO, WARNING, CRITICAL @@ -102,44 +108,49 @@ def get_messages(self, min_priority=INFO): @return: A list of L{Message} objects. @rtype: list """ - msgs = sorted(self.messagebox, key=lambda x:x.timestamp) - msgs = filter(lambda x:x.priority>=min_priority, msgs) + msgs = sorted(self.messagebox, key=lambda x: x.timestamp) + msgs = filter(lambda x: x.priority >= min_priority, msgs) return msgs +@implementer(IBrowserMessages) class BrowserMessageBox(MessageBox): + """Adapter for all persistent objects. + + Provides a method, L{get_messages}, that retrieves L{Message} objects + from the current user's session. """ - Adapter for all persistent objects. Provides a method, L{get_messages}, - that retrieves L{Message} objects from the current user's session. - """ - implements(IBrowserMessages) + def __init__(self, context): - """ - Initialization method. + """Initialize a BrowserMessageBox instance. @param context: The object being adapted. Must have access to the current request object via acquisition. @type context: Persistent """ self.context = context - self.messagebox = self.context.REQUEST.SESSION.get('messages', []) + self.messagebox = self.context.REQUEST.SESSION.get("messages", []) def get_unread(self, min_priority=INFO): - msgs = super(BrowserMessageBox, self).get_unread(min_priority=min_priority) + msgs = super(BrowserMessageBox, self).get_unread( + min_priority=min_priority + ) # force the session to persist if msgs: self.context.REQUEST.SESSION._p_changed = True return msgs + +@implementer(IUserMessages) class UserMessageBox(MessageBox): + """Adapter for all persistent objects. + + Provides a method, L{get_messages}, that retrieves L{Message} objects + from the current user's L{MessageQueue}. """ - Adapter for all persistent objects. Provides a method, L{get_messages}, - that retrieves L{Message} objects from the current user's L{MessageQueue}. - """ - implements(IUserMessages) + def __init__(self, context, user=None): - """ - Initialization method. + """Initialize a UserMessageBox instance. @param context: The object being adapted. Must have access to the dmd via acquisition. @@ -151,20 +162,19 @@ def __init__(self, context, user=None): """ self.context = context self.user = user - users = getToolByName(self.context, 'ZenUsers') + users = getToolByName(self.context, "ZenUsers") us = users.getUserSettings(self.user) self.messagebox = us.messages() +@implementer(IMessageSender) class MessageSender(object): """ Adapts persistent objects in order to provide message sending capability. """ - implements(IMessageSender) def __init__(self, context): - """ - Initialization method. + """Initialize a MessageSender instance. @param context: The object being adapted. Must have access to the dmd and the current request object via acquisition. @@ -172,9 +182,10 @@ def __init__(self, context): """ self.context = context - def sendToBrowser(self, title, body, priority=INFO, image=None, sticky=None): - """ - Create a message and store it on the session object. + def sendToBrowser( + self, title, body, priority=INFO, image=None, sticky=None + ): + """Create a message and store it on the session object. @param title: The message title @type title: str @@ -185,9 +196,9 @@ def sendToBrowser(self, title, body, priority=INFO, image=None, sticky=None): @param image: Optional URL of an image to be displayed in the message @type image: str """ - context = self.context.REQUEST.SESSION.get('messages') + context = self.context.REQUEST.SESSION.get("messages") if context is None: - self.context.REQUEST.SESSION['messages'] = context = [] + self.context.REQUEST.SESSION["messages"] = context = [] m = BrowserMessage(title, body, priority, image, sticky) m.__parent__ = context context.append(m) @@ -211,11 +222,12 @@ def sendToUser(self, title, body, priority=INFO, image=None, user=None): user's queue will be used. @type user: str """ - users = getToolByName(self.context, 'ZenUsers') + users = getToolByName(self.context, "ZenUsers") us = users.getUserSettings(user) - id = ZenRelationshipNameChooser(us.messages).chooseName('msg') + id = ZenRelationshipNameChooser(us.messages).chooseName("msg") # done in here to prevent recursive imports from ZenModelRM from PersistentMessage import PersistentMessage + m = PersistentMessage(id, title, body, priority, image) us.messages._setObject(m.id, m) @@ -233,18 +245,22 @@ def sendToAll(self, title, body, priority=INFO, image=None): @param image: Optional URL of an image to be displayed in the message @type image: str """ - users = getToolByName(self.context, 'ZenUsers') + users = getToolByName(self.context, "ZenUsers") for name in users.getAllUserSettingsNames(): self.sendToUser(title, body, priority, user=name, image=image) class ScriptMessageSender(MessageSender): + """Special message sender for use in scripts. + + Short-circuits sendToBrowser and sendToUser, since they don't really + apply. sendToAll should still work fine though. """ - Special message sender for use in scripts. Short-circuits sendToBrowser and - sendToUser, since they don't really apply. sendToAll should still work fine - though. - """ - def sendToBrowser(self, title, body, priority=INFO, image=None, sticky=None): + + def sendToBrowser( + self, title, body, priority=INFO, image=None, sticky=None + ): pass + def sendToUser(self, title, body, priority=INFO, image=None, user=None): pass diff --git a/Products/ZenWidgets/tests/__init__.py b/Products/ZenWidgets/tests/__init__.py index de5b4971fc..8f3a86088f 100644 --- a/Products/ZenWidgets/tests/__init__.py +++ b/Products/ZenWidgets/tests/__init__.py @@ -1,11 +1,8 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - - - diff --git a/Products/ZenWidgets/tests/test_Portlets.py b/Products/ZenWidgets/tests/test_Portlets.py index 611d19c82f..79095637e8 100644 --- a/Products/ZenWidgets/tests/test_Portlets.py +++ b/Products/ZenWidgets/tests/test_Portlets.py @@ -1,52 +1,59 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2009, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## import json +from Products import Zuul from Products.ZenTestCase.BaseTestCase import BaseTestCase from Products.ZenWidgets.browser.Portlets import ProductionStatePortletView -from Products import Zuul -class TestPortlets(BaseTestCase): +class TestPortlets(BaseTestCase): def afterSetUp(self): super(TestPortlets, self).afterSetUp() - self.facade = Zuul.getFacade('device', self.dmd) + self.facade = Zuul.getFacade("device", self.dmd) def test_ProductionStatePortletView(self): # Create some devices devices = self.dmd.Devices - test_device_maintenance = devices.createInstance('testDeviceMaintenance') - test_device_production = devices.createInstance('testDeviceProduction') + test_device_maintenance = devices.createInstance( + "testDeviceMaintenance" + ) + test_device_production = devices.createInstance("testDeviceProduction") test_device_maintenance.setProdState(300) test_device_production.setProdState(1000) psPortlet = ProductionStatePortletView(self.dmd, self.dmd.REQUEST) - + # filter by maintenance result = json.loads(psPortlet()) - self.assertEqual(len(result['data']), 1) - self.assertEqual(result['data'][0]['Device'], test_device_maintenance.getPrettyLink()) + self.assertEqual(len(result["data"]), 1) + self.assertEqual( + result["data"][0]["Device"], + test_device_maintenance.getPrettyLink(), + ) # filter by production result = json.loads(psPortlet("Production")) - self.assertEqual(len(result['data']), 1) - self.assertEqual(result['data'][0]['Device'], test_device_production.getPrettyLink()) + self.assertEqual(len(result["data"]), 1) + self.assertEqual( + result["data"][0]["Device"], test_device_production.getPrettyLink() + ) # filter by both result = json.loads(psPortlet(["Production", "Maintenance"])) - self.assertEqual(len(result['data']), 2) - + self.assertEqual(len(result["data"]), 2) def test_suite(): from unittest import TestSuite, makeSuite + suite = TestSuite() suite.addTest(makeSuite(TestPortlets)) return suite diff --git a/Products/ZenWidgets/tests/test_messaging.py b/Products/ZenWidgets/tests/test_messaging.py index 99a859d360..54617b1863 100644 --- a/Products/ZenWidgets/tests/test_messaging.py +++ b/Products/ZenWidgets/tests/test_messaging.py @@ -1,25 +1,27 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2009, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - from AccessControl.SecurityManagement import newSecurityManager from Products.ZenTestCase.BaseTestCase import BaseTestCase -from Products.ZenWidgets.messaging import MessageSender -from Products.ZenWidgets.messaging import BrowserMessageBox, UserMessageBox +from Products.ZenWidgets.messaging import ( + BrowserMessageBox, + MessageSender, + UserMessageBox, +) + class DummySession(dict): _p_changed = False class TestMessaging(BaseTestCase): - def afterSetUp(self): super(TestMessaging, self).afterSetUp() self.dmd.REQUEST.SESSION = DummySession() @@ -30,46 +32,45 @@ def _login(self, name): """ uf = self.dmd.zport.acl_users user = uf.getUserById(name) - if not hasattr(user, 'aq_base'): + if not hasattr(user, "aq_base"): user = user.__of__(uf) newSecurityManager(None, user) def test_sending_to_request(self): - MessageSender(self.dmd).sendToBrowser('title', 'This is a message') - us = self.dmd.ZenUsers.getUserSettings('tester') + MessageSender(self.dmd).sendToBrowser("title", "This is a message") + us = self.dmd.ZenUsers.getUserSettings("tester") self.assertEqual(len(us.messages()), 0) - self.assertEqual(len(self.dmd.REQUEST.SESSION['messages']), 1) - self.assertEqual(self.dmd.REQUEST.SESSION['messages'][0].body, - 'This is a message') + self.assertEqual(len(self.dmd.REQUEST.SESSION["messages"]), 1) + self.assertEqual( + self.dmd.REQUEST.SESSION["messages"][0].body, "This is a message" + ) def test_sending_to_user(self): - self._login('tester') - MessageSender(self.dmd).sendToUser('title', 'This is a message') - us = self.dmd.ZenUsers.getUserSettings('tester') + self._login("tester") + MessageSender(self.dmd).sendToUser("title", "This is a message") + us = self.dmd.ZenUsers.getUserSettings("tester") self.assertEqual(len(us.messages), 1) - self.assertEqual(us.messages()[0].body, 'This is a message') + self.assertEqual(us.messages()[0].body, "This is a message") def test_adapters(self): MessageSender(self.dmd).sendToBrowser( - 'title', - 'This is a browser message') - MessageSender(self.dmd).sendToUser( - 'title', - 'This is a user message') + "title", "This is a browser message" + ) + MessageSender(self.dmd).sendToUser("title", "This is a user message") brow = BrowserMessageBox(self.dmd) user = UserMessageBox(self.dmd) browmsgs = brow.get_messages() usermsgs = user.get_messages() self.assertEqual(len(browmsgs), 1) self.assertEqual(len(usermsgs), 1) - self.assertEqual(browmsgs[0].body, 'This is a browser message') - self.assertEqual(usermsgs[0].body, 'This is a user message') + self.assertEqual(browmsgs[0].body, "This is a browser message") + self.assertEqual(usermsgs[0].body, "This is a user message") def test_mark_as_read(self): - MessageSender(self.dmd).sendToBrowser('title', - 'This is a browser message') - MessageSender(self.dmd).sendToUser('title', - 'This is a user message') + MessageSender(self.dmd).sendToBrowser( + "title", "This is a browser message" + ) + MessageSender(self.dmd).sendToUser("title", "This is a user message") brow = BrowserMessageBox(self.dmd) user = UserMessageBox(self.dmd) @@ -85,11 +86,9 @@ def test_mark_as_read(self): self.assertEqual(len(user.get_unread()), 0) - - - def test_suite(): from unittest import TestSuite, makeSuite + suite = TestSuite() suite.addTest(makeSuite(TestMessaging)) return suite From 2aa902789e4eeb11d6ba47167da3a7370dd6972f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 15 Jun 2023 16:02:30 -0500 Subject: [PATCH 012/147] Clean up some overlooked formatting in ZenWidgets. --- Products/ZenWidgets/Portlet.py | 4 +--- Products/ZenWidgets/PortletManager.py | 9 ++++----- Products/ZenWidgets/ZenTableState.py | 11 +++-------- Products/ZenWidgets/__init__.py | 10 +++------- Products/ZenWidgets/messaging.py | 8 ++++---- 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/Products/ZenWidgets/Portlet.py b/Products/ZenWidgets/Portlet.py index 956e0e0fbf..6fd6af4ec7 100644 --- a/Products/ZenWidgets/Portlet.py +++ b/Products/ZenWidgets/Portlet.py @@ -107,9 +107,7 @@ def _read_source(self): f.close() def getPrimaryPath(self, fromNode=None): - """ - Override the default, which doesn't account for things on zport - """ + """Override the default, which doesn't account for things on zport.""" return ("", "zport") + super(Portlet, self).getPrimaryPath(fromNode) def get_source(self, debug_mode=False): diff --git a/Products/ZenWidgets/PortletManager.py b/Products/ZenWidgets/PortletManager.py index eb1e491081..bdd8b61b7e 100644 --- a/Products/ZenWidgets/PortletManager.py +++ b/Products/ZenWidgets/PortletManager.py @@ -60,9 +60,7 @@ class PortletManager(ZenModelRM): def register_extjsPortlet( self, id, title, height=200, permission=ZEN_COMMON ): - """ - Registers an ExtJS portlet - """ + """Registers an ExtJS portlet.""" ppath = os.path.join( "Products", "ZenWidgets", "ZenossPortlets", "ExtPortlet.js" ) @@ -131,10 +129,11 @@ def get_portlets(self): def find(self, id="", sourcepath=""): """Look for a registered portlet with an id or source path.""" for portlet in self.portlets(): - # special case for ExtJs portlets which will all have the same sourcepath + # Special case for ExtJs portlets which will all have the same + # sourcepath. if portlet.id == id or ( portlet.sourcepath == sourcepath - and not "ExtPortlet" in sourcepath + and "ExtPortlet" not in sourcepath ): return portlet return None diff --git a/Products/ZenWidgets/ZenTableState.py b/Products/ZenWidgets/ZenTableState.py index fbf8745c10..9c773eb2df 100644 --- a/Products/ZenWidgets/ZenTableState.py +++ b/Products/ZenWidgets/ZenTableState.py @@ -10,7 +10,6 @@ """ZenTableState Track the state of a given table. - """ from AccessControl import ClassSecurityInfo @@ -28,7 +27,8 @@ class ZenTableState: "filter", "sortedHeader", "sortedSence", - "defaultValue" "onlyMonitored", + "defaultValue", + "onlyMonitored", ] requestAtts = [ @@ -175,11 +175,6 @@ def getPageNavigation(self): def buildPageNavigation(self, objects): self.pagenav = [] - # this conditional is for setting the batchSize on a "showAll" - # if self.showAll: - # self.batchSize = len(objects) - # self.start = 0 - # self.showAll = False if self.batchSize == 0: return self.pagenav lastindex = 0 @@ -192,7 +187,7 @@ def buildPageNavigation(self, objects): self.lastindex = lastindex def _pageLabel(self, objects, index): - """make label for page navigation if field isn't sorted use page #""" + """Make label for page navigation if field isn't sorted use page #.""" pageLabel = "" # do not show the page label if there is only one page if self.totalobjs > self.batchSize: diff --git a/Products/ZenWidgets/__init__.py b/Products/ZenWidgets/__init__.py index d917e8cffb..9331715c86 100644 --- a/Products/ZenWidgets/__init__.py +++ b/Products/ZenWidgets/__init__.py @@ -7,18 +7,14 @@ # ############################################################################## -try: - from Products.CMFCore.DirectoryView import registerDirectory - - registerDirectory("skins", globals()) -except ImportError: - pass - +from Products.CMFCore.DirectoryView import registerDirectory from Products.Five.browser import BrowserView from .ZenossPortlets.ZenossPortlets import register_default_portlets from .ZenTableManager import manage_addZenTableManager, ZenTableManager +registerDirectory("skins", globals()) + def update_portlets(app): """Reread in portlet source on startup. diff --git a/Products/ZenWidgets/messaging.py b/Products/ZenWidgets/messaging.py index 6e123595e3..5b4dde8f4a 100644 --- a/Products/ZenWidgets/messaging.py +++ b/Products/ZenWidgets/messaging.py @@ -125,7 +125,7 @@ def __init__(self, context): """Initialize a BrowserMessageBox instance. @param context: The object being adapted. Must have access to the - current request object via acquisition. + current request object via acquisition. @type context: Persistent """ self.context = context @@ -153,11 +153,11 @@ def __init__(self, context, user=None): """Initialize a UserMessageBox instance. @param context: The object being adapted. Must have access to the dmd - via acquisition. + via acquisition. @type context: Persistent @param user: Optional username corresponding to the queue from which - messages will be retrieved. If left as C{None}, the - current user's queue will be used. + messages will be retrieved. If left as C{None}, the current + user's queue will be used. @type user: str """ self.context = context From 19713922d07385b6a9dc77b503796a1c017230a2 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 16 Jun 2023 15:25:00 -0500 Subject: [PATCH 013/147] Reformatted ZenUtils/Utils.py using the black utility. Refactored the sane_joinpath function and added a lot of doctests. --- Products/ZenUtils/Utils.py | 1175 +++++++++++++++++++++--------------- 1 file changed, 679 insertions(+), 496 deletions(-) diff --git a/Products/ZenUtils/Utils.py b/Products/ZenUtils/Utils.py index 38d8e39ee8..6a2bca490a 100644 --- a/Products/ZenUtils/Utils.py +++ b/Products/ZenUtils/Utils.py @@ -9,58 +9,64 @@ """Utils -General utility functions module +General utility functions module. """ from __future__ import absolute_import -import sys -import select -import popen2 -import fcntl -import time -import os -import types +import asyncore +import contextlib +import copy import ctypes -import tempfile +import fcntl +import httplib import logging +import math +import os +import popen2 import re +import select +import shlex import socket -import math -import contextlib import string +import sys +import tempfile +import time +import types import xmlrpclib -import httplib -import shlex + from decimal import Decimal -import asyncore -import copy from functools import partial -from decorator import decorator +from popen2 import Popen4 from itertools import chain from subprocess import check_call, call, PIPE, STDOUT, Popen -from ZODB.POSException import ConflictError -from popen2 import Popen4 -from twisted.internet import task, reactor, defer -from Acquisition import aq_base, aq_inner, aq_parent -from zExceptions import NotFound from AccessControl import getSecurityManager, Unauthorized from AccessControl.ZopeGuards import guarded_getattr -from ZServer.HTTPServer import zhttp_channel +from Acquisition import aq_base, aq_inner, aq_parent +from decorator import decorator +from twisted.internet import task, reactor, defer +from zExceptions import NotFound +from ZODB.POSException import ConflictError from zope.i18n import translate from zope.interface import providedBy -from zope.schema import getFields from zope.schema._field import Password +from zope.schema import getFields +from ZServer.HTTPServer import zhttp_channel from .Exceptions import ZenPathError, ZentinelException from .jsonutils import unjson from .Logger import ( # noqa: F401 - HtmlFormatter, setWebLoggingStream, clearWebLoggingStream, setLogLevel, + clearWebLoggingStream, + HtmlFormatter, + setLogLevel, + setWebLoggingStream, ) from .Threading import ( # noqa: F401 - ThreadInterrupt, InterruptableThread, LineReader, + InterruptableThread, + LineReader, + ThreadInterrupt, ) log = logging.getLogger("zen.Utils") @@ -69,21 +75,21 @@ class DictAsObj(object): def __init__(self, **kwargs): - for k,v in kwargs.iteritems(): setattr(self, k, v) + for k, v in kwargs.iteritems(): + setattr(self, k, v) def convToUnits(number=0, divby=1024.0, unitstr="B"): - """ - Convert a number to its human-readable form. ie: 4GB, 4MB, etc. + """Convert a number to its human-readable form. ie: 4GB, 4MB, etc. - >>> convToUnits() # Don't do this! - '0.0B' - >>> convToUnits(None) # Don't do this! - '' - >>> convToUnits(123456789) - '117.7MB' - >>> convToUnits(123456789, 1000, "Hz") - '123.5MHz' + >>> convToUnits() # Don't do this! + '0.0B' + >>> convToUnits(None) # Don't do this! + '' + >>> convToUnits(123456789) + '117.7MB' + >>> convToUnits(123456789, 1000, "Hz") + '123.5MHz' @param number: base number @type number: number @@ -94,25 +100,25 @@ def convToUnits(number=0, divby=1024.0, unitstr="B"): @return: number with appropriate units @rtype: string """ - units = map(lambda x:x + unitstr, ('','K','M','G','T','P')) + units = map(lambda x: x + unitstr, ("", "K", "M", "G", "T", "P")) try: numb = float(number) except Exception: - return '' + return "" sign = 1 if numb < 0: numb = abs(numb) sign = -1 for unit in units: - if numb < divby: break + if numb < divby: + break numb /= divby return "%.1f%s" % (numb * sign, unit) def travAndColl(obj, toonerel, collect, collectname): - """ - Walk a series of to one rels collecting collectname into collect + """Walk a series of to one rels collecting collectname into collect. @param obj: object inside of Zope @type obj: object @@ -137,8 +143,8 @@ def travAndColl(obj, toonerel, collect, collectname): def getObjByPath(base, path, restricted=0): - """ - Get a Zope object by its path (e.g. '/Devices/Server/Linux'). + """Get a Zope object by its path (e.g. '/Devices/Server/Linux'). + Mostly a stripdown of unrestrictedTraverse method from Zope 2.8.8. @param base: base part of a path @type base: string @@ -160,13 +166,13 @@ def getObjByPath(base, path, restricted=0): if isinstance(path, str): # Unicode paths are not allowed - path = path.split('/') + path = path.split("/") else: path = list(path) - REQUEST = {'TraversalRequestNameStack': path} + REQUEST = {"TraversalRequestNameStack": path} path.reverse() - path_pop=path.pop + path_pop = path.pop if len(path) > 1 and not path[0]: # Remove trailing slash @@ -181,28 +187,28 @@ def getObjByPath(base, path, restricted=0): # If the path starts with an empty string, go to the root first. path_pop() base = base.getPhysicalRoot() - if (restricted - and not securityManager.validate(None, None, None, base)): - raise Unauthorized( base ) + if restricted and not securityManager.validate(None, None, None, base): + raise Unauthorized(base) obj = base while path: name = path_pop() - if name[0] == '_': + if name[0] == "_": # Never allowed in a URL. - raise NotFound( name ) + raise NotFound(name) - if name == '..': + if name == "..": next = aq_parent(obj) if next is not _none: if restricted and not securityManager.validate( - obj, obj,name, next): - raise Unauthorized( name ) + obj, obj, name, next + ): + raise Unauthorized(name) obj = next continue - bobo_traverse = _getattr(obj, '__bobo_traverse__', _none) + bobo_traverse = _getattr(obj, "__bobo_traverse__", _none) if bobo_traverse is not _none: next = bobo_traverse(REQUEST, name) if restricted: @@ -210,7 +216,7 @@ def getObjByPath(base, path, restricted=0): # The object is wrapped, so the acquisition # context is the container. container = aq_parent(aq_inner(next)) - elif _getattr(next, 'im_self', _none) is not _none: + elif _getattr(next, "im_self", _none) is not _none: # Bound method, the bound instance # is the container container = next.im_self @@ -223,7 +229,8 @@ def getObjByPath(base, path, restricted=0): container = _none try: validated = securityManager.validate( - obj, container, name, next) + obj, container, name, next + ) except Unauthorized: # If next is a simple unwrapped property, it's # parentage is indeterminate, but it may have been @@ -231,11 +238,13 @@ def getObjByPath(base, path, restricted=0): # raise an error, and we can explicitly check that # our value was acquired safely. validated = 0 - if container is _none and \ - guarded_getattr(obj, name, marker) is next: + if ( + container is _none + and guarded_getattr(obj, name, marker) is next + ): validated = 1 if not validated: - raise Unauthorized( name ) + raise Unauthorized(name) else: if restricted: next = guarded_getattr(obj, name, marker) @@ -243,20 +252,22 @@ def getObjByPath(base, path, restricted=0): next = _getattr(obj, name, marker) if next is marker: try: - next=obj[name] + next = obj[name] except AttributeError: # Raise NotFound for easier debugging # instead of AttributeError: __getitem__ - raise NotFound( name ) + raise NotFound(name) if restricted and not securityManager.validate( - obj, obj, _none, next): - raise Unauthorized( name ) + obj, obj, _none, next + ): + raise Unauthorized(name) obj = next return obj + def getObjByPath2(base, path, restricted=0): - """ - Get a Zope object by its path (e.g. '/Devices/Server/Linux'). + """Get a Zope object by its path (e.g. '/Devices/Server/Linux'). + Mostly a stripdown of unrestrictedTraverse method from Zope 2.8.8. @param base: base part of a path @@ -277,13 +288,13 @@ def getObjByPath2(base, path, restricted=0): if isinstance(path, str): # Unicode paths are not allowed - path = path.split('/') + path = path.split("/") else: path = list(path) - REQUEST = {'TraversalRequestNameStack': path} + REQUEST = {"TraversalRequestNameStack": path} path.reverse() - path_pop=path.pop + path_pop = path.pop if len(path) > 1 and not path[0]: # Remove trailing slash @@ -298,28 +309,28 @@ def getObjByPath2(base, path, restricted=0): # If the path starts with an empty string, go to the root first. path_pop() base = base.getPhysicalRoot() - if (restricted - and not securityManager.validate(None, None, None, base)): - raise Unauthorized( base ) + if restricted and not securityManager.validate(None, None, None, base): + raise Unauthorized(base) obj = base while path: name = path_pop() - if name[0] == '_': + if name[0] == "_": # Never allowed in a URL. - raise NotFound( name ) + raise NotFound(name) - if name == '..': + if name == "..": next = aq_parent(obj) if next is not _none: if restricted and not securityManager.validate( - obj, obj,name, next): - raise Unauthorized( name ) + obj, obj, name, next + ): + raise Unauthorized(name) obj = next continue - bobo_traverse = _getattr(obj, '__bobo_traverse__', _none) + bobo_traverse = _getattr(obj, "__bobo_traverse__", _none) if bobo_traverse is not _none: next = bobo_traverse(REQUEST, name) if restricted: @@ -327,7 +338,7 @@ def getObjByPath2(base, path, restricted=0): # The object is wrapped, so the acquisition # context is the container. container = aq_parent(aq_inner(next)) - elif _getattr(next, 'im_self', _none) is not _none: + elif _getattr(next, "im_self", _none) is not _none: # Bound method, the bound instance # is the container container = next.im_self @@ -340,7 +351,8 @@ def getObjByPath2(base, path, restricted=0): container = _none try: validated = securityManager.validate( - obj, container, name, next) + obj, container, name, next + ) except Unauthorized: # If next is a simple unwrapped property, it's # parentage is indeterminate, but it may have been @@ -348,83 +360,107 @@ def getObjByPath2(base, path, restricted=0): # raise an error, and we can explicitly check that # our value was acquired safely. validated = 0 - if container is _none and \ - guarded_getattr(obj, name, marker) is next: + if ( + container is _none + and guarded_getattr(obj, name, marker) is next + ): validated = 1 if not validated: - raise Unauthorized( name ) + raise Unauthorized(name) else: - next=obj._getOb(name, None) + next = obj._getOb(name, None) if next is None: raise NotFound(name) if restricted and not securityManager.validate( - obj, obj, _none, next): - raise Unauthorized( name ) + obj, obj, _none, next + ): + raise Unauthorized(name) obj = next return obj def capitalizeFirstLetter(s): - #Don't use .title or .capitalize, as those will lower-case a camel-cased type + # Don't use .title or .capitalize because they lower-case camel-cased names return s[0].capitalize() + s[1:] if s else s RENAME_DISPLAY_TYPES = { - 'RRDTemplate': 'Template', - 'ThresholdClass': 'Threshold', - 'HoltWintersFailure': 'Threshold', # see Trac #29376 + "RRDTemplate": "Template", + "ThresholdClass": "Threshold", + "HoltWintersFailure": "Threshold", # see Trac #29376 } + def getDisplayType(obj): - """ - Get a printable string representing the type of this object - """ + """Get a printable string representing the type of this object.""" # TODO: better implementation, like meta_display_type per class. - typename = str(getattr(obj, 'meta_type', None) or obj.__class__.__name__) if obj else 'None' + typename = ( + str(getattr(obj, "meta_type", None) or obj.__class__.__name__) + if obj + else "None" + ) typename = capitalizeFirstLetter(typename) return RENAME_DISPLAY_TYPES.get(typename, typename) def _getName(obj): - return getattr(obj, 'getName', None) or getattr(obj, 'name', None) or \ - getattr(obj, 'Name', None) + return ( + getattr(obj, "getName", None) + or getattr(obj, "name", None) + or getattr(obj, "Name", None) + ) + def _getId(obj): - return getattr(obj, 'getId', None) or getattr(obj, 'id', None) or \ - getattr(obj, 'Id', None) or getattr(obj, 'ID', None) + return ( + getattr(obj, "getId", None) + or getattr(obj, "id", None) + or getattr(obj, "Id", None) + or getattr(obj, "ID", None) + ) + def _getUid(obj): - return getattr(obj, 'getPrimaryId', None) or getattr(obj, 'uid', None) \ - or getattr(obj, 'Uid', None) or getattr(obj, 'UID', None) + return ( + getattr(obj, "getPrimaryId", None) + or getattr(obj, "uid", None) + or getattr(obj, "Uid", None) + or getattr(obj, "UID", None) + ) + def getDisplayName(obj): - """ - Get a printable string representing the name of this object. + """Get a printable string representing the name of this object. + Always returns something but it may not be pretty. """ # TODO: better implementation, like getDisplayName() per class. - name = obj.titleOrId() if hasattr(obj, 'titleOrId') else \ - _getName(obj) or _getId(obj) or _getUid(obj) + name = ( + obj.titleOrId() + if hasattr(obj, "titleOrId") + else _getName(obj) or _getId(obj) or _getUid(obj) + ) if name is None: - return str(obj) #we tried our best + return str(obj) # we tried our best return str(name() if callable(name) else name) def getDisplayId(obj): - """ - Get a printable string representing an ID of this object. + """Get a printable string representing an ID of this object. + Always returns something but it may not be pretty. """ # TODO: better implementation, like getDisplayId() per class. dispId = _getUid(obj) or _getId(obj) or _getName(obj) if dispId is None: - return str(obj) #we tried our best - return re.sub(r'^/zport/dmd', '', str(dispId() if callable(dispId) else dispId)) + return str(obj) # we tried our best + return re.sub( + r"^/zport/dmd", "", str(dispId() if callable(dispId) else dispId) + ) def checkClass(myclass, className): - """ - Perform issubclass using class name as string + """Perform issubclass using class name as string. @param myclass: generic object @type myclass: object @@ -441,8 +477,7 @@ def checkClass(myclass, className): def lookupClass(productName, classname=None): - """ - look in sys.modules for our class + """Look in sys.modules for our class. @param productName: object in Products @type productName: string @@ -452,23 +487,22 @@ def lookupClass(productName, classname=None): @rtype: object or None """ if productName in sys.modules: - mod = sys.modules[productName] + mod = sys.modules[productName] - elif "Products."+productName in sys.modules: - mod = sys.modules["Products."+productName] + elif "Products." + productName in sys.modules: + mod = sys.modules["Products." + productName] else: - return None + return None if not classname: - classname = productName.split('.')[-1] + classname = productName.split(".")[-1] - return getattr(mod,classname) + return getattr(mod, classname) def importClass(modulePath, classname=""): - """ - Import a class from the module given. + """Import a class from the module given. @param modulePath: path to module in sys.modules @type modulePath: string @@ -478,7 +512,8 @@ def importClass(modulePath, classname=""): @rtype: class """ try: - if not classname: classname = modulePath.split(".")[-1] + if not classname: + classname = modulePath.split(".")[-1] try: __import__(modulePath, globals(), locals(), classname) mod = sys.modules[modulePath] @@ -487,21 +522,32 @@ def importClass(modulePath, classname=""): return getattr(mod, classname) except AttributeError: - raise ImportError("Failed while importing class %s from module %s" % ( - classname, modulePath)) + raise ImportError( + "Failed while importing class %s from module %s" + % (classname, modulePath) + ) def cleanstring(value): - """ - Take the trailing \x00 off the end of a string + """Take the trailing \x00 off the end of a string. + + >>> txt = 'clean' + >>> cleanstring(txt) == txt + True + >>> cleanstring(txt + chr(0)) == txt + True + >>> cleanstring(txt + chr(0) + chr(0)) == txt + True @param value: sample string @type value: string @return: cleaned string @rtype: string """ - if isinstance(value, basestring) and value.endswith('\0'): - value = value[:-1] + if isinstance(value, basestring): + offset = value.find("\0") + if offset >= 0: + value = value[:offset] return value @@ -513,16 +559,19 @@ def getSubObjects(base, filter=None, descend=None, retobjs=None): @param base: base object to start search @type base: object - @param filter: filter to apply to each object to determine if it gets added to the returned list + @param filter: filter to apply to each object to determine if it gets + added to the returned list. @type filter: function or None - @param descend: function to apply to each object to determine whether or not to continue searching + @param descend: function to apply to each object to determine whether or + not to continue searching. @type descend: function or None @param retobjs: list of objects found @type retobjs: list @return: list of objects found @rtype: list """ - if not retobjs: retobjs = [] + if not retobjs: + retobjs = [] for obj in base.objectValues(): if not filter or filter(obj): retobjs.append(obj) @@ -541,24 +590,27 @@ def getSubObjectsMemo(base, filter=None, descend=None, memo={}): @param base: base object to start search @type base: object - @param filter: filter to apply to each object to determine if it gets added to the returned list + @param filter: filter to apply to each object to determine if it gets + added to the returned list. @type filter: function or None - @param descend: function to apply to each object to determine whether or not to continue searching + @param descend: function to apply to each object to determine whether or + not to continue searching. @type descend: function or None @param memo: dictionary of objects found (unused) @type memo: dictionary @return: list of objects found @rtype: list """ - from Products.ZenRelations.RelationshipManager \ - import RelationshipManager + from Products.ZenRelations.RelationshipManager import RelationshipManager + if base.meta_type == "To One Relationship": objs = [base.obj] else: objs = base.objectValues() for obj in objs: - if (isinstance(obj, RelationshipManager) and - not obj.getPrimaryDmdId().startswith(base.getPrimaryDmdId())): + if isinstance( + obj, RelationshipManager + ) and not obj.getPrimaryDmdId().startswith(base.getPrimaryDmdId()): continue if not filter or filter(obj): yield obj @@ -568,8 +620,7 @@ def getSubObjectsMemo(base, filter=None, descend=None, memo={}): def getAllConfmonObjects(base): - """ - Get all ZenModelRM objects in database + """Get all ZenModelRM objects in database. @param base: base object to start searching @type base: object @@ -578,26 +629,26 @@ def getAllConfmonObjects(base): """ from Products.ZenModel.ZenModelRM import ZenModelRM from Products.ZenModel.ZenModelBase import ZenModelBase - from Products.ZenRelations.ToManyContRelationship \ - import ToManyContRelationship - from Products.ZenRelations.ToManyRelationship \ - import ToManyRelationship - from Products.ZenRelations.ToOneRelationship \ - import ToOneRelationship + from Products.ZenRelations.ToManyContRelationship import ( + ToManyContRelationship, + ) + from Products.ZenRelations.ToManyRelationship import ToManyRelationship + from Products.ZenRelations.ToOneRelationship import ToOneRelationship def descend(obj): - """ - Function to determine whether or not to continue searching + """Function to determine whether or not to continue searching. + @param obj: object @type obj: object @return: True if we want to keep searching @rtype: boolean """ return ( - isinstance(obj, ZenModelBase) or - isinstance(obj, ToManyContRelationship) or - isinstance(obj, ToManyRelationship) or - isinstance(obj, ToOneRelationship)) + isinstance(obj, ZenModelBase) + or isinstance(obj, ToManyContRelationship) + or isinstance(obj, ToManyRelationship) + or isinstance(obj, ToOneRelationship) + ) def filter(obj): """ @@ -615,8 +666,12 @@ def filter(obj): def zenpathsplit(pathstring): - """ - Split a zen path and clean up any blanks or bogus spaces in it + """Return the parts of a path with extraneous spaces removed. + + >>> zenpathsplit('/zport/dmd/Devices') + ['zport', 'dmd', 'Devices'] + >>> zenpathsplit(' a /b / c') + ['a', 'b', 'c'] @param pathstring: a path inside of ZENHOME @type pathstring: string @@ -629,17 +684,23 @@ def zenpathsplit(pathstring): return path - def zenpathjoin(pathar): - """ - Build a zenpath in its string form + """Return a string that is the path formed from its parts. + + The returned path is always an absolute path. + + >>> zenpathjoin(('zport', 'dmd', 'Devices', 'Server')) + '/zport/dmd/Devices/Server' + >>> zenpathjoin(('', 'zport', 'dmd', 'Devices', 'Server')) + '/zport/dmd/Devices/Server' @param pathar: a path @type pathar: string @return: a path @rtype: string """ - return "/" + "/".join(pathar) + path = "/".join(pathar) + return path if path.startswith("/") else "/" + path def createHierarchyObj(root, name, factory, relpath="", llog=None): @@ -654,7 +715,8 @@ def createHierarchyObj(root, name, factory, relpath="", llog=None): @type name: string @param factory: factory object to create @type factory: factory object - @param relpath: relationship within which we will recurse as objects are created, if any + @param relpath: relationship within which we will recurse as objects are + created, if any. @type relpath: object @param llog: unused @type llog: object @@ -664,13 +726,16 @@ def createHierarchyObj(root, name, factory, relpath="", llog=None): unused(llog) rootName = root.id for id in zenpathsplit(name): - if id == rootName: continue + if id == rootName: + continue if id == relpath or getattr(aq_base(root), relpath, False): root = getattr(root, relpath) if not getattr(aq_base(root), id, False): if id == relpath: raise AttributeError("relpath %s not found" % relpath) - log.debug("Creating object with id %s in object %s", id, root.getId()) + log.debug( + "Creating object with id %s in object %s", id, root.getId() + ) newobj = factory(id) root._setObject(id, newobj) root = getattr(root, id) @@ -679,14 +744,14 @@ def createHierarchyObj(root, name, factory, relpath="", llog=None): def getHierarchyObj(root, name, relpath=None): - """ - Return an object using its path relations are optional in the path. + """Return an object using its path relations are optional in the path. @param root: root from which to start @type root: object @param name: path to object @type name: string - @param relpath: relationship within which we will recurse as objects are created, if any + @param relpath: relationship within which we will recurse as objects are + created, if any. @type relpath: object @return: root object of a hierarchy @rtype: object @@ -695,14 +760,15 @@ def getHierarchyObj(root, name, relpath=None): if id == relpath or getattr(aq_base(root), relpath, False): root = getattr(root, relpath) if not getattr(root, id, False): - raise ZenPathError("Path %s id %s not found on object %s" % - (name, id, root.getPrimaryId())) + raise ZenPathError( + "Path %s id %s not found on object %s" + % (name, id, root.getPrimaryId()) + ) root = getattr(root, id, None) return root - def basicAuthUrl(username, password, url): """ Add the username and password to a url in the form @@ -717,16 +783,15 @@ def basicAuthUrl(username, password, url): @return: URL with auth information incorporated @rtype: string """ - urlar = url.split('/') - if not username or not password or urlar[2].find('@') > -1: + urlar = url.split("/") + if not username or not password or urlar[2].find("@") > -1: return url urlar[2] = "%s:%s@%s" % (username, password, urlar[2]) return "/".join(urlar) - -def prepId(id, subchar='_'): - """ +def prepId(id, subchar="_"): + r""" Make an id with valid url characters. Subs [^a-zA-Z0-9-_,.$\(\) ] with subchar. If id then starts with subchar it is removed. @@ -735,22 +800,24 @@ def prepId(id, subchar='_'): @return: valid id @rtype: string """ - _prepId = re.compile(r'[^a-zA-Z0-9-_,.$\(\) ]').sub + _prepId = re.compile(r"[^a-zA-Z0-9-_,.$\(\) ]").sub _cleanend = re.compile(r"%s+$" % subchar).sub if id is None: - raise ValueError('Ids can not be None') + raise ValueError("Ids can not be None") if not isinstance(id, basestring): id = str(id) id = _prepId(subchar, id) while id.startswith(subchar): - if len(id) > 1: id = id[1:] - else: id = "-" - id = _cleanend("",id) - id = id.lstrip(string.whitespace + '_').rstrip() + if len(id) > 1: + id = id[1:] + else: + id = "-" + id = _cleanend("", id) + id = id.lstrip(string.whitespace + "_").rstrip() return str(id) -def sendEmail(emsg, host, port=25, usetls=0, usr='', pwd=''): +def sendEmail(emsg, host, port=25, usetls=0, usr="", pwd=""): """ Send an email. Return a tuple: (sucess, message) where sucess is True or False. @@ -771,24 +838,28 @@ def sendEmail(emsg, host, port=25, usetls=0, usr='', pwd=''): @rtype: tuple """ import smtplib - fromaddr = emsg['From'] - toaddr = map(lambda x: x.strip(), emsg['To'].split(',')) + + fromaddr = emsg["From"] + toaddr = map(lambda x: x.strip(), emsg["To"].split(",")) try: server = smtplib.SMTP(host, port, timeout=DEFAULT_SOCKET_TIMEOUT) if usetls: server.ehlo() server.starttls() server.ehlo() - if len(usr): server.login(usr, pwd) + if len(usr): + server.login(usr, pwd) server.sendmail(fromaddr, toaddr, emsg.as_string()) # Need to catch the quit because some servers using TLS throw an # EOF error on quit, so the email gets sent over and over - try: server.quit() - except Exception: pass + try: + server.quit() + except Exception: + pass except (smtplib.SMTPException, socket.error, socket.timeout): - result = (False, '%s - %s' % tuple(sys.exc_info()[:2])) + result = (False, "%s - %s" % tuple(sys.exc_info()[:2])) else: - result = (True, '') + result = (True, "") return result @@ -807,14 +878,17 @@ def sendPage(recipient, msg, pageCommand, deferred=False): @rtype: tuple """ import subprocess + env = dict(os.environ) env["RECIPIENT"] = recipient msg = str(msg) - p = subprocess.Popen(pageCommand, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - shell=True, - env=env) + p = subprocess.Popen( + pageCommand, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + shell=True, + env=env, + ) p.stdin.write(msg) p.stdin.close() response = p.stdout.read() @@ -822,8 +896,7 @@ def sendPage(recipient, msg, pageCommand, deferred=False): def zdecode(context, value): - """ - Convert a string using the decoding found in zCollectorDecoding + """Convert a string using the decoding found in zCollectorDecoding. @param context: Zope object @type context: object @@ -833,14 +906,14 @@ def zdecode(context, value): @rtype: string """ if isinstance(value, str): - decoding = getattr(context, 'zCollectorDecoding', 'utf-8') + decoding = getattr(context, "zCollectorDecoding", "utf-8") value = value.decode(decoding) return value def localIpCheck(context, ip): - """ - Test to see if an IP should not be included in the network map. + """Test to see if an IP should not be included in the network map. + Uses the zLocalIpAddresses to decide. @param context: Zope object @@ -850,11 +923,12 @@ def localIpCheck(context, ip): @return: regular expression match or None (if not found) @rtype: re match object """ - return re.search(getattr(context, 'zLocalIpAddresses', '^$'), ip) + return re.search(getattr(context, "zLocalIpAddresses", "^$"), ip) + def localInterfaceCheck(context, intname): - """ - Test to see if an interface should not be included in the network map. + """Test to see if an interface should not be included in the network map. + Uses the zLocalInterfaceNames to decide. @param context: Zope object @@ -864,14 +938,13 @@ def localInterfaceCheck(context, intname): @return: regular expression match or None (if not found) @rtype: re match object """ - return re.search(getattr(context, 'zLocalInterfaceNames', '^$'), intname) + return re.search(getattr(context, "zLocalInterfaceNames", "^$"), intname) def cmpClassNames(obj, classnames): """ - Check to see if any of an object's base classes - are in a list of class names. Like isinstance(), - but without requiring a class to compare against. + Check to see if any of an object's base classes are in a list of class + names. Like isinstance(), but without requiring a class to compare against. @param obj: object @type obj: object @@ -886,12 +959,11 @@ def cmpClassNames(obj, classnames): thisclass = x.pop() x.extend(thisclass.__bases__) finalnames.add(thisclass.__name__) - return bool( set(classnames).intersection(finalnames) ) + return bool(set(classnames).intersection(finalnames)) def resequence(context, objects, seqmap, origseq, REQUEST): - """ - Resequence a seqmap + """Resequence a seqmap. @param context: Zope object @type context: object @@ -925,21 +997,19 @@ def resequence(context, objects, seqmap, origseq, REQUEST): def cleanupSkins(dmd): - """ - Prune out objects + """Prune out objects. @param dmd: Device Management Database @type dmd: DMD object """ ps = dmd.getPhysicalRoot().zport.portal_skins layers = ps._objects - layers = filter(lambda x:getattr(ps, x['id'], False), layers) + layers = filter(lambda x: getattr(ps, x["id"], False), layers) ps._objects = tuple(layers) def edgesToXML(edges, start=()): - """ - Convert edges to an XML file + """Convert edges to an XML file. @param edges: edges @type edges: list @@ -957,73 +1027,126 @@ def edgesToXML(edges, start=()): node1 = nodet % (a_id, a_title, a_icon_path, a_color) node2 = nodet % (b_id, b_title, b_icon_path, b_color) edge1 = edget % (a_title, b_id) - if node1 not in nodeels: nodeels.append(node1) - if node2 not in nodeels: nodeels.append(node2) - if edge1 not in edgeels: edgeels.append(edge1) + if node1 not in nodeels: + nodeels.append(node1) + if node2 not in nodeels: + nodeels.append(node2) + if edge1 not in edgeels: + edgeels.append(edge1) xmlels.extend(nodeels) xmlels.extend(edgeels) - xmldoc = "%s" % ''.join(list(xmlels)) + xmldoc = "%s" % "".join(list(xmlels)) return xmldoc -def sane_pathjoin(base_path, *args ): - """ - Joins paths in a saner manner than os.path.join() - - @param base_path: base path to assume everything is rooted from +def _normalize_path(path): + """Return the given path sans extraneous spaces and slashes. + + Trailing slashes are removed. + + >>> _normalize_path('a') + 'a' + >>> _normalize_path('/a') + '/a' + >>> _normalize_path('/a/b/') + '/a/b' + >>> _normalize_path('a/b/') + 'a/b' + >>> _normalize_path('a//b/') + 'a/b' + >>> _normalize_path('//a//b/') + '/a/b' + >>> _normalize_path(' / a / b / ') + '/a/b' + >>> _normalize_path(' a / b / ') + 'a/b' + >>> _normalize_path(' a / b ') + 'a/b' + """ + # removes leading/trailing spaces from path parts and removes path parts + # that are empty strings. + parts = [p.strip() for p in path.split("/")] + # Never eliminate the first part + return "/".join(parts[0:1] + [p for p in parts[1:] if p]) + + +def sane_pathjoin(base_path, *args): + """Returns a path string constructed from the arguments. + + The first argument ('base_path') is always the root part of the path. + This differs from os.path.join's behavior of discarding earlier path + parts if later path parts have a leading slash. + + The base_path and *args are two paths to be joined. If the left-most + parts of *args matches base_path, only the parts after the match are + used in the resulting path. + + >>> sane_pathjoin('a') + 'a' + >>> sane_pathjoin('/a') + '/a' + >>> sane_pathjoin('/a', 'b', 'c') + '/a/b/c' + >>> sane_pathjoin('/a', '/b', '/c') + '/a/b/c' + >>> sane_pathjoin('/a', 'b', '/c') + '/a/b/c' + >>> sane_pathjoin('a', 'b', 'c') + 'a/b/c' + >>> sane_pathjoin('a', '/b', '/c') + 'a/b/c' + >>> sane_pathjoin('a', 'b', '/c') + 'a/b/c' + >>> sane_pathjoin('a', '') + 'a' + >>> sane_pathjoin('a', '', 'b') + 'a/b' + >>> sane_pathjoin('/a ', ' b ', '/ c', '/d ') + '/a/b/c/d' + >>> sane_pathjoin('/a/b', '/a/b', 'c') + '/a/b/c' + >>> sane_pathjoin('/a/b', 'a', 'b', 'c') + '/a/b/c' + >>> sane_pathjoin('a/b', '/a/b', 'c') + 'a/b/c' + >>> sane_pathjoin('a/b', 'a', 'b', 'c') + 'a/b/c' + + @param base_path: Base path to assume everything is rooted from. @type base_path: string - @param *args: path components starting from $ZENHOME - @type *args: strings + @param *args: Path parts that follow base_path. + @type *args: Sequence of strings @return: sanitized path @rtype: string """ - path = base_path - if args: - # Hugely bizarre (but documented!) behaviour with os.path.join() - # >>> import os.path - # >>> os.path.join( '/blue', 'green' ) - # '/blue/green' - # >>> os.path.join( '/blue', '/green' ) - # '/green' - # Work around the brain damage... - base = args[0] - if base.startswith( base_path ): - path_args = [ base ] + [a.strip('/') for a in args[1:] if a != '' ] - else: - path_args = [a.strip('/') for a in args if a != '' ] - - # Empty strings get thrown out so we may not have anything - if len(path_args) > 0: - # What if the user splits up base_path and passes it in? - pathological_case = os.path.join( *path_args ) - if pathological_case.startswith( base_path ): - pass - - elif not base.startswith( base_path ): - path_args.insert( 0, base_path ) - - # Note: passing in a list to os.path.join() returns a list, - # again completely unlike string join() - path = os.path.join( *path_args ) - - # os.path.join( '/blue', '' ) returns '/blue/' -- egads! - return path.rstrip('/') + root = _normalize_path(base_path) + subpath = _normalize_path("/".join(args)) + if subpath: + # subpath should always be a relative path. + if subpath[0] == "/": + subpath = subpath[1:] + # Get a relative path from the root path. + relbase = root[1:] if root[0:1] == "/" else root + if relbase and subpath.startswith(relbase): + subpath = subpath[len(relbase) + 1 :] + return "/".join((root, subpath)) + return root def varPath(*args): + """Return a path relative to /var/zenoss specified by joining args. + + The path is not guaranteed to exist on the filesystem. """ - Return a path relative to /var/zenoss specified by joining args. As with - zenPath(), the path is not guaranteed to exist on the filesystem. - """ - return sane_pathjoin('/var/zenoss', *args) + return sane_pathjoin("/var/zenoss", *args) def zenPath(*args): - """ - Return a path relative to $ZENHOME specified by joining args. The path - is not guaranteed to exist on the filesystem. + """Return a path relative to $ZENHOME specified by joining args. + + The path is not guaranteed to exist on the filesystem. >>> import os >>> zenHome = os.environ['ZENHOME'] @@ -1044,7 +1167,8 @@ def zenPath(*args): True >>> zenPath(zenPath('Products')) == zenPath('Products') True - >>> zenPath(zenPath('Products'), 'orange', 'blue' ) == zenPath('Products', 'orange', 'blue' ) + >>> zenPath(zenPath('Products'), 'orange', 'blue' ) \ + == zenPath('Products', 'orange', 'blue' ) True # Pathological case @@ -1054,16 +1178,17 @@ def zenPath(*args): @param *args: path components starting from $ZENHOME @type *args: strings - @todo: determine what the correct behaviour should be if $ZENHOME is a symlink! + @todo: determine what the correct behaviour should be if $ZENHOME + is a symlink! """ - zenhome = os.environ.get( 'ZENHOME', '' ) + zenhome = os.environ.get("ZENHOME", "") - path = sane_pathjoin( zenhome, *args ) + path = sane_pathjoin(zenhome, *args) - #test if ZENHOME based path exists and if not try bitrock-style path. - #if neither exists return the ZENHOME-based path + # test if ZENHOME based path exists and if not try bitrock-style path. + # if neither exists return the ZENHOME-based path if not os.path.exists(path): - brPath = os.path.realpath(os.path.join(zenhome, '..', 'common')) + brPath = os.path.realpath(os.path.join(zenhome, "..", "common")) testPath = sane_pathjoin(brPath, *args) if os.path.exists(testPath): path = testPath @@ -1088,8 +1213,8 @@ def zopePath(*args): @param *args: path components starting from $ZOPEHOME @type *args: strings """ - zopehome = os.environ.get('ZOPEHOME', '') - return sane_pathjoin( zopehome, *args ) + zopehome = os.environ.get("ZOPEHOME", "") + return sane_pathjoin(zopehome, *args) def binPath(fileName): @@ -1110,17 +1235,21 @@ def binPath(fileName): @rtype: string """ # bin and libexec are the usual suspect locations - paths = [zenPath(d, fileName) for d in ('bin', 'libexec')] + paths = [zenPath(d, fileName) for d in ("bin", "libexec")] # $ZOPEHOME/bin is an additional option for appliance - paths.append(zopePath('bin', fileName)) - # also check the standard locations for Nagios plugins (/usr/lib(64)/nagios/plugins) - paths.extend(sane_pathjoin(d, fileName) for d in ('/usr/lib/nagios/plugins', - '/usr/lib64/nagios/plugins')) + paths.append(zopePath("bin", fileName)) + # Also check the standard locations for Nagios plugins + # (/usr/lib(64)/nagios/plugins) + paths.extend( + sane_pathjoin(d, fileName) + for d in ("/usr/lib/nagios/plugins", "/usr/lib64/nagios/plugins") + ) for path in paths: if os.path.isfile(path): return path - return '' + return "" + def extractPostContent(REQUEST): """ @@ -1140,39 +1269,36 @@ def extractPostContent(REQUEST): # IE return REQUEST.form.keys()[0] except Exception: - return '' + return "" def unused(*args): - """ - A no-op function useful for shutting up pychecker + """A no-op function useful for shutting up pychecker. @param *args: arbitrary arguments @type *args: objects @return: count of the objects @rtype: integer """ - return len(args) + pass def isXmlRpc(REQUEST): - """ - Did we receive a XML-RPC call? + """Did we receive a XML-RPC call? @param REQUEST: Zope REQUEST object @type REQUEST: Zope REQUEST object @return: True if REQUEST is an XML-RPC call @rtype: boolean """ - if REQUEST and REQUEST['CONTENT_TYPE'].find('xml') > -1: + if REQUEST and REQUEST["CONTENT_TYPE"].find("xml") > -1: return True else: return False def setupLoggingHeader(context, REQUEST): - """ - Extract out the 2nd outermost table + """Extract out the 2nd outermost table. @param context: Zope object @type context: Zope object @@ -1193,8 +1319,7 @@ def setupLoggingHeader(context, REQUEST): def executeCommand(cmd, REQUEST, write=None): - """ - Execute the command and return the output + """Execute the command and return the output. @param cmd: command to execute @type cmd: string @@ -1211,11 +1336,13 @@ def executeCommand(cmd, REQUEST, write=None): else: response = sys.stdout if write is None: + def _write(s): response.write(s) response.flush() + write = _write - log.info('Executing command: %s', ' '.join(cmd)) + log.info("Executing command: %s", " ".join(cmd)) f = Popen4(cmd) while 1: s = f.fromchild.readline() @@ -1226,10 +1353,12 @@ def _write(s): else: log.info(s) except ZentinelException as e: - if xmlrpc: return 1 + if xmlrpc: + return 1 log.critical(e) except Exception: - if xmlrpc: return 1 + if xmlrpc: + return 1 raise else: result = f.wait() @@ -1238,8 +1367,8 @@ def _write(s): def ipsort(a, b): - """ - Compare (cmp()) a + b's IP addresses + """Compare (cmp()) a + b's IP addresses. + These addresses may contain subnet mask info. @param a: IP address @@ -1250,16 +1379,19 @@ def ipsort(a, b): @rtype: boolean """ # Use 0.0.0.0 instead of blank string - if not a: a = "0.0.0.0" - if not b: b = "0.0.0.0" + if not a: + a = "0.0.0.0" + if not b: + b = "0.0.0.0" # Strip off netmasks - a, b = map(lambda x:x.rsplit("/")[0], (a, b)) + a, b = map(lambda x: x.rsplit("/")[0], (a, b)) return cmp(*map(socket.inet_aton, (a, b))) + def ipsortKey(a): - """ - Key function to replace cmp version of ipsort + """Key function to replace cmp version of ipsort. + @param a: IP address @type a: string @return: result of socket.inet_aton(a.ip) @@ -1267,12 +1399,12 @@ def ipsortKey(a): """ if not a: a = "0.0.0.0" - a = a.rsplit('/')[0] + a = a.rsplit("/")[0] return socket.inet_aton(a) + def unsigned(v): - """ - Convert negative 32-bit values into the 2's complement unsigned value + """Convert negative 32-bit values into the 2's complement unsigned value. >>> str(unsigned(-1)) '4294967295' @@ -1304,8 +1436,7 @@ def nanToNone(value): def executeStreamCommand(cmd, writefunc, timeout=30): - """ - Execute cmd in the shell and send the output to writefunc. + """Execute cmd in the shell and send the output to writefunc. @param cmd: command to execute @type cmd: string @@ -1320,46 +1451,45 @@ def executeStreamCommand(cmd, writefunc, timeout=30): pollPeriod = 1 endtime = time.time() + timeout firstPass = True - while time.time() < endtime and ( - firstPass or child.poll()==-1): + while time.time() < endtime and (firstPass or child.poll() == -1): firstPass = False - r,w,e = select.select([child.fromchild],[],[],pollPeriod) + r, w, e = select.select([child.fromchild], [], [], pollPeriod) if r: t = child.fromchild.read() if t: writefunc(t) - if child.poll()==-1: - writefunc('Command timed out') + if child.poll() == -1: + writefunc("Command timed out") import signal + os.kill(child.pid, signal.SIGKILL) def monkeypatch(target): - """ - A decorator to patch the decorated function into the given class. - - >>> @monkeypatch('Products.ZenModel.DataRoot.DataRoot') - ... def do_nothing_at_all(self): - ... print "I do nothing at all." - ... - >>> from Products.ZenModel.DataRoot import DataRoot - >>> hasattr(DataRoot, 'do_nothing_at_all') - True - >>> DataRoot('dummy').do_nothing_at_all() - I do nothing at all. + """A decorator to patch the decorated function into the given class. + + >>> @monkeypatch('Products.ZenModel.DataRoot.DataRoot') + ... def do_nothing_at_all(self): + ... print "I do nothing at all." + ... + >>> from Products.ZenModel.DataRoot import DataRoot + >>> hasattr(DataRoot, 'do_nothing_at_all') + True + >>> DataRoot('dummy').do_nothing_at_all() + I do nothing at all. You can also call the original within the new method using a special variable available only locally. - >>> @monkeypatch('Products.ZenModel.DataRoot.DataRoot') - ... def getProductName(self): - ... print "Doing something additional." - ... return 'core' or original(self) - ... - >>> from Products.ZenModel.DataRoot import DataRoot - >>> DataRoot('dummy').getProductName() - Doing something additional. - 'core' + >>> @monkeypatch('Products.ZenModel.DataRoot.DataRoot') + ... def getProductName(self): + ... print "Doing something additional." + ... return 'core' or original(self) + ... + >>> from Products.ZenModel.DataRoot import DataRoot + >>> DataRoot('dummy').getProductName() + Doing something additional. + 'core' You can also stack monkeypatches. @@ -1382,8 +1512,9 @@ def monkeypatch(target): @rtype: function """ if isinstance(target, basestring): - mod, klass = target.rsplit('.', 1) + mod, klass = target.rsplit(".", 1) target = importClass(mod, klass) + def patcher(func): original = getattr(target, func.__name__, None) if original is None: @@ -1391,19 +1522,22 @@ def patcher(func): return func new_globals = copy.copy(func.func_globals) - new_globals['original'] = original - new_func = types.FunctionType(func.func_code, - globals=new_globals, - name=func.func_name, - argdefs=func.func_defaults, - closure=func.func_closure) + new_globals["original"] = original + new_func = types.FunctionType( + func.func_code, + globals=new_globals, + name=func.func_name, + argdefs=func.func_defaults, + closure=func.func_closure, + ) setattr(target, func.__name__, new_func) return func + return patcher + def nocache(f): - """ - Decorator to set headers which force browser to not cache request + """Decorator to set headers which force browser to not cache request. This is intended to decorate methods of BrowserViews. @@ -1412,9 +1546,9 @@ def nocache(f): @return: decorator function return @rtype: function """ + def inner(self, *args, **kwargs): - """ - Inner portion of the decorator + """Inner portion of the decorator. @param *args: arguments @type *args: possible list @@ -1423,18 +1557,22 @@ def inner(self, *args, **kwargs): @return: decorator function return @rtype: function """ - self.request.response.setHeader('Cache-Control', 'no-cache, must-revalidate') - self.request.response.setHeader('Pragma', 'no-cache') - self.request.response.setHeader('Expires', 'Sat, 13 May 2006 18:02:00 GMT') + self.request.response.setHeader( + "Cache-Control", "no-cache, must-revalidate" + ) + self.request.response.setHeader("Pragma", "no-cache") + self.request.response.setHeader( + "Expires", "Sat, 13 May 2006 18:02:00 GMT" + ) # Get rid of kw used to prevent browser caching - kwargs.pop('_dc', None) + kwargs.pop("_dc", None) return f(self, *args, **kwargs) return inner + def formreq(f): - """ - Decorator to pass in request.form information as arguments to a method. + """Decorator to pass in request.form information as arguments to a method. These are intended to decorate methods of BrowserViews. @@ -1443,9 +1581,9 @@ def formreq(f): @return: decorator function return @rtype: function """ + def inner(self, *args, **kwargs): - """ - Inner portion of the decorator + """Inner portion of the decorator. @param *args: arguments @type *args: possible list @@ -1454,7 +1592,7 @@ def inner(self, *args, **kwargs): @return: decorator function return @rtype: function """ - if self.request.REQUEST_METHOD=='POST': + if self.request.REQUEST_METHOD == "POST": content = extractPostContent(self.request) try: args += (unjson(content),) @@ -1463,9 +1601,9 @@ def inner(self, *args, **kwargs): else: kwargs.update(self.request.form) # Get rid of useless Zope thing that appears when no querystring - kwargs.pop('-C', None) + kwargs.pop("-C", None) # Get rid of kw used to prevent browser caching - kwargs.pop('_dc', None) + kwargs.pop("_dc", None) return f(self, *args, **kwargs) return inner @@ -1479,20 +1617,21 @@ class Singleton(type): of the class itself, then checking that attribute for later constructor calls. """ + def __init__(cls, *args, **kwargs): super(Singleton, cls).__init__(*args, **kwargs) cls._singleton_instance = None def __call__(cls, *args, **kwargs): if cls._singleton_instance is None: - cls._singleton_instance = super( - Singleton, cls).__call__(*args, **kwargs) + cls._singleton_instance = super(Singleton, cls).__call__( + *args, **kwargs + ) return cls._singleton_instance def readable_time(seconds, precision=1): - """ - Convert some number of seconds into a human-readable string. + """Convert some number of seconds into a human-readable string. @param seconds: The number of seconds to convert @type seconds: int @@ -1500,42 +1639,49 @@ def readable_time(seconds, precision=1): @type precision: int @rtype: str - >>> readable_time(None) - '0 seconds' - >>> readable_time(0) - '0 seconds' - >>> readable_time(0.12) - '0 seconds' - >>> readable_time(1) - '1 second' - >>> readable_time(1.5) - '1 second' - >>> readable_time(60) - '1 minute' - >>> readable_time(60*60*3+12) - '3 hours' - >>> readable_time(60*60*3+12, 2) - '3 hours 12 seconds' - + >>> readable_time(None) + '0 seconds' + >>> readable_time(0) + '0 seconds' + >>> readable_time(0.12) + '0 seconds' + >>> readable_time(1) + '1 second' + >>> readable_time(1.5) + '1 second' + >>> readable_time(60) + '1 minute' + >>> readable_time(60*60*3+12) + '3 hours' + >>> readable_time(60*60*3+12, 2) + '3 hours 12 seconds' """ if seconds is None: - return '0 seconds' + return "0 seconds" remaining = abs(seconds) if remaining < 1: - return '0 seconds' - - names = ('year', 'month', 'week', 'day', 'hour', 'minute', 'second') - mults = (60*60*24*365, 60*60*24*30, 60*60*24*7, 60*60*24, 60*60, 60, 1) + return "0 seconds" + + names = ("year", "month", "week", "day", "hour", "minute", "second") + mults = ( + 60 * 60 * 24 * 365, + 60 * 60 * 24 * 30, + 60 * 60 * 24 * 7, + 60 * 60 * 24, + 60 * 60, + 60, + 1, + ) result = [] for name, div in zip(names, mults): - num = Decimal(str(math.floor(remaining/div))) - remaining -= int(num)*div + num = Decimal(str(math.floor(remaining / div))) + remaining -= int(num) * div num = int(num) if num: - result.append('%d %s%s' %(num, name, num>1 and 's' or '')) - if len(result)==precision: + result.append("%d %s%s" % (num, name, num > 1 and "s" or "")) + if len(result) == precision: break - return ' '.join(result) + return " ".join(result) def relative_time(t, precision=1, cmptime=None): @@ -1552,32 +1698,30 @@ def relative_time(t, precision=1, cmptime=None): @type cmptime: int @rtype: str - >>> relative_time(time.time() - 60*10) - '10 minutes ago' - >>> relative_time(time.time() - 60*10-3, precision=2) - '10 minutes 3 seconds ago' - >>> relative_time(time.time() - 60*60*24*10, precision=2) - '1 week 3 days ago' - >>> relative_time(time.time() - 60*60*24*365-1, precision=2) - '1 year 1 second ago' - >>> relative_time(time.time() + 1 + 60*60*24*7*2) # Add 1 for rounding - 'in 2 weeks' - + >>> relative_time(time.time() - 60*10) + '10 minutes ago' + >>> relative_time(time.time() - 60*10-3, precision=2) + '10 minutes 3 seconds ago' + >>> relative_time(time.time() - 60*60*24*10, precision=2) + '1 week 3 days ago' + >>> relative_time(time.time() - 60*60*24*365-1, precision=2) + '1 year 1 second ago' + >>> relative_time(time.time() + 1 + 60*60*24*7*2) # Add 1 for rounding + 'in 2 weeks' """ if cmptime is None: cmptime = time.time() seconds = Decimal(str(t - cmptime)) result = readable_time(seconds, precision) if seconds < 0: - result += ' ago' + result += " ago" else: - result = 'in ' + result + result = "in " + result return result def is_browser_connection_open(request): - """ - Check to see if the TCP connection to the browser is still open. + """Check to see if the TCP connection to the browser is still open. This might be used to interrupt an infinite while loop, which would preclude the thread from being destroyed even though the connection has @@ -1596,15 +1740,22 @@ def is_browser_connection_open(request): EXIT_CODE_MAPPING = { - 0:'Success', - 1:'General error', - 2:'Misuse of shell builtins', - 126:'Command invoked cannot execute, permissions problem or command is not an executable', - 127:'Command not found', - 128:'Invalid argument to exit, exit takes only integers in the range 0-255', - 130:'Fatal error signal: 2, Command terminated by Control-C' + 0: "Success", + 1: "General error", + 2: "Misuse of shell builtins", + 126: ( + "Command invoked cannot execute, permissions problem or command " + "is not an executable" + ), + 127: "Command not found", + 128: ( + "Invalid argument to exit, exit takes only integers in the " + "range 0-255" + ), + 130: "Fatal error signal: 2, Command terminated by Control-C", } + def getExitMessage(exitCode): """ Return a nice exit message that corresponds to the given exit status code @@ -1617,32 +1768,35 @@ def getExitMessage(exitCode): if exitCode in EXIT_CODE_MAPPING.keys(): return EXIT_CODE_MAPPING[exitCode] elif exitCode >= 255: - return 'Exit status out of range, exit takes only integer arguments in the range 0-255' + return ( + "Exit status out of range, exit takes only integer arguments " + "in the range 0-255" + ) elif exitCode > 128: - return 'Fatal error signal: %s' % (exitCode-128) - return 'Unknown error code: %s' % exitCode + return "Fatal error signal: %s" % (exitCode - 128) + return "Unknown error code: %s" % exitCode def set_context(ob): - """ - Wrap an object in a REQUEST context. - """ + """Wrap an object in a REQUEST context.""" from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.BaseRequest import RequestContainer + resp = HTTPResponse(stdout=None) env = { - 'SERVER_NAME':'localhost', - 'SERVER_PORT':'8080', - 'REQUEST_METHOD':'GET' - } + "SERVER_NAME": "localhost", + "SERVER_PORT": "8080", + "REQUEST_METHOD": "GET", + } req = HTTPRequest(None, env, resp) - return ob.__of__(RequestContainer(REQUEST = req)) + return ob.__of__(RequestContainer(REQUEST=req)) + def dumpCallbacks(deferred): - """ - Dump the callback chain of a Twisted Deferred object. The chain will be - displayed on standard output. + """Dump the callback chain of a Twisted Deferred object. + + The chain will be displayed on standard output. @param deferred: the twisted Deferred object to dump @type deferred: a Deferred object @@ -1658,9 +1812,6 @@ def dumpCallbacks(deferred): print "%-39.39s %-39.39s" % (callbackName, errbackName) -# add __iter__ method to LazyMap (used to implement catalog queries) to handle -# errors while iterating over the query results using __getitem__ -from Products.ZCatalog.Lazy import LazyMap def LazyMap__iter__(self): for i in range(len(self._seq)): try: @@ -1672,7 +1823,17 @@ def LazyMap__iter__(self): except IndexError: break -LazyMap.__iter__ = LazyMap__iter__ + +def _monkeypath_LazyMap__iter__(): + # Add __iter__ method to LazyMap (used to implement catalog queries) to + # handle errors while iterating over the query results using __getitem__. + from Products.ZCatalog.Lazy import LazyMap + + LazyMap.__iter__ = LazyMap__iter__ + + +_monkeypath_LazyMap__iter__() + def getObjectsFromCatalog(catalog, query=None, log=None): """ @@ -1708,13 +1869,12 @@ def getObjectsFromModelCatalog(catalog, query=None, log=None): def load_config(file, package=None, execute=True): - """ - Load a ZCML file into the context (and avoids duplicate imports). - """ + """Load a ZCML file into the context (and avoids duplicate imports).""" global _LOADED_CONFIGS key = (file, package) - if not key in _LOADED_CONFIGS: + if key not in _LOADED_CONFIGS: from Zope2.App import zcml + zcml.load_config(file, package, execute) _LOADED_CONFIGS.add(key) @@ -1726,42 +1886,38 @@ def load_config_override(file, package=None, execute=True): """ global _LOADED_CONFIGS key = (file, package) - if not key in _LOADED_CONFIGS: + if key not in _LOADED_CONFIGS: from zope.configuration import xmlconfig from Zope2.App.zcml import _context + xmlconfig.includeOverrides(_context, file, package=package) if execute: _context.execute_actions() _LOADED_CONFIGS.add(key) + def has_feature(name): """Return True if named feature is provided, otherwise return False.""" from Zope2.App.zcml import _context + return _context.hasFeature(name) + def rrd_daemon_running(): - """ - The RRD methods in this module are deprecated. - """ - pass + """The RRD methods in this module are deprecated.""" + def rrd_daemon_args(): - """ - The RRD methods in this module are deprecated. - """ - pass + """The RRD methods in this module are deprecated.""" + def rrd_daemon_reset(): - """ - The RRD methods in this module are deprecated. - """ - pass + """The RRD methods in this module are deprecated.""" + def rrd_daemon_retry(fn): - """ - The RRD methods in this module are deprecated. - """ - pass + """The RRD methods in this module are deprecated.""" + @contextlib.contextmanager def get_temp_dir(): @@ -1773,16 +1929,14 @@ def get_temp_dir(): finally: shutil.rmtree(dirname) + def getDefaultZopeUrl(): - """ - Returns the default Zope URL. - """ - return 'http://localhost:8080' + """Returns the default Zope URL.""" + return "http://localhost:8080" def swallowExceptions(log, msg=None, showTraceback=True, returnValue=None): - """ - USE THIS CAUTIOUSLY. Don't hide exceptions carelessly. + """USE THIS CAUTIOUSLY. Don't hide exceptions carelessly. Decorator to safely call a method, logging exceptions without raising them. @@ -1796,6 +1950,7 @@ def closeFilesBeforeExit(): @param showTraceback True to include the stacktrace (the default). @param returnValue The return value on error. """ + @decorator def callSafely(func, *args, **kwargs): try: @@ -1812,9 +1967,9 @@ def callSafely(func, *args, **kwargs): return callSafely + def getAllParserOptionsGen(parser): - """ - Returns a generator of all valid options for the optparse.OptionParser. + """Returns a generator of all valid options for the optparse.OptionParser. @param parser The parser to retrieve options for. @type parser optparse.OptionParser @@ -1833,9 +1988,9 @@ def ipv6_available(): except socket.error: return False + def atomicWrite(filename, data, raiseException=True, createDir=False): - """ - atomicWrite writes data in an atmomic manner to filename. + """Atomically writes data to filename. @param filename Complete path of file to write to. @type filename string @@ -1856,7 +2011,7 @@ def atomicWrite(filename, data, raiseException=True, createDir=False): # create a file in the same directory as the destination file with tempfile.NamedTemporaryFile(dir=dirName, delete=False) as tfile: tfile.write(data) - os.rename(tfile.name, filename) # atomic operation on POSIX systems + os.rename(tfile.name, filename) # atomic operation on POSIX systems except Exception as ex: if tfile is not None and os.path.exists(tfile.name): try: @@ -1869,62 +2024,80 @@ def atomicWrite(filename, data, raiseException=True, createDir=False): def isRunning(daemon): - """ - Determines whether a specific daemon is running by calling 'daemon status' - """ - return call([daemon, 'status'], stdout=PIPE, stderr=STDOUT) == 0 + """Return True if the specified daemon is running.""" + return call([daemon, "status"], stdout=PIPE, stderr=STDOUT) == 0 + def requiresDaemonShutdown(daemon, logger=log): - """ - Performs an operation while the requested daemon is not running. Will stop - and restart the daemon automatically. Throws a CalledProcessError if either - shutdown or restart fails. + """Performs an operation while the requested daemon is not running. + + Will stop and restart the daemon automatically. + + Throws a CalledProcessError if either shutdown or restart fails. @param daemon Which daemon to bring down for the operation. @param logger Which logger to use, or None to not log. """ + @decorator def callWithShutdown(func, *args, **kwargs): cmd = binPath(daemon) running = isRunning(cmd) if running: - if logger: logger.info('Shutting down %s for %s operation...', daemon, func.__name__) - check_call([cmd, 'stop']) + if logger: + logger.info( + "Shutting down %s for %s operation...", + daemon, + func.__name__, + ) + check_call([cmd, "stop"]) # make sure the daemon is actually shut down for i in range(30): nowrunning = isRunning(cmd) - if not nowrunning: break + if not nowrunning: + break time.sleep(1) else: - raise Exception('Failed to terminate daemon %s with command %s' % (daemon, cmd + ' stop')) + raise Exception( + "Failed to terminate daemon %s with command %s" + % (daemon, cmd + " stop") + ) try: return func(*args, **kwargs) except Exception as ex: - if logger: logger.error('Error performing %s operation: %s', func.__name__, ex) + if logger: + logger.error( + "Error performing %s operation: %s", func.__name__, ex + ) raise finally: if running: - if logger: logger.info('Starting %s after %s operation...', daemon, func.__name__) - check_call([cmd, 'start']) + if logger: + logger.info( + "Starting %s after %s operation...", + daemon, + func.__name__, + ) + check_call([cmd, "start"]) return callWithShutdown + def isZenBinFile(name): - """ - Check if given name is a valid file in $ZENHOME/bin. - """ + """Check if given name is a valid file in $ZENHOME/bin.""" if os.path.sep in name: return False return os.path.isfile(binPath(name)) def wait(seconds): - """ - Delays execution of subsequent code. Example: + """Delays execution of subsequent code. + + Example: @defer.inlineCallbacks def incrOne(a): @@ -1949,7 +2122,7 @@ def incrOne(a): giveTimeToReactor = partial(task.deferLater, reactor, 0) -def addXmlServerTimeout(server,timeout=socket._GLOBAL_DEFAULT_TIMEOUT): +def addXmlServerTimeout(server, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): """ Given an instance of xmlrpclib.ServerProxy (same as xmlrpclib.Server), attach a timeout for the underlying http/socket connection. @@ -1980,10 +2153,10 @@ def _timeout_make_connection(self, host): return self._connection[1] chost, self._extra_headers, x509 = self.get_host_info(host) - self._connection = host, httplib.HTTPConnection(chost,timeout=timeout) + self._connection = host, httplib.HTTPConnection(chost, timeout=timeout) return self._connection[1] - def _timeout_make_safe_connection(self,host): + def _timeout_make_safe_connection(self, host): if self._connection and host == self._connection[0]: return self._connection[1] try: @@ -1991,7 +2164,7 @@ def _timeout_make_safe_connection(self,host): except AttributeError: raise NotImplementedError( "your version of httplib doesn't support HTTPS" - ) + ) else: chost, self._extra_headers, x509 = self.get_host_info(host) kwargs = dict(timeout=timeout) @@ -2001,25 +2174,33 @@ def _timeout_make_safe_connection(self,host): return self._connection[1] transport = server._ServerProxy__transport - if isinstance( transport, xmlrpclib.SafeTransport ): - transport.make_connection = types.MethodType( _timeout_make_safe_connection, transport ) + if isinstance(transport, xmlrpclib.SafeTransport): + transport.make_connection = types.MethodType( + _timeout_make_safe_connection, transport + ) else: - transport.make_connection = types.MethodType( _timeout_make_connection, transport ) + transport.make_connection = types.MethodType( + _timeout_make_connection, transport + ) return server + def snmptranslate(*args): - command = ' '.join(['snmptranslate', '-Ln'] + list(args)) + command = " ".join(["snmptranslate", "-Ln"] + list(args)) proc = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) output, errors = proc.communicate() proc.wait() if proc.returncode != 0: - log.error("snmptranslate returned errors for %s: %s", list(args), errors) - return 'Error translating: %s' % list(args) + log.error( + "snmptranslate returned errors for %s: %s", list(args), errors + ) + return "Error translating: %s" % list(args) return output.strip() -def getTranslation(msgId, REQUEST, domain='zenoss'): + +def getTranslation(msgId, REQUEST, domain="zenoss"): """ Take a string like: 'en-us,en;q=0.7,ja;q=0.3' @@ -2028,23 +2209,23 @@ def getTranslation(msgId, REQUEST, domain='zenoss'): Assumes that the input msgId is """ - langs = REQUEST.get('HTTP_ACCEPT_LANGUAGE').split(',') + langs = REQUEST.get("HTTP_ACCEPT_LANGUAGE").split(",") langOrder = [] for lang in langs: - data = lang.split(';q=') + data = lang.split(";q=") if len(data) == 1: - langOrder.append( (1.0, lang) ) + langOrder.append((1.0, lang)) else: - langOrder.append( (data[1], data[0]) ) + langOrder.append((data[1], data[0])) # Search for translations for weight, lang in sorted(langOrder): - msg = translate(msgId, domain=domain, - target_language=lang) + msg = translate(msgId, domain=domain, target_language=lang) # Relies on Zenoss currently using the text as the msgId if msg != msgId: return msg return msg + def unpublished(func): """Makes decorated method unpublished. @@ -2067,12 +2248,14 @@ def executeSshCommand(device, cmd, writefunc): loginTimeout=device.zCommandLoginTimeout, commandTimeout=device.zCommandCommandTimeout, keyPath=device.zKeyPath, - concurrentSessions=device.zSshConcurrentSessions + concurrentSessions=device.zSshConcurrentSessions, + ) + connection = SshClient( + device, + device.manageIp, + device.zCommandPort, + options=ssh_client_options, ) - connection = SshClient(device, - device.manageIp, - device.zCommandPort, - options=ssh_client_options) connection.clientFinished = reactor.stop connection.workList.append(cmd) connection._commands.append(cmd) @@ -2086,8 +2269,8 @@ def executeSshCommand(device, cmd, writefunc): def escapeSpecChars(value): - escape_re = re.compile(r'(?[$&|+\-!(){}[\]^~*?:])') - return escape_re.sub(r'\\\g', value) + escape_re = re.compile(r"(?[$&|+\-!(){}[\]^~*?:])") + return escape_re.sub(r"\\\g", value) def getPasswordFields(interface): @@ -2105,5 +2288,5 @@ def getPasswordFields(interface): def maskSecureProperties(data, secure_properties=[]): for prop in secure_properties: if data.get(prop, None): - data.update({prop: '*' * len(data[prop])}) + data.update({prop: "*" * len(data[prop])}) return data From 6cf1dcd3ca37b6d18dd4f8b19d60b48f68dde239 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 10 Jul 2023 13:23:48 -0500 Subject: [PATCH 014/147] Add/improve docstrings and comments on some ZenHub services. (#4246) --- Products/ZenCollector/services/config.py | 149 ++++++++++------------- Products/ZenHub/HubService.py | 62 ++++++++-- 2 files changed, 118 insertions(+), 93 deletions(-) diff --git a/Products/ZenCollector/services/config.py b/Products/ZenCollector/services/config.py index 1efe4c8c8b..bf6a7f7184 100644 --- a/Products/ZenCollector/services/config.py +++ b/Products/ZenCollector/services/config.py @@ -10,6 +10,7 @@ import base64 import hashlib import logging +import traceback from Acquisition import aq_parent from cryptography.fernet import Fernet @@ -18,6 +19,7 @@ from ZODB.transact import transact from zope import component +from Products.ZenEvents.ZenEventClasses import Critical from Products.ZenHub.HubService import HubService from Products.ZenHub.interfaces import IBatchNotifier from Products.ZenHub.PBDaemon import translateError @@ -38,25 +40,22 @@ class DeviceProxy(pb.Copyable, pb.RemoteCopy): - def __init__(self): - """ - Do not use base classes initializers - """ + """Used to proxy device objects to collection services.""" @property def configId(self): """ - This is the id used by the framework to keep track of configurations, + This is the ID used by the framework to keep track of configurations, what to run, delete etc... - Use this instead of id since certain daemons can use a - configuration id that is different than the underlying device id. + + Use this instead of `id` since certain daemons can use a + configuration ID that is different than the underlying device ID. """ - retval = getattr(self, "_config_id", None) - return retval if (retval is not None) else self.id + cfgId = getattr(self, "_config_id", None) + return cfgId if (cfgId is not None) else self.id @property def deviceGuid(self): - """ """ return getattr(self, "_device_guid", None) def __str__(self): @@ -69,7 +68,7 @@ def __repr__(self): pb.setUnjellyableForClass(DeviceProxy, DeviceProxy) -# TODO: doc me! +# Default attributes copied to every device proxy. BASE_ATTRIBUTES = ( "id", "manageIp", @@ -77,17 +76,16 @@ def __repr__(self): class CollectorConfigService(HubService, ThresholdMixin): + """Base class for ZenHub configuration service classes.""" + def __init__(self, dmd, instance, deviceProxyAttributes=()): """ - Constructs a new CollectorConfig instance. - - Subclasses must call this __init__ method but cannot do so with - the super() since parents of this class are not new-style classes. + Initializes a CollectorConfigService instance. @param dmd: the Zenoss DMD reference @param instance: the collector instance name @param deviceProxyAttributes: a tuple of names for device attributes - that should be copied to every device proxy created + that should be copied to every device proxy created @type deviceProxyAttributes: tuple """ HubService.__init__(self, dmd, instance) @@ -96,7 +94,7 @@ def __init__(self, dmd, instance, deviceProxyAttributes=()): # Get the collector information (eg the 'localhost' collector) self._prefs = self.dmd.Monitors.Performance._getOb(self.instance) - self.config = self._prefs # TODO fix me, needed for ThresholdMixin + self.config = self._prefs # Needed for ThresholdMixin self.configFilter = None # When about to notify daemons about device changes, wait for a little @@ -125,35 +123,31 @@ def _wrapFunction(self, functor, *args, **kwargs): except Exception as ex: msg = "Unhandled exception in zenhub service %s: %s" % ( self.__class__, - str(ex), + ex, ) self.log.exception(msg) - - import traceback - from Products.ZenEvents.ZenEventClasses import Critical - - evt = dict( - severity=Critical, - component=str(self.__class__), - traceback=traceback.format_exc(), - summary=msg, - device=self.instance, - methodCall="%s(%s, %s)" % (functor.__name__, args, kwargs), + self.sendEvent( + dict( + severity=Critical, + component=str(self.__class__), + traceback=traceback.format_exc(), + summary=msg, + device=self.instance, + methodCall="%s(%s, %s)" % (functor.__name__, args, kwargs), + ) ) - self.sendEvent(evt) - return None @onUpdate(PerformanceConf) - def perfConfUpdated(self, object, event): + def perfConfUpdated(self, conf, event): with gc_cache_every(1000, db=self.dmd._p_jar._db): - if object.id == self.instance: + if conf.id == self.instance: for listener in self.listeners: listener.callRemote( - "setPropertyItems", object.propertyItems() + "setPropertyItems", conf.propertyItems() ) @onUpdate(ZenPack) - def zenPackUpdated(self, object, event): + def zenPackUpdated(self, zenpack, event): with gc_cache_every(1000, db=self.dmd._p_jar._db): for listener in self.listeners: try: @@ -167,50 +161,48 @@ def zenPackUpdated(self, object, event): ) @onUpdate(Device) - def deviceUpdated(self, object, event): + def deviceUpdated(self, device, event): with gc_cache_every(1000, db=self.dmd._p_jar._db): - self._notifyAll(object) + self._notifyAll(device) @onUpdate(None) # Matches all - def notifyAffectedDevices(self, object, event): + def notifyAffectedDevices(self, entity, event): # FIXME: This is horrible with gc_cache_every(1000, db=self.dmd._p_jar._db): - if isinstance(object, self._getNotifiableClasses()): - self._reconfigureIfNotify(object) + if isinstance(entity, self._getNotifiableClasses()): + self._reconfigureIfNotify(entity) else: - if isinstance(object, Device): + if isinstance(entity, Device): return - # something else... mark the devices as out-of-date + # Something else... mark the devices as out-of-date template = None - while object: + while entity: # Don't bother with privately managed objects; the ZenPack # will handle them on its own - if is_private(object): + if is_private(entity): return - # walk up until you hit an organizer or a device - if isinstance(object, RRDTemplate): - template = object - if isinstance(object, DeviceClass): - uid = (self.__class__.__name__, self.instance) + # Walk up until you hit an organizer or a device + if isinstance(entity, RRDTemplate): + template = entity + if isinstance(entity, DeviceClass): + uid = (self.name(), self.instance) devfilter = None if template: devfilter = _HasTemplate(template, self.log) self._notifier.notify_subdevices( - object, uid, self._notifyAll, devfilter + entity, uid, self._notifyAll, devfilter ) break - - if isinstance(object, Device): - self._notifyAll(object) + if isinstance(entity, Device): + self._notifyAll(entity) break - - object = aq_parent(object) + entity = aq_parent(entity) @onDelete(Device) - def deviceDeleted(self, object, event): + def deviceDeleted(self, device, event): with gc_cache_every(1000, db=self.dmd._p_jar._db): - devid = object.id - collector = object.getPerformanceServer().getId() + devid = device.id + collector = device.getPerformanceServer().getId() # The invalidation is only sent to the collector where the # deleted device was. if collector == self.instance: @@ -293,7 +285,7 @@ def remote_getEncryptionKey(self): # per collector daemon. s = hashlib.sha256() s.update(key) - s.update(self.__class__.__name__) + s.update(self.name()) return base64.urlsafe_b64encode(s.digest()) def _postCreateDeviceProxy(self, deviceConfigs): @@ -382,17 +374,14 @@ def _filterDevices(self, devices): @rtype: list """ filteredDevices = [] - - for dev in filter(None, devices): + for dev in (d for d in devices if d is not None): try: device = dev.primaryAq() - if self._perfIdFilter(device) and self._filterDevice(device): filteredDevices.append(device) self.log.debug("Device %s included by filter", device.id) else: - # don't use .id just in case there is something - # crazy returned. + # don't use .id just in case something crazy returned. self.log.debug("Device %r excluded by filter", device) except Exception: if self.log.isEnabledFor(logging.DEBUG): @@ -412,17 +401,13 @@ def _perfIdFilter(self, obj): or obj.perfServer.getRelatedId() == self.instance ) - def _notifyAll(self, object): - """ - Notify all instances (daemons) of a change for the device - """ + def _notifyAll(self, device): + """Notify all instances (daemons) of a change for the device.""" # procrastinator schedules a call to _pushConfig - self._procrastinator.doLater(object) + self._procrastinator.doLater(device) def _pushConfig(self, device): - """ - push device config and deletes to relevent collectors/instances - """ + """Push device config and deletes to relevent collectors/instances.""" deferreds = [] if self._perfIdFilter(device) and self._filterDevice(device): @@ -484,9 +469,6 @@ def _pushConfig(self, device): return defer.DeferredList(deferreds) def _sendDeviceProxy(self, listener, proxy): - """ - TODO - """ return listener.callRemote("updateDeviceConfig", proxy) def sendDeviceConfigs(self, configs): @@ -495,7 +477,7 @@ def sendDeviceConfigs(self, configs): def errback(failure): self.log.critical( "Unable to update configs for service instance %s: %s", - self.__class__.__name__, + self.name(), failure, ) @@ -513,18 +495,17 @@ def errback(failure): # FIXME: Don't use _getNotifiableClasses, use @onUpdate(myclasses) def _getNotifiableClasses(self): """ - a tuple of classes. When any object of a type in the sequence is - modified the collector connected to the service will be notified to - update its configuration + Return a tuple of classes. + + When any object of a type in the sequence is modified the collector + connected to the service will be notified to update its configuration. @rtype: tuple """ return () def _pushReconfigure(self, value): - """ - notify the collector to reread the entire configuration - """ + """Notify the collector to reread the entire configuration.""" # value is unused but needed for the procrastinator framework for listener in self.listeners: listener.callRemote("notifyConfigChanged") @@ -546,6 +527,7 @@ def _notifyConfigChange(self, object): """ Called when an object of a type from _getNotifiableClasses is encountered + @return: should a notify config changed be sent @rtype: boolean """ @@ -553,7 +535,8 @@ def _notifyConfigChange(self, object): class _HasTemplate(object): - """Predicate class that checks whether a given device has a template + """ + Predicate class that checks whether a given device has a template matching the given template. """ diff --git a/Products/ZenHub/HubService.py b/Products/ZenHub/HubService.py index 758c2cf0c8..97b259de7c 100644 --- a/Products/ZenHub/HubService.py +++ b/Products/ZenHub/HubService.py @@ -17,20 +17,54 @@ class HubService(pb.Referenceable): + """ + The base class for a ZenHub service class. + + :attr log: The logger object for this service. + :type log: logging.Logger + :attr fqdn: This attribute is deprecated. + :type fqdn: str + :attr dmd: Root ZODB object + :type dmd: Products.ZenModel.DataRoot.DataRoot + :attr instance: The name of the Collection Hub. + :type instance: str + :attr callTime: The total time, in seconds, this service has spent processing remote requests. + :type callTime: float + + :attr listeners: ZenHub clients for this service + :type listeners: List[twisted.spread.pb.RemoteReference] + :attr listenerOptions: Options associated with the client for this service. + :type listenerOptions: Mapping[twisted.spread.pb.RemoteReference, Mapping[Any, Any]] + """ # noqa E501 + def __init__(self, dmd, instance): + """ + Initialize a HubService instance. + + :param dmd: The root Zenoss object in ZODB. + :type dmd: Products.ZenModel.DataRoot.DataRoot + :param instance: The name of the collection monitor. + :type instance: str + """ self.log = logging.getLogger("zen.hub") self.fqdn = socket.getfqdn() self.dmd = dmd self.zem = dmd.ZenEventManager self.instance = instance - self.listeners = [] + self.callTime = 0.0 + self.listeners = [] # Clients of this service self.listenerOptions = {} - self.callTime = 0 def getPerformanceMonitor(self): + """ + Return the performance monitor (collection hub) instance. + + :return type: Products.ZenModel.interfaces.IMonitor + """ return self.dmd.Monitors.getPerformanceMonitor(self.instance) def remoteMessageReceived(self, broker, message, args, kw): + """Overrides pb.Referenceable method.""" self.log.debug("Servicing %s in %s", message, self.name()) now = time.time() try: @@ -53,6 +87,11 @@ def deleted(self, object): pass def name(self): + """ + Return the name of this ZenHub service class. + + :return type: str + """ return self.__class__.__name__ def addListener(self, remote, options=None): @@ -63,15 +102,18 @@ def addListener(self, remote, options=None): self.listenerOptions[remote] = options def removeListener(self, listener): - self.log.debug( - "removing listener for %s:%s", self.instance, self.name() - ) - try: + if listener in self.listeners: self.listeners.remove(listener) - except ValueError: - self.warning("Unable to remove listener... ignoring") - - self.listenerOptions.pop(listener, None) + self.listenerOptions.pop(listener, None) + self.log.debug( + "removed listener for %s:%s", self.instance, self.name() + ) + else: + self.log.debug( + "listener is not registered for %s:%s", + self.instance, + self.name(), + ) def sendEvents(self, events): map(self.sendEvent, events) From 627f42598e07ecd0a71fbc11297e28e017aa12e3 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 20 Jul 2023 14:55:09 -0500 Subject: [PATCH 015/147] Rework docstrings, comments, and use of some zope APIs. --- Products/ZenCollector/daemon.py | 319 +++++++++++++++----------------- 1 file changed, 150 insertions(+), 169 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 2533ad3489..d66cebc2d4 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -15,13 +15,12 @@ from optparse import SUPPRESS_HELP -import zope.interface - from metrology import Metrology from metrology.instruments import Gauge from twisted.internet import defer, reactor, task from twisted.python.failure import Failure -from zope.component import getUtilitiesFor +from zope.component import getUtilitiesFor, provideUtility, queryUtility +from zope.interface import implementer from Products.ZenHub.PBDaemon import PBDaemon, FakeRemote from Products.ZenRRD.RRDDaemon import RRDDaemon @@ -48,29 +47,29 @@ log = logging.getLogger("zen.daemon") -@zope.interface.implementer(IConfigurationListener) +@implementer(IConfigurationListener) class DummyListener(object): + """ + No-op implementation of a listener that can be registered with instances + of ConfigListenerNotifier class. + """ + def deleted(self, configurationId): - """ - Called when a configuration is deleted from the collector - """ log.debug("DummyListener: configuration %s deleted", configurationId) def added(self, configuration): - """ - Called when a configuration is added to the collector - """ log.debug("DummyListener: configuration %s added", configuration) def updated(self, newConfiguration): - """ - Called when a configuration is updated in collector - """ log.debug("DummyListener: configuration %s updated", newConfiguration) -@zope.interface.implementer(IConfigurationListener) +@implementer(IConfigurationListener) class ConfigListenerNotifier(object): + """ + Registers other IConfigurationListener objects and notifies them when + this object is notified of configuration removals, adds, and updates. + """ _listeners = [] @@ -79,49 +78,61 @@ def addListener(self, listener): def deleted(self, configurationId): """ - Called when a configuration is deleted from the collector + Notify listener when a configuration is deleted. + + :param configurationId: The ID of the deleted configuration. + :type configurationId: str """ for listener in self._listeners: listener.deleted(configurationId) def added(self, configuration): """ - Called when a configuration is added to the collector + Notify the listeners when a configuration is added. + + :param configuration: The added configuration object. + :type configuration: DeviceProxy """ for listener in self._listeners: listener.added(configuration) def updated(self, newConfiguration): """ - Called when a configuration is updated in collector + Notify the listeners when a configuration has changed. + + :param newConfiguration: The updated configuration object. + :type newConfiguration: DeviceProxy """ for listener in self._listeners: listener.updated(newConfiguration) -@zope.interface.implementer(IConfigurationListener) +@implementer(IConfigurationListener) class DeviceGuidListener(object): + """ + Manages configuration IDs on the given 'daemon' object, making the + necessary changes when notified of configuration additions, removals, + and updates. + """ + def __init__(self, daemon): + """ + Initialize a DeviceGuidListener instance. + + :param daemon: The daemon object. + :type daemon: CollectorDaemon + """ self._daemon = daemon def deleted(self, configurationId): - """ - Called when a configuration is deleted from the collector - """ self._daemon._deviceGuids.pop(configurationId, None) def added(self, configuration): - """ - Called when a configuration is added to the collector - """ deviceGuid = getattr(configuration, "deviceGuid", None) if deviceGuid: self._daemon._deviceGuids[configuration.id] = deviceGuid def updated(self, newConfiguration): - """ - Called when a configuration is updated in collector - """ deviceGuid = getattr(newConfiguration, "deviceGuid", None) if deviceGuid: self._daemon._deviceGuids[newConfiguration.id] = deviceGuid @@ -131,20 +142,23 @@ def updated(self, newConfiguration): CONFIG_LOADER_NAME = "configLoader" -@zope.interface.implementer(ICollector, IDataService, IEventService) +@implementer(ICollector, IDataService, IEventService) class CollectorDaemon(RRDDaemon): - """ - The daemon class for the entire ZenCollector framework. This class bridges - the gap between the older daemon framework and ZenCollector. New collectors - no longer should extend this class to implement a new collector. - """ + """The daemon class for the entire ZenCollector framework.""" _frameworkFactoryName = "" + """ + Identifies the IFrameworkFactory implementation to use. + + :type: str + """ @property def preferences(self): """ - Preferences for this daemon + The preferences object of this daemon. + + :rtype: ICollectorPreferences """ return self._prefs @@ -157,58 +171,56 @@ def __init__( stoppingCallback=None, ): """ - Constructs a new instance of the CollectorDaemon framework. Normally - only a singleton instance of a CollectorDaemon should exist within a - process, but this is not enforced. - - @param preferences: the collector configuration - @type preferences: ICollectorPreferences - @param taskSplitter: the task splitter to use for this collector - @type taskSplitter: ITaskSplitter - @param initializationCallback: a callable that will be executed after - connection to the hub but before - retrieving configuration information - @type initializationCallback: any callable - @param stoppingCallback: a callable that will be executed first during - the stopping process. Exceptions will be - logged but otherwise ignored. - @type stoppingCallback: any callable - """ - # create the configuration first, so we have the collector name + Initializes a CollectorDaemon instance. + + :param preferences: the collector configuration + :type preferences: ICollectorPreferences + :param taskSplitter: the task splitter to use for this collector + :type taskSplitter: ITaskSplitter + :param configurationListener: A listener that can react to + notifications on configuration changes. + :type configurationListener: IConfigurationListener + :param initializationCallback: a callable that will be executed after + connection to the hub but before retrieving configuration + information. + :type initializationCallback: any callable, optional + :param stoppingCallback: a callable that will be executed first during + the stopping process. Exceptions will be logged but otherwise + ignored. + :type stoppingCallback: any callable, optional + """ + # Create the configuration first, so we have the collector name # available before activating the rest of the Daemon class hierarchy. if not ICollectorPreferences.providedBy(preferences): raise TypeError("configuration must provide ICollectorPreferences") - else: - self._prefs = ObservableProxy(preferences) - self._prefs.attachAttributeObserver( - "configCycleInterval", self._rescheduleConfig - ) - if not ITaskSplitter.providedBy(taskSplitter): raise TypeError("taskSplitter must provide ITaskSplitter") - else: - self._taskSplitter = taskSplitter - if not IConfigurationListener.providedBy(configurationListener): raise TypeError( "configurationListener must provide IConfigurationListener" ) + + self._prefs = ObservableProxy(preferences) + self._prefs.attachAttributeObserver( + "configCycleInterval", self._rescheduleConfig + ) + self._taskSplitter = taskSplitter self._configListener = ConfigListenerNotifier() self._configListener.addListener(configurationListener) self._configListener.addListener(DeviceGuidListener(self)) self._initializationCallback = initializationCallback self._stoppingCallback = stoppingCallback - # register the various interfaces we provide the rest of the system so + # Register the various interfaces we provide the rest of the system so # that collector implementors can easily retrieve a reference back here # if needed - zope.component.provideUtility(self, ICollector) - zope.component.provideUtility(self, IEventService) - zope.component.provideUtility(self, IDataService) + provideUtility(self, ICollector) + provideUtility(self, IEventService) + provideUtility(self, IDataService) - # register the collector's own preferences object so it may be easily + # Register the collector's own preferences object so it may be easily # retrieved by factories, tasks, etc. - zope.component.provideUtility( + provideUtility( self.preferences, ICollectorPreferences, self.preferences.collectorName, @@ -218,7 +230,7 @@ def __init__( name=self.preferences.collectorName ) self._statService = StatisticsService() - zope.component.provideUtility(self._statService, IStatisticsService) + provideUtility(self._statService, IStatisticsService) if self.options.cycle: # setup daemon statistics (deprecated names) @@ -280,13 +292,13 @@ def value(self): self._derivative_tracker = None self.reconfigureTimeout = None - # keep track of pending tasks if we're doing a single run, and not a + # Keep track of pending tasks if we're doing a single run, and not a # continuous cycle if not self.options.cycle: self._completedTasks = 0 self._pendingTasks = [] - frameworkFactory = zope.component.queryUtility( + frameworkFactory = queryUtility( IFrameworkFactory, self._frameworkFactoryName ) self._configProxy = frameworkFactory.getConfigurationProxy() @@ -296,35 +308,32 @@ def value(self): frameworkFactory.getConfigurationLoaderTask() ) - # OLD - set the initialServices attribute so that the PBDaemon class + # Set the initialServices attribute so that the PBDaemon class # will load all of the remote services we need. self.initialServices = PBDaemon.initialServices + [ self.preferences.configurationService ] - # trap SIGUSR2 so that we can display detailed statistics + # Trap SIGUSR2 so that we can display detailed statistics signal.signal(signal.SIGUSR2, self._signalHandler) - # let the configuration do any additional startup it might need + # Let the configuration do any additional startup it might need self.preferences.postStartup() self.addedPostStartupTasks = False # Variables used by enterprise collector in resmgr # # Flag that indicates we have finished loading the configs for the - # first time after a restart. + # first time after a restart self.firstConfigLoadDone = False # Flag that indicates the daemon has received the encryption key - # from zenhub. + # from zenhub self.encryptionKeyInitialized = False - # flag that indicates the daemon is loading the cached configs + # Flag that indicates the daemon is loading the cached configs self.loadingCachedConfigs = False def buildOptions(self): - """ - Method called by CmdBase.__init__ to build all of the possible - command-line options for this collector daemon. - """ + """Overrides base class to add additional configuration options.""" super(CollectorDaemon, self).buildOptions() maxTasks = getattr(self.preferences, "maxTasks", None) @@ -361,7 +370,7 @@ def buildOptions(self): help="trace metrics whose key value matches this regex", ) - frameworkFactory = zope.component.queryUtility( + frameworkFactory = queryUtility( IFrameworkFactory, self._frameworkFactoryName ) if hasattr(frameworkFactory, "getFrameworkBuildOptions"): @@ -376,6 +385,7 @@ def buildOptions(self): self.preferences.buildOptions(self.parser) def parseOptions(self): + """Overrides base class to process configuration options.""" super(CollectorDaemon, self).parseOptions() self.preferences.options = self.options @@ -384,21 +394,11 @@ def parseOptions(self): self.preferences.configFilter = configFilter def connected(self): - """ - Method called by PBDaemon after a connection to ZenHub is established. - """ + """Invoked after a connection to ZenHub is established.""" return self._startup() - def _getInitializationCallback(self): - def doNothing(): - pass - - if self._initializationCallback is not None: - return self._initializationCallback - else: - return doNothing - def connectTimeout(self): + """Invoked after timing out while connecting to ZenHub.""" super(CollectorDaemon, self).connectTimeout() return self._startup() @@ -410,13 +410,17 @@ def _startup(self): d.addErrback(self._errorStop) return d + def _getInitializationCallback(self): + if self._initializationCallback is not None: + return self._initializationCallback + return lambda: None + @defer.inlineCallbacks def _initEncryptionKey(self, prv_cb_result=None): - # encrypt dummy msg in order to initialize the encryption key - data = yield self._configProxy.encrypt( - "Hello" - ) # block until we get the key - if data: # encrypt returns None if an exception is raised + # Encrypt dummy msg in order to initialize the encryption key. + # The 'yield' does not return until the key is initialized. + data = yield self._configProxy.encrypt("Hello") + if data: # Encrypt returns None if an exception is raised self.encryptionKeyInitialized = True self.log.info("Daemon's encryption key initialized") @@ -424,15 +428,13 @@ def watchdogCycleTime(self): """ Return our cycle time (in minutes) - @return: cycle time - @rtype: integer + :return: cycle time + :rtype: integer """ return self.preferences.cycleInterval * 2 def getRemoteConfigServiceProxy(self): - """ - Called to retrieve the remote configuration service proxy object. - """ + """Return the remote configuration service proxy object.""" return self.services.get( self.preferences.configurationService, FakeRemote() ) @@ -450,7 +452,9 @@ def should_trace_metric(self, metric, contextkey): """ Tracer implementation - use this function to indicate whether a given metric/contextkey combination is to be traced. + :param metric: name of the metric in question + :type metric: str :param contextkey: context key of the metric in question :return: boolean indicating whether to trace this metric/key """ @@ -459,9 +463,7 @@ def should_trace_metric(self, metric, contextkey): tests.append((self.options.traceMetricName, metric)) if self.options.traceMetricKey: tests.append((self.options.traceMetricKey, contextkey)) - result = [bool(re.search(exp, subj)) for exp, subj in tests] - return len(result) > 0 and all(result) @defer.inlineCallbacks @@ -480,26 +482,25 @@ def writeMetric( contextUUID=None, deviceUUID=None, ): - """ Writes the metric to the metric publisher. - @param contextKey: This is who the metric applies to. This is usually - the return value of rrdPath() for a component or - device. - @param metric: the name of the metric, we expect it to be of the form - datasource_datapoint - @param value: the value of the metric - @param metricType: type of the metric (e.g. 'COUNTER', 'GAUGE', + + :param contextKey: This is who the metric applies to. This is usually + the return value of rrdPath() for a component or device. + :param metric: the name of the metric, we expect it to be of the form + datasource_datapoint. + :param value: the value of the metric. + :param metricType: type of the metric (e.g. 'COUNTER', 'GAUGE', 'DERIVE' etc) - @param contextId: used for the threshold events, the id of who - this metric is for. - @param timestamp: defaults to time.time() if not specified, + :param contextId: used for the threshold events, the id of who this + metric is for. + :param timestamp: defaults to time.time() if not specified, the time the metric occurred. - @param min: used in the derive the min value for the metric - @param max: used in the derive the max value for the metric - @param threshEventData: extra data put into threshold events - @param deviceId: the id of the device for this metric - @return: a deferred that fires when the metric gets published + :param min: used in the derive the min value for the metric. + :param max: used in the derive the max value for the metric. + :param threshEventData: extra data put into threshold events. + :param deviceId: the id of the device for this metric. + :return: a deferred that fires when the metric gets published. """ timestamp = int(time.time()) if timestamp == "N" else timestamp tags = {"contextUUID": contextUUID, "key": contextKey} @@ -555,7 +556,6 @@ def writeMetricWithMetadata( threshEventData={}, metadata=None, ): - metadata = metadata or {} try: key = metadata["contextKey"] @@ -598,11 +598,9 @@ def writeRRD( timestamp="N", allowStaleDatapoint=True, ): - """ - Use writeMetric - """ - # We rely on the fact that rrdPath now returns more information - # than just the path. + """Use writeMetric instead.""" + # We rely on the fact that rrdPath now returns more information than + # just the path metricinfo, metric = path.rsplit("/", 1) if "METRIC_DATA" not in str(metricinfo): raise Exception( @@ -632,18 +630,14 @@ def stop(self, ignored=""): super(CollectorDaemon, self).stop(ignored) def remote_deleteDevice(self, devId): - """ - Called remotely by ZenHub when a device we're monitoring is deleted. - """ + """Remote method invoked by ZenHub when a device is deleted.""" # guard against parsing updates during a disconnect if devId is None: return self._deleteDevice(devId) def remote_deleteDevices(self, deviceIds): - """ - Called remotely by ZenHub when devices we're monitoring are deleted. - """ + """Remote method invoked by ZenHub when many devices are deleted.""" # guard against parsing updates during a disconnect if deviceIds is None: return @@ -651,10 +645,7 @@ def remote_deleteDevices(self, deviceIds): self._deleteDevice(devId) def remote_updateDeviceConfig(self, config): - """ - Called remotely by ZenHub when asynchronous configuration - updates occur. - """ + """Remote method invoked by ZenHub when a device config is updated.""" # guard against parsing updates during a disconnect if config is None: return @@ -666,8 +657,7 @@ def remote_updateDeviceConfig(self, config): def remote_updateDeviceConfigs(self, configs): """ - Called remotely by ZenHub when asynchronous configuration - updates occur. + Remote method invoked by ZenHub for multiple device config updates. """ if configs is None: return @@ -683,7 +673,8 @@ def remote_updateDeviceConfigs(self, configs): def remote_notifyConfigChanged(self): """ - Called from zenhub to notify that the entire config should be updated + Remote method invoked by ZenHub when the all the device configs + should be replaced. """ if self.reconfigureTimeout and self.reconfigureTimeout.active(): # We will run along with the already scheduled task @@ -739,7 +730,7 @@ def _updateConfig(self, cfg): """ Update device configuration. - Returns true if config is updated, false if config is skipped + Return true if config is updated, false if config is skipped. """ # guard against parsing updates during a disconnect @@ -817,10 +808,10 @@ def _updateConfig(self, cfg): @defer.inlineCallbacks def _updateDeviceConfigs(self, updatedConfigs, purgeOmitted): """ - Update the device configurations for the devices managed by this - collector. - @param deviceConfigs a list of device configurations - @type deviceConfigs list of name,value tuples + Update the device configs for the devices this collector manages. + + :param deviceConfigs: a list of device configurations + :type deviceConfigs: list of name,value tuples """ self.log.debug( "updateDeviceConfigs: updatedConfigs=%s", @@ -837,11 +828,11 @@ def _updateDeviceConfigs(self, updatedConfigs, purgeOmitted): def _purgeOmittedDevices(self, updatedDevices): """ - Delete all current devices that are omitted from the list of - devices being updated. + Delete all current devices that are omitted from the list of devices + being updated. - @param updatedDevices a collection of device ids - @type updatedDevices a sequence of strings + :param updatedDevices: a collection of device ids + :type updatedDevices: a sequence of strings """ # remove tasks for the deleted devices deletedDevices = set(self._devices) - set(updatedDevices) @@ -863,8 +854,8 @@ def _errorStop(self, result): """ Twisted callback to receive fatal messages. - @param result: the Twisted failure - @type result: failure object + :param result: the Twisted failure + :type result: failure object """ if isinstance(result, Failure): msg = result.getErrorMessage() @@ -889,20 +880,13 @@ def _startConfigCycle(self, result=None, startDelay=0): return defer.succeed("Configuration loader task started") def setPropertyItems(self, items): - """ - Override so that preferences are updated - """ + """Override so that preferences are updated.""" super(CollectorDaemon, self).setPropertyItems(items) self._setCollectorPreferences(dict(items)) def _setCollectorPreferences(self, preferenceItems): for name, value in preferenceItems.iteritems(): if not hasattr(self.preferences, name): - # TODO: make a super-low level debug mode? - # The following message isn't helpful - # self.log.debug( - # "Preferences object does not have attribute %s", name - # ) setattr(self.preferences, name, value) elif getattr(self.preferences, name) != value: self.log.debug("Updated %s preference to %s", name, value) @@ -940,8 +924,9 @@ def _startMaintenance(self, ignored=None): @defer.inlineCallbacks def _maintenanceCycle(self, ignored=None): """ - Perform daemon maintenance processing on a periodic schedule. Initially - called after the daemon configuration loader task is added, + Perform daemon maintenance processing on a periodic schedule. + + Initially called after the daemon configuration loader task is added, but afterward will self-schedule each run. """ try: @@ -1053,20 +1038,16 @@ def _signalHandler(self, signum, frame): @property def worker_count(self): - """ - worker_count for this daemon - """ + """The count of service instances.""" return getattr(self.options, "workers", 1) @property def worker_id(self): - """ - worker_id for this particular peer - """ + """The ID of this particular service instance.""" return getattr(self.options, "workerid", 0) -@zope.interface.implementer(IStatistic) +@implementer(IStatistic) class Statistic(object): def __init__(self, name, type, **kwargs): self.value = 0 @@ -1075,7 +1056,7 @@ def __init__(self, name, type, **kwargs): self.kwargs = kwargs -@zope.interface.implementer(IStatisticsService) +@implementer(IStatisticsService) class StatisticsService(object): def __init__(self): self._stats = {} From 62426899defc42d069f3506f6149c4baa6273349 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 24 Jul 2023 10:17:03 -0500 Subject: [PATCH 016/147] Minor refactoring of invalidation releated code. --- Products/ZenHub/invalidationfilter.py | 78 +++--- Products/ZenHub/invalidationmanager.py | 40 +-- Products/ZenHub/invalidationoid.py | 13 +- .../ZenHub/tests/test_invalidationfilter.py | 253 +++++++++--------- .../ZenHub/tests/test_invalidationmanager.py | 14 +- Products/ZenHub/tests/test_zenhub.py | 4 - Products/ZenHub/zenhub.py | 6 - .../ZenRelations/PrimaryPathObjectManager.py | 4 +- Products/ZenRelations/__init__.py | 64 ++--- 9 files changed, 235 insertions(+), 241 deletions(-) diff --git a/Products/ZenHub/invalidationfilter.py b/Products/ZenHub/invalidationfilter.py index 26f4b552a9..a831b68e86 100644 --- a/Products/ZenHub/invalidationfilter.py +++ b/Products/ZenHub/invalidationfilter.py @@ -13,7 +13,7 @@ from cStringIO import StringIO from hashlib import md5 -from zope.interface import implements +from zope.interface import implementer from Products.ZenModel.DeviceClass import DeviceClass from Products.ZenModel.GraphDefinition import GraphDefinition @@ -29,16 +29,12 @@ from .interfaces import IInvalidationFilter, FILTER_EXCLUDE, FILTER_CONTINUE -log = logging.getLogger("zen.InvalidationFilter") +log = logging.getLogger("zen.{}".format(__name__.split(".")[-1].lower())) +@implementer(IInvalidationFilter) class IgnorableClassesFilter(object): - """ - This filter specifies which classes we want to ignore the - invalidations on. - """ - - implements(IInvalidationFilter) + """Ignore invalidations on certain classes.""" CLASSES_TO_IGNORE = ( IpAddress, @@ -60,16 +56,36 @@ def include(self, obj): return FILTER_CONTINUE +_iszorcustprop = re.compile("[zc][A-Z]").match + + +def _getZorCProperties(organizer): + for zId in sorted(organizer.zenPropertyIds(pfilt=_iszorcustprop)): + try: + if organizer.zenPropIsPassword(zId): + propertyString = organizer.getProperty(zId, "") + else: + propertyString = organizer.zenPropertyString(zId) + yield zId, propertyString + except AttributeError: + # ZEN-3666: If an attribute error is raised on a zProperty + # assume it was produced by a zenpack + # install whose daemons haven't been restarted and continue + # excluding the offending property. + log.debug("Excluding '%s' property", zId) + + +@implementer(IInvalidationFilter) class BaseOrganizerFilter(object): """ - Base invalidation filter for organizers. Calculates a checksum for - the organizer based on its sorted z/c properties. - """ + Base invalidation filter for organizers. - implements(IInvalidationFilter) + The default implementation will reject organizers that do not have + updated calculated checksum values. The checksum is calculated using + accumulation of each 'z' and 'c' property associated with organizer. + """ weight = 10 - iszorcustprop = re.compile("[zc][A-Z]").match def __init__(self, types): self._types = types @@ -89,24 +105,9 @@ def initialize(self, context): log.warn("Unable to retrieve object: %s", brain.getPath()) self.checksum_map = results - def getZorCProperties(self, organizer): - for zId in sorted(organizer.zenPropertyIds(pfilt=self.iszorcustprop)): - try: - if organizer.zenPropIsPassword(zId): - propertyString = organizer.getProperty(zId, "") - else: - propertyString = organizer.zenPropertyString(zId) - yield zId, propertyString - except AttributeError: - # ZEN-3666: If an attribute error is raised on a zProperty - # assume it was produced by a zenpack - # install whose daemons haven't been restarted and continue - # excluding the offending property. - log.debug("Excluding '%s' property", zId) - def generateChecksum(self, organizer, md5_checksum): # Checksum all zProperties and custom properties - for zId, propertyString in self.getZorCProperties(organizer): + for zId, propertyString in _getZorCProperties(organizer): md5_checksum.update("%s|%s" % (zId, propertyString)) def organizerChecksum(self, organizer): @@ -135,9 +136,10 @@ def include(self, obj): class DeviceClassInvalidationFilter(BaseOrganizerFilter): """ - Subclass of BaseOrganizerFilter with specific logic for - Device classes. Uses both z/c properties as well as locally - bound RRD templates to create the checksum. + Invalidation filter for DeviceClass organizers. + + Uses both 'z' and 'c' properties as well as locally bound RRD templates + to create the checksum. """ def __init__(self): @@ -167,10 +169,7 @@ def generateChecksum(self, organizer, md5_checksum): class OSProcessOrganizerFilter(BaseOrganizerFilter): - """ - Invalidation filter for OSProcessOrganizer objects. This filter only - looks at z/c properties defined on the organizer. - """ + """Invalidation filter for OSProcessOrganizer objects.""" def __init__(self): super(OSProcessOrganizerFilter, self).__init__((OSProcessOrganizer,)) @@ -181,9 +180,10 @@ def getRoot(self, context): class OSProcessClassFilter(BaseOrganizerFilter): """ - Invalidation filter for OSProcessClass objects. This filter uses - z/c properties as well as local _properties defined on the organizer - to create a checksum. + Invalidation filter for OSProcessClass objects. + + This filter uses 'z' and 'c' properties as well as local _properties + defined on the organizer to create a checksum. """ def __init__(self): diff --git a/Products/ZenHub/invalidationmanager.py b/Products/ZenHub/invalidationmanager.py index 9e57a8dc21..d320edb022 100644 --- a/Products/ZenHub/invalidationmanager.py +++ b/Products/ZenHub/invalidationmanager.py @@ -36,7 +36,7 @@ ) from .invalidations import INVALIDATIONS_PAUSED -log = logging.getLogger("zen.ZenHub.invalidationmanager") +log = logging.getLogger("zen.{}".format(__name__.split(".")[-1].lower())) class InvalidationManager(object): @@ -76,7 +76,9 @@ def __init__( self.totalEvents = 0 self.totalTime = 0 - self.initialize_invalidation_filters() + self._invalidation_filters = self.initialize_invalidation_filters( + self.__dmd + ) self.processor = getUtility(IInvalidationProcessor) log.debug("got InvalidationProcessor %s", self.processor) app = self.__dmd.getPhysicalRoot() @@ -84,33 +86,39 @@ def __init__( app, self._invalidation_filters, self._queue ) - def initialize_invalidation_filters(self): - """Get Invalidation Filters, initialize them, - store them in the _invalidation_filters list, and return the list + @staticmethod + def initialize_invalidation_filters(ctx): + """ + Return initialized IInvalidationFilter objects in a list. + + :param ctx: Used to initialize the IInvalidationFilter objects. + :type ctx: DataRoot + :return: Initialized IInvalidationFilter objects + :rtype: List[IInvalidationFilter] """ try: filters = (f for n, f in getUtilitiesFor(IInvalidationFilter)) - self._invalidation_filters = [] + invalidation_filters = [] for fltr in sorted( filters, key=lambda f: getattr(f, "weight", 100) ): - fltr.initialize(self.__dmd) - self._invalidation_filters.append(fltr) - self.log.info( + fltr.initialize(ctx) + invalidation_filters.append(fltr) + log.info( "Registered %s invalidation filters.", - len(self._invalidation_filters), + len(invalidation_filters), ) - self.log.info( - "invalidation filters: %s", self._invalidation_filters - ) - return self._invalidation_filters + log.info("invalidation filters: %s", invalidation_filters) + return invalidation_filters except Exception: log.exception("error in initialize_invalidation_filters") @inlineCallbacks def process_invalidations(self): - """Periodically process database changes. - synchronize with the database, and poll invalidated oids from it, + """ + Periodically process database changes. + + Synchronize with the database, and poll invalidated oids from it, filter the oids, send them to the invalidation_processor @return: None diff --git a/Products/ZenHub/invalidationoid.py b/Products/ZenHub/invalidationoid.py index 0d88b916f9..dbcedaf2e3 100644 --- a/Products/ZenHub/invalidationoid.py +++ b/Products/ZenHub/invalidationoid.py @@ -9,8 +9,8 @@ import logging -from zope.interface import implements -from zope.component import adapts +from zope.interface import implementer +from zope.component import adapter from Products.ZenRelations.PrimaryPathObjectManager import ( PrimaryPathObjectManager, @@ -18,13 +18,12 @@ from .interfaces import IInvalidationOid - -log = logging.getLogger("zen.InvalidationOid") +log = logging.getLogger("zen.{}".format(__name__.split(".")[-1].lower())) +@adapter(PrimaryPathObjectManager) +@implementer(IInvalidationOid) class DefaultOidTransform(object): - implements(IInvalidationOid) - adapts(PrimaryPathObjectManager) def __init__(self, obj): self._obj = obj @@ -33,8 +32,8 @@ def transformOid(self, oid): return oid +@implementer(IInvalidationOid) class DeviceOidTransform(object): - implements(IInvalidationOid) def __init__(self, obj): self._obj = obj diff --git a/Products/ZenHub/tests/test_invalidationfilter.py b/Products/ZenHub/tests/test_invalidationfilter.py index 767f002d9b..7f4ac561c2 100644 --- a/Products/ZenHub/tests/test_invalidationfilter.py +++ b/Products/ZenHub/tests/test_invalidationfilter.py @@ -4,6 +4,8 @@ from zope.interface.verify import verifyObject from Products.ZenHub.invalidationfilter import ( + _getZorCProperties, + _iszorcustprop, BaseOrganizerFilter, DeviceClass, DeviceClassInvalidationFilter, @@ -25,40 +27,40 @@ class IgnorableClassesFilterTest(TestCase): - def setUp(self): - self.icf = IgnorableClassesFilter() + def setUp(t): + t.icf = IgnorableClassesFilter() - def test_init(self): - IInvalidationFilter.providedBy(self.icf) + def test_init(t): + IInvalidationFilter.providedBy(t.icf) # current version fails because weight attribute is not defined # icf.weight = 1 # verifyObject(IInvalidationFilter, icf) - self.assertTrue(hasattr(self.icf, "CLASSES_TO_IGNORE")) + t.assertTrue(hasattr(t.icf, "CLASSES_TO_IGNORE")) - def test_initialize(self): + def test_initialize(t): context = Mock(name="context") - self.icf.initialize(context) + t.icf.initialize(context) # No return or side-effects - def test_include(self): + def test_include(t): obj = Mock(name="object") - out = self.icf.include(obj) - self.assertEqual(out, FILTER_CONTINUE) + out = t.icf.include(obj) + t.assertEqual(out, FILTER_CONTINUE) - def test_include_excludes_classes_to_ignore(self): - self.icf.CLASSES_TO_IGNORE = str - out = self.icf.include("ignore me!") - self.assertEqual(out, FILTER_EXCLUDE) + def test_include_excludes_classes_to_ignore(t): + t.icf.CLASSES_TO_IGNORE = str + out = t.icf.include("ignore me!") + t.assertEqual(out, FILTER_EXCLUDE) class BaseOrganizerFilterTest(TestCase): - def setUp(self): - self.types = Mock(name="types") - self.bof = BaseOrganizerFilter(self.types) + def setUp(t): + t.types = Mock(name="types") + t.bof = BaseOrganizerFilter(t.types) # @patch with autospec fails (https://bugs.python.org/issue23078) # manually spec ZenPropertyManager - self.organizer = Mock( + t.organizer = Mock( name="Products.ZenRelations.ZenPropertyManager", spec_set=[ "zenPropertyIds", @@ -68,148 +70,157 @@ def setUp(self): ], ) - def test_init(self): - IInvalidationFilter.providedBy(self.bof) - verifyObject(IInvalidationFilter, self.bof) - self.assertEqual(self.bof.weight, 10) - self.assertEqual(self.bof._types, self.types) - - def test_iszorcustprop(self): - match = self.bof.iszorcustprop("no match") - self.assertEqual(match, None) - match = self.bof.iszorcustprop("cProperty") - self.assertTrue(match) - match = self.bof.iszorcustprop("zProperty") - self.assertTrue(match) - - def test_getRoot(self): + def test_init(t): + IInvalidationFilter.providedBy(t.bof) + verifyObject(IInvalidationFilter, t.bof) + t.assertEqual(t.bof.weight, 10) + t.assertEqual(t.bof._types, t.types) + + def test_iszorcustprop(t): + result = _iszorcustprop("no match") + t.assertEqual(result, None) + result = _iszorcustprop("cProperty") + t.assertTrue(result) + result = _iszorcustprop("zProperty") + t.assertTrue(result) + + def test_getRoot(t): context = Mock(name="context") - root = self.bof.getRoot(context) - self.assertEqual(root, context.dmd.primaryAq()) + root = t.bof.getRoot(context) + t.assertEqual(root, context.dmd.primaryAq()) @patch( "{invalidationfilter}.IModelCatalogTool".format(**PATH), autospec=True, spec_set=True, ) - def test_initialize(self, IModelCatalogTool): + def test_initialize(t, IModelCatalogTool): # Create a Mock object that provides the ICatalogBrain interface ICatalogBrainMock = create_interface_mock(ICatalogBrain) brain = ICatalogBrainMock() IModelCatalogTool.return_value.search.return_value = [brain] - checksum = create_autospec(self.bof.organizerChecksum) - self.bof.organizerChecksum = checksum + checksum = create_autospec(t.bof.organizerChecksum) + t.bof.organizerChecksum = checksum context = Mock(name="context") - self.bof.initialize(context) + t.bof.initialize(context) - self.assertEqual( - self.bof.checksum_map, + t.assertEqual( + t.bof.checksum_map, {brain.getPath.return_value: checksum.return_value}, ) - def test_getZorCProperties(self): + def test_getZorCProperties(t): zprop = Mock(name="zenPropertyId", spec_set=[]) - self.organizer.zenPropertyIds.return_value = [zprop, zprop] + t.organizer.zenPropertyIds.return_value = [zprop, zprop] # getZorCProperties returns a generator - results = self.bof.getZorCProperties(self.organizer) + results = _getZorCProperties(t.organizer) - self.organizer.zenPropIsPassword.return_value = False + t.organizer.zenPropIsPassword.return_value = False zId, propertyString = next(results) - self.assertEqual(zId, zprop) - self.assertEqual( - propertyString, self.organizer.zenPropertyString.return_value + t.assertEqual(zId, zprop) + t.assertEqual( + propertyString, t.organizer.zenPropertyString.return_value ) - self.organizer.zenPropertyString.assert_called_with(zprop) + t.organizer.zenPropertyString.assert_called_with(zprop) - self.organizer.zenPropIsPassword.return_value = True + t.organizer.zenPropIsPassword.return_value = True zId, propertyString = next(results) - self.assertEqual(zId, zprop) - self.assertEqual( - propertyString, self.organizer.getProperty.return_value + t.assertEqual(zId, zprop) + t.assertEqual( + propertyString, t.organizer.getProperty.return_value ) - self.organizer.getProperty.assert_called_with(zprop, "") + t.organizer.getProperty.assert_called_with(zprop, "") - with self.assertRaises(StopIteration): + with t.assertRaises(StopIteration): next(results) - def test_generateChecksum(self): - getZorCProperties = create_autospec(self.bof.getZorCProperties) + @patch( + "{invalidationfilter}._getZorCProperties".format(**PATH), + autospec=True, + spec_set=True, + ) + def test_generateChecksum(t, _getZorCProps): zprop = Mock(name="zenPropertyId", spec_set=[]) - getZorCProperties.return_value = [(zprop, "property_string")] - self.bof.getZorCProperties = getZorCProperties - md5_checksum = md5() - - self.bof.generateChecksum(self.organizer, md5_checksum) + data = (zprop, "property_string") + _getZorCProps.return_value = [data] + actual = md5() expect = md5() - expect.update("%s|%s" % (getZorCProperties(self.organizer)[0])) - getZorCProperties.assert_called_with(self.organizer) - self.assertEqual(md5_checksum.hexdigest(), expect.hexdigest()) + expect.update("%s|%s" % data) - def test_organizerChecksum(self): - getZorCProperties = create_autospec(self.bof.getZorCProperties) + t.bof.generateChecksum(t.organizer, actual) + + _getZorCProps.assert_called_with(t.organizer) + t.assertEqual(actual.hexdigest(), expect.hexdigest()) + + @patch( + "{invalidationfilter}._getZorCProperties".format(**PATH), + autospec=True, + spec_set=True, + ) + def test_organizerChecksum(t, _getZorCProps): zprop = Mock(name="zenPropertyId", spec_set=[]) - getZorCProperties.return_value = [(zprop, "property_string")] - self.bof.getZorCProperties = getZorCProperties + data = (zprop, "property_string") + _getZorCProps.return_value = [data] - out = self.bof.organizerChecksum(self.organizer) + out = t.bof.organizerChecksum(t.organizer) expect = md5() - expect.update("%s|%s" % (getZorCProperties(self.organizer)[0])) - self.assertEqual(out, expect.hexdigest()) - - def test_include_ignores_non_matching_types(self): - self.bof._types = (str,) - ret = self.bof.include(False) - self.assertEqual(ret, FILTER_CONTINUE) - - def test_include_if_checksum_changed(self): - organizerChecksum = create_autospec(self.bof.organizerChecksum) - self.bof.organizerChecksum = organizerChecksum - self.bof._types = (Mock,) + expect.update("%s|%s" % data) + t.assertEqual(out, expect.hexdigest()) + + def test_include_ignores_non_matching_types(t): + t.bof._types = (str,) + ret = t.bof.include(False) + t.assertEqual(ret, FILTER_CONTINUE) + + def test_include_if_checksum_changed(t): + organizerChecksum = create_autospec(t.bof.organizerChecksum) + t.bof.organizerChecksum = organizerChecksum + t.bof._types = (Mock,) obj = Mock(name="object", spec_set=["getPrimaryPath"]) obj.getPrimaryPath.return_value = ["dmd", "brain"] organizer_path = "/".join(obj.getPrimaryPath()) - self.bof.checksum_map = {organizer_path: "existing_checksum"} + t.bof.checksum_map = {organizer_path: "existing_checksum"} organizerChecksum.return_value = "current_checksum" - ret = self.bof.include(obj) + ret = t.bof.include(obj) - self.assertEqual(ret, FILTER_CONTINUE) + t.assertEqual(ret, FILTER_CONTINUE) - def test_include_if_checksum_unchanged(self): - organizerChecksum = create_autospec(self.bof.organizerChecksum) - self.bof.organizerChecksum = organizerChecksum + def test_include_if_checksum_unchanged(t): + organizerChecksum = create_autospec(t.bof.organizerChecksum) + t.bof.organizerChecksum = organizerChecksum existing_checksum = "checksum" current_checksum = "checksum" organizerChecksum.return_value = current_checksum - self.bof._types = (Mock,) + t.bof._types = (Mock,) obj = Mock(name="object", spec_set=["getPrimaryPath"]) obj.getPrimaryPath.return_value = ["dmd", "brain"] organizer_path = "/".join(obj.getPrimaryPath()) - self.bof.checksum_map = {organizer_path: existing_checksum} + t.bof.checksum_map = {organizer_path: existing_checksum} - ret = self.bof.include(obj) + ret = t.bof.include(obj) - self.assertEqual(ret, FILTER_EXCLUDE) + t.assertEqual(ret, FILTER_EXCLUDE) class DeviceClassInvalidationFilterTest(TestCase): - def setUp(self): - self.dcif = DeviceClassInvalidationFilter() + def setUp(t): + t.dcif = DeviceClassInvalidationFilter() - def test_init(self): - IInvalidationFilter.providedBy(self.dcif) - verifyObject(IInvalidationFilter, self.dcif) - self.assertEqual(self.dcif._types, (DeviceClass,)) + def test_init(t): + IInvalidationFilter.providedBy(t.dcif) + verifyObject(IInvalidationFilter, t.dcif) + t.assertEqual(t.dcif._types, (DeviceClass,)) - def test_getRoot(self): + def test_getRoot(t): context = Mock(name="context") - root = self.dcif.getRoot(context) - self.assertEqual(root, context.dmd.Devices.primaryAq()) + root = t.dcif.getRoot(context) + t.assertEqual(root, context.dmd.Devices.primaryAq()) @patch( "{invalidationfilter}.BaseOrganizerFilter.generateChecksum".format( @@ -218,7 +229,7 @@ def test_getRoot(self): autospec=True, spec_set=True, ) - def test_generateChecksum(self, super_generateChecksum): + def test_generateChecksum(t, super_generateChecksum): md5_checksum = md5() organizer = Mock( name="Products.ZenRelations.ZenPropertyManager", @@ -228,44 +239,44 @@ def test_generateChecksum(self, super_generateChecksum): rrdTemplate.exportXml.return_value = "some exemel" organizer.rrdTemplates.return_value = [rrdTemplate] - self.dcif.generateChecksum(organizer, md5_checksum) + t.dcif.generateChecksum(organizer, md5_checksum) # We cannot validate the output of the current version, refactor needed rrdTemplate.exportXml.was_called_once() super_generateChecksum.assert_called_with( - self.dcif, organizer, md5_checksum + t.dcif, organizer, md5_checksum ) class OSProcessOrganizerFilterTest(TestCase): - def test_init(self): + def test_init(t): ospof = OSProcessOrganizerFilter() IInvalidationFilter.providedBy(ospof) verifyObject(IInvalidationFilter, ospof) - self.assertEqual(ospof._types, (OSProcessOrganizer,)) + t.assertEqual(ospof._types, (OSProcessOrganizer,)) - def test_getRoot(self): + def test_getRoot(t): ospof = OSProcessOrganizerFilter() context = Mock(name="context") root = ospof.getRoot(context) - self.assertEqual(root, context.dmd.Processes.primaryAq()) + t.assertEqual(root, context.dmd.Processes.primaryAq()) class OSProcessClassFilterTest(TestCase): - def setUp(self): - self.ospcf = OSProcessClassFilter() + def setUp(t): + t.ospcf = OSProcessClassFilter() - def test_init(self): - IInvalidationFilter.providedBy(self.ospcf) - verifyObject(IInvalidationFilter, self.ospcf) + def test_init(t): + IInvalidationFilter.providedBy(t.ospcf) + verifyObject(IInvalidationFilter, t.ospcf) - self.assertEqual(self.ospcf._types, (OSProcessClass,)) + t.assertEqual(t.ospcf._types, (OSProcessClass,)) - def test_getRoot(self): + def test_getRoot(t): context = Mock(name="context") - root = self.ospcf.getRoot(context) - self.assertEqual(root, context.dmd.Processes.primaryAq()) + root = t.ospcf.getRoot(context) + t.assertEqual(root, context.dmd.Processes.primaryAq()) @patch( "{invalidationfilter}.BaseOrganizerFilter.generateChecksum".format( @@ -274,7 +285,7 @@ def test_getRoot(self): autospec=True, spec_set=True, ) - def test_generateChecksum(self, super_generateChecksum): + def test_generateChecksum(t, super_generateChecksum): organizer = Mock( name="Products.ZenRelations.ZenPropertyManager", spec_set=["property_id", "_properties"], @@ -284,11 +295,11 @@ def test_generateChecksum(self, super_generateChecksum): organizer.property_id = "value" md5_checksum = md5() - self.ospcf.generateChecksum(organizer, md5_checksum) + t.ospcf.generateChecksum(organizer, md5_checksum) expect = md5() expect.update("%s|%s" % (prop["id"], getattr(organizer, prop["id"]))) - self.assertEqual(md5_checksum.hexdigest(), expect.hexdigest()) + t.assertEqual(md5_checksum.hexdigest(), expect.hexdigest()) super_generateChecksum.assert_called_with( - self.ospcf, organizer, md5_checksum + t.ospcf, organizer, md5_checksum ) diff --git a/Products/ZenHub/tests/test_invalidationmanager.py b/Products/ZenHub/tests/test_invalidationmanager.py index fb2cd7460a..89b756dbf9 100644 --- a/Products/ZenHub/tests/test_invalidationmanager.py +++ b/Products/ZenHub/tests/test_invalidationmanager.py @@ -79,21 +79,21 @@ def test___init__(t): def test_initialize_invalidation_filters(t, getUtilitiesFor): MockIInvalidationFilter = create_interface_mock(IInvalidationFilter) filters = [MockIInvalidationFilter() for i in range(3)] - # weighted in reverse order - for i, filter in enumerate(filters): - filter.weight = 10 - i + # Weighted in reverse order + for i, fltr in enumerate(filters): + fltr.weight = 10 - i getUtilitiesFor.return_value = [ ("f%s" % i, f) for i, f in enumerate(filters) ] - t.im.initialize_invalidation_filters() + initialized_filters = t.im.initialize_invalidation_filters(t.dmd) - for filter in filters: - filter.initialize.assert_called_with(t.dmd) + for fltr in filters: + fltr.initialize.assert_called_with(t.dmd) # check sorted by weight filters.reverse() - t.assertEqual(t.im._invalidation_filters, filters) + t.assertListEqual(initialized_filters, filters) @patch("{src}.time".format(**PATH), autospec=True) def test_process_invalidations(t, time): diff --git a/Products/ZenHub/tests/test_zenhub.py b/Products/ZenHub/tests/test_zenhub.py index 46cee949f1..9dd768cb3a 100644 --- a/Products/ZenHub/tests/test_zenhub.py +++ b/Products/ZenHub/tests/test_zenhub.py @@ -400,10 +400,6 @@ def test_processQueue(t): t.zh.processQueue() t.zh._invalidation_manager.process_invalidations.assert_called_with() - def test__initialize_invalidation_filters(t): - t.zh._initialize_invalidation_filters() - t.zh._invalidation_manager.initialize_invalidation_filters.assert_called_with() # noqa E501 - @patch("{src}.Event".format(**PATH), autospec=True) def test_sendEvent(t, Event): event = {"device": "x", "component": "y", "summary": "msg"} diff --git a/Products/ZenHub/zenhub.py b/Products/ZenHub/zenhub.py index b8422229c4..249fb07f91 100755 --- a/Products/ZenHub/zenhub.py +++ b/Products/ZenHub/zenhub.py @@ -267,12 +267,6 @@ def processQueue(self): """Periodically process database changes.""" yield self._invalidation_manager.process_invalidations() - # Legacy API - def _initialize_invalidation_filters(self): - self._invalidation_filters = ( - self._invalidation_manager.initialize_invalidation_filters() - ) - def sendEvent(self, **kw): """Post events to the EventManager. diff --git a/Products/ZenRelations/PrimaryPathObjectManager.py b/Products/ZenRelations/PrimaryPathObjectManager.py index c243a94dcb..a6fd22ba8a 100644 --- a/Products/ZenRelations/PrimaryPathObjectManager.py +++ b/Products/ZenRelations/PrimaryPathObjectManager.py @@ -94,9 +94,7 @@ class PrimaryPathObjectManager( PrimaryPathManager, App.Undo.UndoSupport, ): - """ - PrimaryPathObjectManager with basic Zope persistent classes. - """ + """PrimaryPathObjectManager with basic Zope persistent classes.""" manage_options = ( ObjectManager.manage_options diff --git a/Products/ZenRelations/__init__.py b/Products/ZenRelations/__init__.py index 388fdf1e95..4624754731 100644 --- a/Products/ZenRelations/__init__.py +++ b/Products/ZenRelations/__init__.py @@ -7,44 +7,29 @@ # ############################################################################## -__doc__ = """__init__ - -Initialize the RelationshipManager Product - -""" - -import logging - -from .RelationshipManager import ( - addRelationshipManager, - manage_addRelationshipManager, - RelationshipManager, -) -from .ToOneRelationship import ( - addToOneRelationship, - manage_addToOneRelationship, - ToOneRelationship, -) -from .ToManyRelationship import ( - addToManyRelationship, - manage_addToManyRelationship, - ToManyRelationship, -) -from .ToManyContRelationship import ( - addToManyContRelationship, - manage_addToManyContRelationship, - ToManyContRelationship, -) -from .ZenPropertyManager import setDescriptors - -log = logging.getLogger("zen.ZenRelations") - - -class ZODBConnectionError(Exception): - pass - def initialize(registrar): + from .RelationshipManager import ( + addRelationshipManager, + manage_addRelationshipManager, + RelationshipManager, + ) + from .ToManyContRelationship import ( + addToManyContRelationship, + manage_addToManyContRelationship, + ToManyContRelationship, + ) + from .ToManyRelationship import ( + addToManyRelationship, + manage_addToManyRelationship, + ToManyRelationship, + ) + from .ToOneRelationship import ( + addToOneRelationship, + manage_addToOneRelationship, + ToOneRelationship, + ) + registrar.registerClass( RelationshipManager, constructors=(addRelationshipManager, manage_addRelationshipManager), @@ -71,12 +56,15 @@ def initialize(registrar): def registerDescriptors(event): """ - Handler for IZopeApplicationOpenedEvent which registers property - descriptors. + IZopeApplicationOpenedEvent handler which registers property descriptors. """ zport = getattr(event.app, "zport", None) # zport may not exist if we are using zenbuild to initialize the database if zport: + from logging import getLogger + from .ZenPropertyManager import setDescriptors + + log = getLogger("zen.{}".format(__name__.split(".")[-1].lower())) try: setDescriptors(zport.dmd) except Exception as e: From c30ce39c34b4d35851d9838a81c896c24c1d99ea Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 24 Jul 2023 13:42:18 -0500 Subject: [PATCH 017/147] Replace usage of implements() with implementer() in Products.ZenHub --- Products/ZenHub/PBDaemon.py | 11 ++++------- Products/ZenHub/invalidations.py | 5 ++--- Products/ZenHub/metricpublisher/publisher.py | 6 +++--- Products/ZenHub/tests/testPBDaemon.py | 5 ++--- Products/ZenHub/zodb.py | 8 +++++--- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index dcb2903342..a832fa39d9 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -9,7 +9,7 @@ """PBDaemon -Base for daemons that connect to zenhub +Base for daemons that connect to zenhub. """ @@ -40,7 +40,7 @@ from twisted.spread import pb from ZODB.POSException import ConflictError from zope.component import getUtilitiesFor -from zope.interface import implements +from zope.interface import implementer from Products.ZenEvents.ZenEventClasses import ( App_Start, @@ -182,12 +182,9 @@ def callRemote(self, *args, **kwargs): return defer.fail(ex) +@implementer(ICollectorEventFingerprintGenerator) class DefaultFingerprintGenerator(object): - """ - Generates a fingerprint using a checksum of properties of the event. - """ - - implements(ICollectorEventFingerprintGenerator) + """Generates a fingerprint using a checksum of properties of the event.""" weight = 100 diff --git a/Products/ZenHub/invalidations.py b/Products/ZenHub/invalidations.py index 45117adb97..28091ebda6 100644 --- a/Products/ZenHub/invalidations.py +++ b/Products/ZenHub/invalidations.py @@ -13,7 +13,7 @@ from twisted.internet import defer from ZODB.utils import u64 from zope.component import adapter, getGlobalSiteManager -from zope.interface import implements, providedBy +from zope.interface import implementer, providedBy from Products.ZenModel.DeviceComponent import DeviceComponent from Products.ZenRelations.PrimaryPathObjectManager import ( @@ -68,6 +68,7 @@ def handle_oid(dmd, oid): return betterObjectEventNotify(event) +@implementer(IInvalidationProcessor) class InvalidationProcessor(object): """ Registered as a global utility. Given a database hook and a list of oids, @@ -75,8 +76,6 @@ class InvalidationProcessor(object): cause collectors to be pushed updates. """ - implements(IInvalidationProcessor) - _invalidation_queue = None _hub = None _hub_ready = None diff --git a/Products/ZenHub/metricpublisher/publisher.py b/Products/ZenHub/metricpublisher/publisher.py index 8df570a6bd..0d90b6a11b 100644 --- a/Products/ZenHub/metricpublisher/publisher.py +++ b/Products/ZenHub/metricpublisher/publisher.py @@ -21,7 +21,7 @@ from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer from txredis import RedisClientFactory -from zope.interface import implements +from zope.interface import implementer from Products.ZenUtils.MetricServiceRequest import getPool @@ -447,10 +447,10 @@ def _put(self, scheduled): return self._make_request() +@implementer(IBodyProducer) class StringProducer(object): - implements(IBodyProducer) """ - Implements twisted interface for writing a string to HTTP output stream + Implements twisted interface for writing a string to HTTP output stream. """ def __init__(self, postBody): diff --git a/Products/ZenHub/tests/testPBDaemon.py b/Products/ZenHub/tests/testPBDaemon.py index 0fb2987bf6..64ce71781f 100644 --- a/Products/ZenHub/tests/testPBDaemon.py +++ b/Products/ZenHub/tests/testPBDaemon.py @@ -11,7 +11,7 @@ import os from twisted.internet.defer import failure -from zope.interface import implements +from zope.interface import implementer from zope.component import getGlobalSiteManager from Products.ZenTestCase.BaseTestCase import BaseTestCase @@ -170,9 +170,8 @@ class MockOptions(object): return options def testAddEventDroppedTransform(self): + @implementer(ICollectorEventTransformer) class DroppingTransformer(object): - implements(ICollectorEventTransformer) - def __init__(self): self.num_dropped = 0 diff --git a/Products/ZenHub/zodb.py b/Products/ZenHub/zodb.py index f47a1ebab5..5ed96d9ce3 100644 --- a/Products/ZenHub/zodb.py +++ b/Products/ZenHub/zodb.py @@ -11,8 +11,8 @@ from zope.component import provideHandler from zope.component.interfaces import ObjectEvent +from zope.interface import implementer from zope.interface.advice import addClassAdvisor -from zope.interface import implements from .interfaces import IUpdateEvent, IDeletionEvent @@ -25,12 +25,14 @@ def __init__(self, object, oid): self.oid = oid +@implementer(IUpdateEvent) class UpdateEvent(InvalidationEvent): - implements(IUpdateEvent) + pass +@implementer(IDeletionEvent) class DeletionEvent(InvalidationEvent): - implements(IDeletionEvent) + pass def _listener_decorator_factory(eventtype): From ee07697dcf0a5fdd876e88ae2c470bf2361883d9 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 9 Aug 2023 10:20:38 -0500 Subject: [PATCH 018/147] Fixed improperly formatted device name argument. Placing quotes around an argument is incorrect when the arguments are not processed through a shell. --- Products/ZenModel/PerformanceConf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenModel/PerformanceConf.py b/Products/ZenModel/PerformanceConf.py index 15301717be..f6cd7e07f6 100644 --- a/Products/ZenModel/PerformanceConf.py +++ b/Products/ZenModel/PerformanceConf.py @@ -582,7 +582,7 @@ def _getZenModelerCommand( cmd = [zm] deviceName = self._escapeParentheses(deviceName) options = [ - 'run', '--now', '-d', '"{}"'.format(deviceName), '--monitor', performanceMonitor, + 'run', '--now', '-d', deviceName, '--monitor', performanceMonitor, '--collect={}'.format(collectPlugins) ] cmd.extend(options) From 26eba4c63dd4008a3b1e3274d07fa7fc8b4d4cf7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 9 Aug 2023 15:37:13 -0500 Subject: [PATCH 019/147] Don't fail script if image removal fails. --- jenkins_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins_build.sh b/jenkins_build.sh index 2edce67824..46e7024b39 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -51,8 +51,8 @@ cleanup() { RC="$?" if [[ $RC == 0 ]]; then zendev drop ${ZENDEV_ENV} - docker image rm zendev/devimg:${ZENDEV_ENV} zendev/product-base:${ZENDEV_ENV} - docker image rm zendev/mariadb:${ZENDEV_ENV} zendev/mariadb-base:${ZENDEV_ENV} + docker image rm -f zendev/devimg:${ZENDEV_ENV} zendev/product-base:${ZENDEV_ENV} + docker image rm -f zendev/mariadb:${ZENDEV_ENV} zendev/mariadb-base:${ZENDEV_ENV} fi } trap cleanup INT TERM EXIT From 9b3668751c60116e2d031e4d9bdd8ee5ef417b9c Mon Sep 17 00:00:00 2001 From: Deer-Warlord Date: Wed, 9 Aug 2023 15:25:02 +0300 Subject: [PATCH 020/147] Replace old import from MySQL-python lib to mysqlclient lib in zeneventmigrate.py ZEN-34463 --- Products/ZenEvents/zeneventmigrate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Products/ZenEvents/zeneventmigrate.py b/Products/ZenEvents/zeneventmigrate.py index 1be669772b..095a647c04 100644 --- a/Products/ZenEvents/zeneventmigrate.py +++ b/Products/ZenEvents/zeneventmigrate.py @@ -50,9 +50,8 @@ from Products.ZenUtils.mysql import MySQLdb -from MySQLdb import connect +from MySQLdb import connect, escape_string from MySQLdb.cursors import DictCursor -from _mysql import escape_string from zenoss.protocols.protobufs.zep_pb2 import (EventSummary, ZepRawEvent, STATUS_NEW, STATUS_ACKNOWLEDGED, STATUS_SUPPRESSED, STATUS_CLOSED, STATUS_CLEARED, From 7a456a2c69395f4023e42ca11b050323fb58977e Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 14 Aug 2023 09:46:03 -0500 Subject: [PATCH 021/147] Reformatted a few files, cleaned up config file generation. Config file output won't duplicate 'default' value in message if help text contains `%default`. --- Products/ZenUtils/CmdBase.py | 795 +++++++++++++++----------- Products/ZenUtils/Executor.py | 39 +- Products/ZenUtils/GlobalConfig.py | 123 ++-- Products/ZenUtils/MySqlZodbFactory.py | 4 +- Products/ZenUtils/ZCmdBase.py | 139 +++-- Products/ZenUtils/ZenDaemon.py | 372 +++++++----- Products/ZenUtils/ZenScriptBase.py | 91 ++- Products/ZenUtils/ZodbFactory.py | 31 +- Products/ZenUtils/config.py | 206 ++++--- 9 files changed, 1056 insertions(+), 744 deletions(-) diff --git a/Products/ZenUtils/CmdBase.py b/Products/ZenUtils/CmdBase.py index 04f0920566..a0505d2923 100644 --- a/Products/ZenUtils/CmdBase.py +++ b/Products/ZenUtils/CmdBase.py @@ -7,49 +7,60 @@ # ############################################################################## +from __future__ import absolute_import, print_function -__doc__="""CmdBase - -Provide utility functions for logging and config file parsing -to command-line programs -""" - -import os -import os.path -import sys import datetime import logging +import os +import os.path import re +import sys + from copy import copy +from optparse import ( + BadOptionError, + NO_DEFAULT, + Option, + OptionGroup, + OptionParser, + OptionValueError, + SUPPRESS_HELP, +) +from urllib import quote + import zope.component + from zope.traversing.adapters import DefaultTraversable from Zope2.App import zcml -from optparse import ( - OptionParser, OptionGroup, Option, - SUPPRESS_HELP, NO_DEFAULT, OptionValueError, BadOptionError, - ) -from urllib import quote - -from Products.ZenUtils.Utils import unused, load_config_override, zenPath, getAllParserOptionsGen -from Products.ZenUtils.GlobalConfig import _convertConfigLinesToArguments, applyGlobalConfToParser +from .Utils import ( + getAllParserOptionsGen, + load_config_override, + unused, + zenPath, +) +from .GlobalConfig import ( + _convertConfigLinesToArguments, + applyGlobalConfToParser, +) -class DMDError: pass +class DMDError: + pass def checkLogLevel(option, opt, value): - if re.match(r'^\d+$', value): + if re.match(r"^\d+$", value): value = int(value) else: intval = getattr(logging, value.upper(), None) - if intval: - value = intval - else: + if intval is None: raise OptionValueError('"%s" is not a valid log level.' % value) + value = intval return value + def remove_args(argv, remove_args_novals, remove_args_vals): """ Removes arguments from the argument list. Arguments in @@ -66,15 +77,16 @@ def remove_args(argv, remove_args_novals, remove_args_vals): for remove_arg in remove_args_vals: if remove_arg == arg: add_arg = False - it.next() # Skip the argument value + it.next() # Skip the argument value break - elif arg.startswith(remove_arg + '='): + elif arg.startswith(remove_arg + "="): add_arg = False break if add_arg: new_args.append(arg) return new_args + class LogSeverityOption(Option): TYPES = Option.TYPES + ("loglevel",) TYPE_CHECKER = copy(Option.TYPE_CHECKER) @@ -82,27 +94,31 @@ class LogSeverityOption(Option): class CmdBase(object): + """ + Base class used for most Zenoss commands. + """ doesLogging = True - """ - Class used for all Zenoss commands - """ def __init__(self, noopts=0, args=None, should_log=None): zope.component.provideAdapter(DefaultTraversable, (None,)) # This explicitly loads all of the products - must happen first! from OFS.Application import import_products + import_products() - #make sure we aren't in debug mode + # make sure we aren't in debug mode import Globals + Globals.DevelopmentMode = False # We must import ZenossStartup at this point so that all Zenoss daemons # and tools will have any ZenPack monkey-patched methods available. import Products.ZenossStartup + unused(Products.ZenossStartup) zcml.load_site() import Products.ZenWidgets - load_config_override('scriptmessaging.zcml', Products.ZenWidgets) + + load_config_override("scriptmessaging.zcml", Products.ZenWidgets) self.usage = "%prog [options]" self.noopts = noopts @@ -124,7 +140,9 @@ def __init__(self, noopts=0, args=None, should_log=None): applyGlobalConfToParser(self.parser) self.parseOptions() if self.options.configfile: - self.parser.defaults = self.getConfigFileDefaults(self.options.configfile) + self.parser.defaults = self.getConfigFileDefaults( + self.options.configfile + ) # We've updated the parser with defaults from configs, now we need # to reparse our command-line to get the correct overrides from # the command-line @@ -132,27 +150,29 @@ def __init__(self, noopts=0, args=None, should_log=None): if should_log is not None: self.doesLogging = should_log - + if self.doesLogging: self.setupLogging() - def buildParser(self): """ - Create the options parser + Create the options parser. """ if not self.parser: from Products.ZenModel.ZenossInfo import ZenossInfo + try: - zinfo= ZenossInfo('') - version= str(zinfo.getZenossVersion()) + zinfo = ZenossInfo("") + version = str(zinfo.getZenossVersion()) except Exception: from Products.ZenModel.ZVersion import VERSION - version= VERSION - self.parser = OptionParser(usage=self.usage, - version="%prog " + version, - option_class=LogSeverityOption) + version = VERSION + self.parser = OptionParser( + usage=self.usage, + version="%prog " + version, + option_class=LogSeverityOption, + ) def buildOptions(self): """ @@ -163,67 +183,81 @@ def buildOptions(self): if self.doesLogging: group = OptionGroup(self.parser, "Logging Options") group.add_option( - '-v', '--logseverity', - dest='logseverity', default='INFO', type='loglevel', - help='Logging severity threshold', + "-v", + "--logseverity", + dest="logseverity", + default="INFO", + type="loglevel", + help="Logging severity threshold", ) group.add_option( - '--logpath', dest='logpath', default=zenPath('log'), type='str', - help='Override the default logging path; default $ZENHOME/log' + "--logpath", + dest="logpath", + default=zenPath("log"), + type="str", + help="Override the default logging path; default %default", ) group.add_option( - '--maxlogsize', - dest='maxLogKiloBytes', default=10240, type='int', - help='Max size of log file in KB; default 10240', + "--maxlogsize", + dest="maxLogKiloBytes", + default=10240, + type="int", + help="Max size of log file in KB; default %default", ) group.add_option( - '--maxbackuplogs', - dest='maxBackupLogs', default=3, type='int', - help='Max number of back up log files; default 3', + "--maxbackuplogs", + dest="maxBackupLogs", + default=3, + type="int", + help="Max number of back up log files; default %default", ) self.parser.add_option_group(group) - self.parser.add_option("-C", "--configfile", - dest="configfile", - help="Use an alternate configuration file" ) - - self.parser.add_option("--genconf", - action="store_true", - default=False, - help="Generate a template configuration file" ) + self.parser.add_option( + "-C", + "--configfile", + dest="configfile", + help="Use an alternate configuration file", + ) - self.parser.add_option("--genxmltable", - action="store_true", - default=False, - help="Generate a Docbook table showing command-line switches." ) + self.parser.add_option( + "--genconf", + action="store_true", + default=False, + help="Generate a template configuration file", + ) - self.parser.add_option("--genxmlconfigs", - action="store_true", - default=False, - help="Generate an XML file containing command-line switches." ) + self.parser.add_option( + "--genxmltable", + action="store_true", + default=False, + help="Generate a Docbook table showing command-line switches.", + ) + self.parser.add_option( + "--genxmlconfigs", + action="store_true", + default=False, + help="Generate an XML file containing command-line switches.", + ) def parseOptions(self): """ - Uses the optparse parse previously populated and performs common options. + Uses the optparse parse previously populated and performs common + options. """ - - if self.noopts: - args = [] - else: - args = self.inputArgs + args = [] if self.noopts else self.inputArgs (self.options, self.args) = self.parser.parse_args(args=args) if self.options.genconf: - self.generate_configs( self.parser, self.options ) + self.generate_configs(self.parser, self.options) if self.options.genxmltable: - self.generate_xml_table( self.parser, self.options ) + self.generate_xml_table(self.parser, self.options) if self.options.genxmlconfigs: - self.generate_xml_configs( self.parser, self.options ) - + self.generate_xml_configs(self.parser, self.options) def getConfigFileDefaults(self, filename, correctErrors=True): # TODO: This should be refactored - duplicated code with CmdBase. @@ -231,33 +265,38 @@ def getConfigFileDefaults(self, filename, correctErrors=True): Parse a config file which has key-value pairs delimited by white space, and update the parser's option defaults with these values. - @parameter filename: name of configuration file - @type filename: string + :parameter filename: name of configuration file + :type filename: string """ - options = self.parser.get_default_values() lines = self.loadConfigFile(filename) if lines: - lines, errors = self.validateConfigFile(filename, lines, - correctErrors=correctErrors) + lines, errors = self.validateConfigFile( + filename, lines, correctErrors=correctErrors + ) args = self.getParamatersFromConfig(lines) try: self.parser._process_args([], args, options) except (BadOptionError, OptionValueError) as err: - print >>sys.stderr, 'WARN: %s in config file %s' % (err, filename) + print( + "WARN: %s in config file %s" + % ( + err, + filename, + ), + file=sys.stderr, + ) return options.__dict__ - def getGlobalConfigFileDefaults(self): - # Deprecated: This method is going away - it is duplicated in GlobalConfig.py + # Deprecated: This method is duplicated in GlobalConfig.py """ Parse a config file which has key-value pairs delimited by white space, and update the parser's option defaults with these values. """ - - filename = zenPath('etc', 'global.conf') + filename = zenPath("etc", "global.conf") options = self.parser.get_default_values() lines = self.loadConfigFile(filename) if lines: @@ -266,14 +305,13 @@ def getGlobalConfigFileDefaults(self): try: self.parser._process_args([], args, options) except (BadOptionError, OptionValueError): - # Ignore it, we only care about our own options as defined in the parser + # Ignore it, we only care about our own options as + # defined in the parser. pass return options.__dict__ - def loadConfigFile(self, filename): - # TODO: This should be refactored - duplicated code with CmdBase. """ Parse a config file which has key-value pairs delimited by white space. @@ -286,37 +324,60 @@ def loadConfigFile(self, filename): try: with open(filename) as file: for line in file: - if line.lstrip().startswith('#') or line.strip() == '': - lines.append(dict(type='comment', line=line)) + if line.lstrip().startswith("#") or line.strip() == "": + lines.append(dict(type="comment", line=line)) else: try: - # add default blank string for keys with no default value - # valid delimiters are space, ':' and/or '=' (see ZenUtils/config.py) - key, value = (re.split(r'[\s:=]+', line.strip(), 1) + ['',])[:2] + # Add default blank string for keys with no + # default value. + # Valid delimiters are space, ':' and/or '=' + # (see ZenUtils/config.py) + key, value = ( + re.split(r"[\s:=]+", line.strip(), 1) + [""] + )[:2] except ValueError: - lines.append(dict(type='option', line=line, key=line.strip(), value=None, option=None)) + lines.append( + dict( + type="option", + line=line, + key=line.strip(), + value=None, + option=None, + ) + ) else: - option = self.parser.get_option('--%s' % key) - lines.append(dict(type='option', line=line, key=key, value=value, option=option)) + option = self.parser.get_option("--%s" % key) + lines.append( + dict( + type="option", + line=line, + key=key, + value=value, + option=option, + ) + ) except IOError as e: - errorMessage = ('WARN: unable to read config file {filename} ' - '-- skipping. ({exceptionName}: {exception})').format( + errorMessage = ( + "WARN: unable to read config file {filename} " + "-- skipping. ({exceptionName}: {exception})" + ).format( filename=filename, exceptionName=e.__class__.__name__, - exception=e + exception=e, ) - print >>sys.stderr, errorMessage + print(errorMessage, file=sys.stderr) return [] return lines - - def validateConfigFile(self, filename, lines, correctErrors=True, warnErrors=True): + def validateConfigFile( + self, filename, lines, correctErrors=True, warnErrors=True + ): """ - Validate config file lines which has key-value pairs delimited by white space, - and validate that the keys exist for this command's option parser. If - the option does not exist or has an empty value it will comment it out - in the config file. + Validate config file lines which has key-value pairs delimited by + white space, and validate that the keys exist for this command's + option parser. If the option does not exist or has an empty value it + will comment it out in the config file. @parameter filename: path to the configuration file @type filename: string @@ -331,82 +392,107 @@ def validateConfigFile(self, filename, lines, correctErrors=True, warnErrors=Tru errors = [] validLines = [] date = datetime.datetime.now().isoformat() - errorTemplate = '## Commenting out by config parser (%s) on %s: %%s\n' % ( - sys.argv[0], date) + errorTemplate = ( + "## Commenting out by config parser (%s) on %s: %%s\n" + % (sys.argv[0], date) + ) for lineno, line in enumerate(lines): - if line['type'] == 'comment': - output.append(line['line']) - elif line['type'] == 'option': - if line['value'] is None: - errors.append((lineno + 1, 'missing value for "%s"' % line['key'])) - output.append(errorTemplate % 'missing value') - output.append('## %s' % line['line']) - elif line['option'] is None: - errors.append((lineno + 1, 'unknown option "%s"' % line['key'])) - output.append(errorTemplate % 'unknown option') - output.append('## %s' % line['line']) + if line["type"] == "comment": + output.append(line["line"]) + elif line["type"] == "option": + if line["value"] is None: + errors.append( + (lineno + 1, 'missing value for "%s"' % line["key"]) + ) + output.append(errorTemplate % "missing value") + output.append("## %s" % line["line"]) + elif line["option"] is None: + errors.append( + (lineno + 1, 'unknown option "%s"' % line["key"]) + ) + output.append(errorTemplate % "unknown option") + output.append("## %s" % line["line"]) else: validLines.append(line) - output.append(line['line']) + output.append(line["line"]) else: - errors.append((lineno + 1, 'unknown line "%s"' % line['line'])) - output.append(errorTemplate % 'unknown line') - output.append('## %s' % line['line']) + errors.append((lineno + 1, 'unknown line "%s"' % line["line"])) + output.append(errorTemplate % "unknown line") + output.append("## %s" % line["line"]) if errors: if correctErrors: for lineno, message in errors: - print >>sys.stderr, 'INFO: Commenting out %s on line %d in %s' % (message, lineno, filename) - - with open(filename, 'w') as file: + print( + "INFO: Commenting out %s on line %d in %s" + % ( + message, + lineno, + filename, + ), + file=sys.stderr, + ) + + with open(filename, "w") as file: file.writelines(output) if warnErrors: for lineno, message in errors: - print >>sys.stderr, 'WARN: %s on line %d in %s' % (message, lineno, filename) + print( + "WARN: %s on line %d in %s" + % ( + message, + lineno, + filename, + ), + file=sys.stderr, + ) return validLines, errors - def getParamatersFromConfig(self, lines): # Deprecated: This method is going away return _convertConfigLinesToArguments(self.parser, lines) - def setupLogging(self): """ - Set common logging options + Set common logging options. """ rlog = logging.getLogger() rlog.setLevel(logging.WARN) mname = self.__class__.__name__ - self.log = logging.getLogger("zen."+ mname) + self.log = logging.getLogger("zen." + mname) zlog = logging.getLogger("zen") try: loglevel = int(self.options.logseverity) except ValueError: - loglevel = getattr(logging, self.options.logseverity.upper(), logging.INFO) + loglevel = getattr( + logging, self.options.logseverity.upper(), logging.INFO + ) zlog.setLevel(loglevel) logdir = self.checkLogpath() if logdir: - logfile = os.path.join(logdir, mname.lower()+".log") + logfile = os.path.join(logdir, mname.lower() + ".log") maxBytes = self.options.maxLogKiloBytes * 1024 backupCount = self.options.maxBackupLogs - h = logging.handlers.RotatingFileHandler(logfile, maxBytes=maxBytes, - backupCount=backupCount) - h.setFormatter(logging.Formatter( - "%(asctime)s %(levelname)s %(name)s: %(message)s", - "%Y-%m-%d %H:%M:%S")) + h = logging.handlers.RotatingFileHandler( + logfile, maxBytes=maxBytes, backupCount=backupCount + ) + h.setFormatter( + logging.Formatter( + "%(asctime)s %(levelname)s %(name)s: %(message)s", + "%Y-%m-%d %H:%M:%S", + ) + ) rlog.addHandler(h) else: logging.basicConfig() - def checkLogpath(self): """ - Validate the logpath is valid + Validate the logpath is valid. """ if not self.options.logpath: return None @@ -417,93 +503,99 @@ def checkLogpath(self): try: os.makedirs(logdir) except OSError: - raise SystemExit("logpath:%s doesn't exist and cannot be created" % logdir) + raise SystemExit( + "logpath:%s doesn't exist and cannot be created" + % logdir + ) elif not os.path.isdir(logdir): - raise SystemExit("logpath:%s exists but is not a directory" % logdir) + raise SystemExit( + "logpath:%s exists but is not a directory" % logdir + ) return logdir - - def pretty_print_config_comment( self, comment ): + def pretty_print_config_comment(self, comment): """ - Quick and dirty pretty printer for comments that happen to be longer than can comfortably -be seen on the display. + Quick and dirty pretty printer for comments that happen to be longer + than can comfortably be seen on the display. """ - - max_size= 40 + max_size = 40 # # As a heuristic we'll accept strings that are +- text_window # size in length. # - text_window= 5 + text_window = 5 - if len( comment ) <= max_size + text_window: - return comment + if len(comment) <= max_size + text_window: + return comment # - # First, take care of embedded newlines and expand them out to array entries + # First, take care of embedded newlines and expand them out to + # array entries. # - new_comment= [] - all_lines= comment.split( '\n' ) + new_comment = [] + all_lines = comment.split("\n") for line in all_lines: - if len(line) <= max_size + text_window: - new_comment.append( line ) + if len(line) <= max_size + text_window: + new_comment.append(line) continue - start_position= max_size - text_window - while len(line) > max_size + text_window: - index= line.find( ' ', start_position ) + start_position = max_size - text_window + while len(line) > max_size + text_window: + index = line.find(" ", start_position) if index > 0: - new_comment.append( line[ 0:index ] ) - line= line[ index: ] + new_comment.append(line[0:index]) + line = line[index:] else: - if start_position == 0: + if start_position == 0: # - # If we get here it means that the line is just one big string with no spaces - # in it. There's nothing that we can do except print it out. Doh! + # If we get here it means that the line is just one + # big string with no spaces in it. There's nothing + # that we can do except print it out. Doh! # - new_comment.append( line ) + new_comment.append(line) break - # - # Okay, haven't found anything to split on -- go back and try again - # - start_position= start_position - text_window - if start_position < 0: - start_position= 0 - - else: - new_comment.append( line ) - - return "\n# ".join( new_comment ) + # + # Okay, haven't found anything to split on + # -- go back and try again + # + start_position = start_position - text_window + if start_position < 0: + start_position = 0 + else: + new_comment.append(line) + return "\n# ".join(new_comment) - def generate_configs( self, parser, options ): + def generate_configs(self, parser, options): """ Create a configuration file based on the long-form of the option names - @parameter parser: an optparse parser object which contains defaults, help - @parameter options: parsed options list containing actual values + :param parser: an optparse parser object which contains defaults, help + :param options: parsed options list containing actual values """ # # Header for the configuration file # unused(options) - daemon_name= os.path.basename( sys.argv[0] ) - daemon_name= daemon_name.replace( '.py', '' ) + daemon_name = os.path.basename(sys.argv[0]) + daemon_name = daemon_name.replace(".py", "") - print """# + print( + """# # Configuration file for %s # # To enable a particular option, uncomment the desired entry. # # Parameter Setting -# --------- -------""" % ( daemon_name ) - +# --------- -------""" + % (daemon_name) + ) - options_to_ignore= ( 'help', 'version', '', 'genconf', 'genxmltable' ) + options_to_ignore = ("help", "version", "", "genconf", "genxmltable") # # Create an entry for each of the command line flags @@ -512,74 +604,82 @@ def generate_configs( self, parser, options ): # entries, rather than the command line options. # import re + for opt in getAllParserOptionsGen(parser): - if opt.help is SUPPRESS_HELP: - continue - - # - # Get rid of the short version of the command - # - option_name= re.sub( r'.*/--', '', "%s" % opt ) - - # - # And what if there's no short version? - # - option_name= re.sub( r'^--', '', "%s" % option_name ) - - # - # Don't display anything we shouldn't be displaying - # - if option_name in options_to_ignore: - continue - - # - # Find the actual value specified on the command line, if any, - # and display it - # - - value= getattr( parser.values, opt.dest ) - - default_value= parser.defaults.get( opt.dest ) - if default_value is NO_DEFAULT or default_value is None: - default_value= "" - default_string= "" + if opt.help is SUPPRESS_HELP: + continue + + # + # Get rid of the short version of the command + # + option_name = re.sub(r".*/--", "", "%s" % opt) + + # + # And what if there's no short version? + # + option_name = re.sub(r"^--", "", "%s" % option_name) + + # + # Don't display anything we shouldn't be displaying + # + if option_name in options_to_ignore: + continue + + # + # Find the actual value specified on the command line, if any, + # and display it + # + + value = getattr(parser.values, opt.dest) + + default_value = parser.defaults.get(opt.dest) + if default_value is NO_DEFAULT or default_value is None: + default_value = "" + + if "%default" in opt.help: + help_txt = opt.help.replace("%default", str(default_value)) + else: + default_string = "" if default_value != "": - default_string= ", default: " + str( default_value ) + default_string = ", default: " + str(default_value) + help_txt = opt.help + default_string - comment= self.pretty_print_config_comment( opt.help + default_string ) + comment = self.pretty_print_config_comment(help_txt) - # - # NB: I would prefer to use tabs to separate the parameter name - # and value, but I don't know that this would work. - # - print """# + # + # NB: I would prefer to use tabs to separate the parameter name + # and value, but I don't know that this would work. + # + print( + """# # %s -#%s %s""" % ( comment, option_name, value ) +#%s %s""" + % (comment, option_name, value) + ) # # Pretty print and exit # - print "#" - sys.exit( 0 ) + print("#") + sys.exit(0) - - - def generate_xml_table( self, parser, options ): + def generate_xml_table(self, parser, options): """ Create a Docbook table based on the long-form of the option names - @parameter parser: an optparse parser object which contains defaults, help - @parameter options: parsed options list containing actual values + :param parser: an optparse parser object which contains defaults, help + :param options: parsed options list containing actual values """ # # Header for the configuration file # unused(options) - daemon_name= os.path.basename( sys.argv[0] ) - daemon_name= daemon_name.replace( '.py', '' ) + daemon_name = os.path.basename(sys.argv[0]) + daemon_name = daemon_name.replace(".py", "") - print """ + print( + """
-""" % ( daemon_name, daemon_name, daemon_name, daemon_name ) - +""" # noqa E501 + % ( + daemon_name, + daemon_name, + daemon_name, + daemon_name, + ) + ) - options_to_ignore= ( 'help', 'version', '', 'genconf', 'genxmltable' ) + options_to_ignore = ("help", "version", "", "genconf", "genxmltable") # # Create an entry for each of the command line flags @@ -618,73 +724,97 @@ def generate_xml_table( self, parser, options ): # entries, rather than the command line options. # import re + for opt in getAllParserOptionsGen(parser): - if opt.help is SUPPRESS_HELP: - continue - - # - # Create a Docbook-happy version of the option strings - # Yes, would be better semantically, but the output - # just looks goofy in a table. Use literal instead. - # - all_options= '' + re.sub( r'/', ', ', "%s" % opt ) + '' - - # - # Don't display anything we shouldn't be displaying - # - option_name= re.sub( r'.*/--', '', "%s" % opt ) - option_name= re.sub( r'^--', '', "%s" % option_name ) - if option_name in options_to_ignore: - continue - - default_value= parser.defaults.get( opt.dest ) - if default_value is NO_DEFAULT or default_value is None: - default_value= "" - default_string= "" - if default_value != "": - default_string= " Default: " + str( default_value ) + "\n" + if opt.help is SUPPRESS_HELP: + continue - comment= self.pretty_print_config_comment( opt.help ) + # + # Create a Docbook-happy version of the option strings + # Yes, would be better semantically, but the output + # just looks goofy in a table. Use literal instead. + # + all_options = ( + "" + + re.sub( + r"/", ", ", "%s" % opt + ) + + "" + ) -# -# TODO: Determine the variable name used and display the --option_name=variable_name -# - if opt.action in [ 'store_true', 'store_false' ]: - print """ + # + # Don't display anything we shouldn't be displaying + # + option_name = re.sub(r".*/--", "", "%s" % opt) + option_name = re.sub(r"^--", "", "%s" % option_name) + if option_name in options_to_ignore: + continue + + default_value = parser.defaults.get(opt.dest) + if default_value is NO_DEFAULT or default_value is None: + default_value = "" + default_string = "" + if default_value != "": + default_string = ( + " Default: " + + str(default_value) + + "\n" + ) + + comment = self.pretty_print_config_comment(opt.help) + + # + # TODO: Determine the variable name used and display the + # --option_name=variable_name + # + if opt.action in ["store_true", "store_false"]: + print( + """ %s %s %s -""" % ( all_options, comment, default_string ) +""" + % ( + all_options, + comment, + default_string, + ) + ) - else: - target= '=' + opt.dest.lower() + '' - all_options= all_options + target - all_options= re.sub( r',', target + ',', all_options ) - print """ + else: + target = "=" + opt.dest.lower() + "" + all_options = all_options + target + all_options = re.sub(r",", target + ",", all_options) + print( + """ %s %s %s -""" % ( all_options, comment, default_string ) - - +""" + % ( + all_options, + comment, + default_string, + ) + ) # # Close the table elements # - print """ + print( + """
""" - sys.exit( 0 ) - - + ) + sys.exit(0) - def generate_xml_configs( self, parser, options ): + def generate_xml_configs(self, parser, options): """ Create an XML file that can be used to create Docbook files as well as used as the basis for GUI-based daemon option @@ -695,21 +825,27 @@ def generate_xml_configs( self, parser, options ): # Header for the configuration file # unused(options) - daemon_name= os.path.basename( sys.argv[0] ) - daemon_name= daemon_name.replace( '.py', '' ) + daemon_name = os.path.basename(sys.argv[0]) + daemon_name = daemon_name.replace(".py", "") export_date = datetime.datetime.now() - print """ + print( + """ +""" + % (export_date, daemon_name) + ) -""" % ( export_date, daemon_name ) - - options_to_ignore= ( - 'help', 'version', '', 'genconf', 'genxmltable', - 'genxmlconfigs', + options_to_ignore = ( + "help", + "version", + "", + "genconf", + "genxmltable", + "genxmlconfigs", ) # @@ -718,42 +854,61 @@ def generate_xml_configs( self, parser, options ): # NB: Ideally, this should print out only the option parser dest # entries, rather than the command line options. # - import re for opt in getAllParserOptionsGen(parser): - if opt.help is SUPPRESS_HELP: - continue - - # - # Don't display anything we shouldn't be displaying - # - option_name= re.sub( r'.*/--', '', "%s" % opt ) - option_name= re.sub( r'^--', '', "%s" % option_name ) - if option_name in options_to_ignore: - continue - - default_value= parser.defaults.get( opt.dest ) - if default_value is NO_DEFAULT or default_value is None: - default_string= "" - else: - default_string= str( default_value ) + if opt.help is SUPPRESS_HELP: + continue -# -# TODO: Determine the variable name used and display the --option_name=variable_name -# - if opt.action in [ 'store_true', 'store_false' ]: - print """ -""" - sys.exit( 0 ) + print( + """ +""" + ) + sys.exit(0) diff --git a/Products/ZenUtils/Executor.py b/Products/ZenUtils/Executor.py index fdcd1abdff..935d8e3944 100644 --- a/Products/ZenUtils/Executor.py +++ b/Products/ZenUtils/Executor.py @@ -19,21 +19,25 @@ def makeExecutor(queue=None, limit=0, log=log, startnow=True): - """Return a new task executor. + """ + Return a new task executor. A limit of zero implies no limit. - @param name: Name of the executor - @type name: str - - @param queue: A queue-like object for storing tasks - @type queue: defer.DeferredQueue() or similiar - - @param limit: The maximum number of concurrent tasks - @type limit: int - - @param log: The log object - @type log: logging.Logger + If startnow is False, then the `start` method must be called on the + returned executor to start the executor running. + + :param name: Name of the executor + :type name: str + :param queue: A queue-like object for storing tasks + :type queue: defer.DeferredQueue() or equivalent + :param limit: The maximum number of concurrent tasks + :type limit: int + :param log: The log object + :type log: logging.Logger + :param startnow: If True, start the executor immediately (default True). + :type startnow: boolean + :rtype: AsyncExecutor """ if queue is None: queue = defer.DeferredQueue() @@ -136,10 +140,15 @@ def queued(self): def submit(self, call, timeout=None, label=""): """Submit a callable to run asynchronously. - @param call: A callable to be executed - @type call: callable + :param call: A callable to be executed + :type call: callable + :param timeout: how long a task can run before timeout? + :type timeout: float + :param label: A optional value to apply to a task. This value can + be used to associate related tasks. + :type label: str - @return: A Deferred that returns the return value of the callable + :return: A Deferred that returns the return value of the callable if it does not raise an exception. If the callable raises an exception, the Deferred returns the exception. """ diff --git a/Products/ZenUtils/GlobalConfig.py b/Products/ZenUtils/GlobalConfig.py index 3244e5c6c1..f80cc715bc 100644 --- a/Products/ZenUtils/GlobalConfig.py +++ b/Products/ZenUtils/GlobalConfig.py @@ -1,43 +1,46 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2010, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## +from __future__ import absolute_import, print_function import logging -log = logging.getLogger('zen.GlobalConfig') -import sys -from optparse import OptionValueError, BadOptionError -import re import os.path +import re +import sys -from Products.ZenUtils.Utils import zenPath, getAllParserOptionsGen -from Products.ZenUtils.config import Config, ConfigLoader - +from optparse import OptionValueError, BadOptionError -CONFIG_FILE = zenPath('etc', 'global.conf') +from .config import Config, ConfigLoader +from .Utils import zenPath, getAllParserOptionsGen +log = logging.getLogger("zen.GlobalConfig") +CONFIG_FILE = zenPath("etc", "global.conf") +_KEYVALUE = re.compile( + r"^[\s ]*(?P[a-z_]+[a-z0-9_-]*)[\s]+(?P[^\s#]+)", re.IGNORECASE +).search -_KEYVALUE = re.compile("^[\s ]*(?P[a-z_]+[a-z0-9_-]*)[\s]+(?P[^\s#]+)", re.IGNORECASE).search def globalConfToDict(): settings = {} - globalConfFile = zenPath('etc','global.conf') + globalConfFile = zenPath("etc", "global.conf") if os.path.exists(globalConfFile): - with open(globalConfFile, 'r') as f: + with open(globalConfFile, "r") as f: for line in f.xreadlines(): match = _KEYVALUE(line) if match: - value = match.group('value') + value = match.group("value") if value.isdigit(): value = int(value) - settings[match.group('key')] = value + settings[match.group("key")] = value return settings + class GlobalConfig(Config): """ A method for retrieving the global configuration options @@ -46,19 +49,23 @@ class GlobalConfig(Config): @todo Add validation for expected keys and values """ - pass + _GLOBAL_CONFIG = ConfigLoader(CONFIG_FILE, GlobalConfig) + + def getGlobalConfiguration(): return _GLOBAL_CONFIG() def flagToConfig(flag): return flag.trim().lstrip("-").replace("-", "_") - + + def configToFlag(option): return "--" + option.strip().replace("_", "-") + def _convertConfigLinesToArguments(parser, lines): """ Converts configuration file lines of the format: @@ -73,39 +80,50 @@ def _convertConfigLinesToArguments(parser, lines): @parameter parser: OptionParser object containing configuration options. @type parser: OptionParser - @parameter lines: List of dictionary object parsed from a configuration file. - Each option is expected to have 'type', 'key', 'value' entries. + @parameter lines: List of dictionary object parsed from a configuration + file. Each option is expected to have 'type', 'key', 'value' entries. @type lines: list of dictionaries. - @return: List of command-line arguments corresponding to the configuration file. + @return: List of command-line arguments corresponding to the + configuration file. @rtype: list of strings """ # valid key # an option's string without the leading "--" # can differ from an option's destination - validOpts = set((opt.get_opt_string() for opt in getAllParserOptionsGen(parser))) - + validOpts = set( + (opt.get_opt_string() for opt in getAllParserOptionsGen(parser)) + ) + args = [] for line in lines: - if line.get('type', None) != 'option': + if line.get("type", None) != "option": continue - optstring = configToFlag(line['key']) + optstring = configToFlag(line["key"]) if optstring in validOpts: option = parser.get_option(optstring) - value = line.get('value', '') - boolean_value = value.lower() in ('true','yes','1') if value else False - if option.action == 'store_true': + value = line.get("value", "") + boolean_value = ( + value.lower() in ("true", "yes", "1") if value else False + ) + if option.action == "store_true": if boolean_value: args.append(optstring) - elif option.action == 'store_false': + elif option.action == "store_false": if not boolean_value: args.append(optstring) else: - args.extend([optstring, line['value'],]) + args.extend( + [ + optstring, + line["value"], + ] + ) else: log.debug("Unknown option: %s", optstring) return args + class _GlobalConfParserAdapter(object): def __init__(self, parser): self.parser = parser @@ -126,8 +144,9 @@ def _getGlobalConfigFileDefaults(self): args = _convertConfigLinesToArguments(self.parser, lines) try: self.parser._process_args([], args, options) - except (BadOptionError, OptionValueError) as err: - # Ignore it, we only care about our own options as defined in the parser + except (BadOptionError, OptionValueError): + # Ignore it, we only care about our own options as defined + # in the parser. pass return options.__dict__ @@ -143,24 +162,42 @@ def _loadConfigFile(self, filename): try: with open(filename) as file: for line in file: - if line.lstrip().startswith('#') or line.strip() == '': - lines.append(dict(type='comment', line=line)) + if line.lstrip().startswith("#") or line.strip() == "": + lines.append(dict(type="comment", line=line)) else: try: key, value = line.strip().split(None, 1) except ValueError: - lines.append(dict(type='option', line=line, key=line.strip(), value=None, option=None)) + lines.append( + dict( + type="option", + line=line, + key=line.strip(), + value=None, + option=None, + ) + ) else: - option = self.parser.get_option('--%s' % key) - lines.append(dict(type='option', line=line, key=key, value=value, option=option)) + option = self.parser.get_option("--%s" % key) + lines.append( + dict( + type="option", + line=line, + key=key, + value=value, + option=option, + ) + ) except IOError as e: - errorMessage = 'WARN: unable to read config file {filename} \ - -- skipping. ({exceptionName}: {exception})'.format( - filename=filename, - exceptionName=e.__class__.__name__, - exception=e + errorMessage = ( + "WARN: unable to read config file {filename} -- skipping. " + "({exceptionName}: {exception})".format( + filename=filename, + exceptionName=e.__class__.__name__, + exception=e, + ) ) - print >>sys.stderr, errorMessage + print(errorMessage, file=sys.stderr) return [] return lines diff --git a/Products/ZenUtils/MySqlZodbFactory.py b/Products/ZenUtils/MySqlZodbFactory.py index 84fe8368ea..639b5dc074 100644 --- a/Products/ZenUtils/MySqlZodbFactory.py +++ b/Products/ZenUtils/MySqlZodbFactory.py @@ -160,7 +160,7 @@ def buildOptions(self, parser): dest="zodb_cachesize", default=1000, type="int", - help="in memory cachesize default: 1000", + help="in memory cachesize default: %default", ) group.add_option( "--zodb-host", @@ -222,7 +222,7 @@ def buildOptions(self, parser): help=( "Specify the number of seconds a database connection will " "wait to acquire a database 'commit' lock before failing " - "(defaults to 30 seconds if not specified)." + "(defaults to %default seconds if not specified)." ), ) parser.add_option_group(group) diff --git a/Products/ZenUtils/ZCmdBase.py b/Products/ZenUtils/ZCmdBase.py index a1f309be24..8b831ae516 100644 --- a/Products/ZenUtils/ZCmdBase.py +++ b/Products/ZenUtils/ZCmdBase.py @@ -1,50 +1,49 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -__doc__="""ZenDaemon - -$Id: ZC.py,v 1.9 2004/02/16 17:19:31 edahl Exp $""" - -__version__ = "$Revision: 1.9 $"[11:-2] +from __future__ import absolute_import, print_function import time -from collections import Iterator +from collections import Iterator from threading import Lock + +from AccessControl.SecurityManagement import ( + newSecurityManager, + noSecurityManager, +) from twisted.internet import defer from zope.component import getUtility -from AccessControl.SecurityManagement import newSecurityManager -from AccessControl.SecurityManagement import noSecurityManager +from Products.ZenRelations.ZenPropertyManager import setDescriptors -from Products.ZenUtils.Utils import getObjByPath, zenPath -from Products.ZenUtils.ZodbFactory import IZodbFactoryLookup +from .Exceptions import ZentinelException +from .mysql import MySQLdb +from .Utils import getObjByPath, wait, zenPath +from .ZenDaemon import ZenDaemon +from .ZodbFactory import IZodbFactoryLookup -from Exceptions import ZentinelException -from ZenDaemon import ZenDaemon +defaultCacheDir = zenPath("var") -from Products.ZenRelations.ZenPropertyManager import setDescriptors -from Products.ZenUtils.mysql import MySQLdb -from Products.ZenUtils.Utils import wait -defaultCacheDir = zenPath('var') +class DataRootError(Exception): + pass -class DataRootError(Exception):pass -def login(context, name='admin', userfolder=None): +def login(context, name="admin", userfolder=None): """Logs in.""" if userfolder is None: userfolder = context.getPhysicalRoot().acl_users user = userfolder.getUserById(name) - if user is None: return - if not hasattr(user, 'aq_base'): + if user is None: + return + if not hasattr(user, "aq_base"): user = user.__of__(userfolder) newSecurityManager(None, user) return user @@ -86,6 +85,7 @@ def next(self): class ZCmdBase(ZenDaemon): + """Base class for daemons that need ZODB access.""" def __init__(self, noopts=0, app=None, keeproot=False): ZenDaemon.__init__(self, noopts, keeproot) @@ -102,52 +102,47 @@ def __init__(self, noopts=0, app=None, keeproot=False): def zodbConnect(self): connectionFactory = getUtility(IZodbFactoryLookup).get() - self.db, self.storage = connectionFactory.getConnection(**self.options.__dict__) + self.db, self.storage = connectionFactory.getConnection( + **self.options.__dict__ + ) - def login(self, name='admin', userfolder=None): + def login(self, name="admin", userfolder=None): """Logs in.""" login(self.dmd, name, userfolder) - def logout(self): """Logs out.""" noSecurityManager() - def getConnection(self): - """Return a wrapped app connection from the connection pool. - """ + """Return a wrapped app connection from the connection pool.""" if not self.db: raise ZentinelException( - "running inside zope can't open connections.") + "running inside zope can't open connections." + ) with self.poollock: - connection=self.db.open() - root=connection.root() - app=root['Application'] + connection = self.db.open() + root = connection.root() + app = root["Application"] app = self.getContext(app) app._p_jar.sync() return app - def closeAll(self): - """Close all connections in both free an inuse pools. - """ + """Close all connections in both free an inuse pools.""" self.db.close() - def opendb(self): - if self.app: return - self.connection=self.db.open() - root=self.connection.root() - app=root['Application'] + if self.app: + return + self.connection = self.db.open() + root = self.connection.root() + app = root["Application"] self.app = self.getContext(app) - @defer.inlineCallbacks def async_syncdb(self): - """ - Asynchronous version of the syncdb method. - """ + """Asynchronous version of the syncdb method.""" last_exc = None for delay in _RetryIterator(): try: @@ -156,7 +151,8 @@ def async_syncdb(self): last_exc = str(exc) self.log.warn( "Connection to ZODB interrupted, will try " - "to reconnect again in %.3f seconds.", delay + "to reconnect again in %.3f seconds.", + delay, ) # yield back to reactor for 'delay' seconds yield wait(delay) @@ -170,18 +166,20 @@ def async_syncdb(self): "Timed out trying to reconnect to ZODB: %s", last_exc ) - def syncdb(self): MAX_RETRY_TIME_MINUTES = 10 MAX_RETRY_DELAY_SECONDS = 30 retryStartedAt = None + def timedOut(): if retryStartedAt is None: return False else: - return retryStartedAt + MAX_RETRY_TIME_MINUTES * 60 < time.time() - + return ( + retryStartedAt + MAX_RETRY_TIME_MINUTES * 60 < time.time() + ) + retryMultiplier = 1.618 retryDelay = 1 @@ -196,17 +194,21 @@ def timedOut(): self.log.exception(e) keepTrying = False break - + if retryDelay * retryMultiplier >= MAX_RETRY_DELAY_SECONDS: retryDelay = MAX_RETRY_DELAY_SECONDS else: retryDelay *= retryMultiplier - self.log.warn("Connection to ZODB interrupted, will try to reconnect again in %d seconds.", retryDelay) - + self.log.warn( + "Connection to ZODB interrupted, will try to " + "reconnect again in %d seconds.", + retryDelay, + ) + if retryStartedAt is None: retryStartedAt = time.time() - + try: time.sleep(retryDelay) except Exception as e: @@ -214,49 +216,46 @@ def timedOut(): else: keepTrying = False - def closedb(self): self.connection.close() - #self.db.close() + # self.db.close() self.app = None self.dataroot = None self.dmd = None - def getDataRoot(self): - if not self.app: self.opendb() + if not self.app: + self.opendb() if not self.dataroot: self.dataroot = getObjByPath(self.app, self.options.dataroot) self.dmd = self.dataroot - def getContext(self, app): from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.BaseRequest import RequestContainer + resp = HTTPResponse(stdout=None) env = { - 'SERVER_NAME':'localhost', - 'SERVER_PORT':'8080', - 'REQUEST_METHOD':'GET' - } + "SERVER_NAME": "localhost", + "SERVER_PORT": "8080", + "REQUEST_METHOD": "GET", + } req = HTTPRequest(None, env, resp) - return app.__of__(RequestContainer(REQUEST = req)) - + return app.__of__(RequestContainer(REQUEST=req)) def getDmdObj(self, path): - """return an object based on a path starting from the dmd""" - return getObjByPath(self.app, self.options.dataroot+path) - + """Return an object based on a path starting from the dmd.""" + return getObjByPath(self.app, self.options.dataroot + path) def findDevice(self, name): - """return a device based on its FQDN""" + """Return a device based on its FQDN.""" devices = self.dataroot.getDmdRoot("Devices") return devices.findDevice(name) def buildOptions(self): - """basic options setup sub classes can add more options here""" + """Basic options setup sub classes can add more options here.""" ZenDaemon.buildOptions(self) connectionFactory = getUtility(IZodbFactoryLookup).get() diff --git a/Products/ZenUtils/ZenDaemon.py b/Products/ZenUtils/ZenDaemon.py index 0a79d9d4df..5b73443802 100755 --- a/Products/ZenUtils/ZenDaemon.py +++ b/Products/ZenUtils/ZenDaemon.py @@ -7,29 +7,30 @@ # ############################################################################## +from __future__ import absolute_import, print_function -"""ZenDaemon - -Base class for making daemon programs -""" - -import re -import sys +import logging import os import pwd +import re +import signal import socket -import logging +import sys -from twisted.internet import defer -from twisted.python import log as twisted_log +from platform import system +from urllib import getproxies + +from twisted.internet import defer, reactor from twisted.logger import globalLogBeginner +from twisted.python import log as twisted_log from Products.ZenMessaging.audit import audit -from Products.ZenUtils.CmdBase import CmdBase -from Products.ZenUtils.Utils import zenPath, HtmlFormatter, binPath, setLogLevel -from Products.ZenUtils.Watchdog import Reporter from Products.Zuul.utils import safe_hasattr as hasattr -from Products.ZenUtils.dumpthreads import dump_threads + +from .CmdBase import CmdBase +from .dumpthreads import dump_threads +from .Utils import binPath, HtmlFormatter, setLogLevel, zenPath +from .Watchdog import Reporter # Daemon creation code below based on Recipe by Chad J. Schroeder # File mode creation mask of the daemon. @@ -46,9 +47,7 @@ class ZenDaemon(CmdBase): - """ - Base class for creating daemons - """ + """Base class for creating daemons.""" pidfile = None @@ -62,28 +61,32 @@ def __init__(self, noopts=0, keeproot=False): self.keeproot = keeproot self.reporter = None self.fqdn = socket.getfqdn() - from twisted.internet import reactor - reactor.addSystemEventTrigger('before', 'shutdown', self.sigTerm) + + reactor.addSystemEventTrigger("before", "shutdown", self.sigTerm) if not noopts: if self.options.daemon: self.changeUser() self.becomeDaemon() - if self.options.pidfile or self.options.daemon or self.options.watchdogPath: + if ( + self.options.pidfile + or self.options.daemon + or self.options.watchdogPath + ): try: self.writePidFile() except OSError: raise SystemExit( - "ERROR: unable to open PID file %s" - % (self.pidfile or "(unknown)",) - ) + "ERROR: unable to open PID file %s" + % (self.pidfile or "(unknown)",) + ) if self.options.watchdog and not self.options.watchdogPath: self.becomeWatchdog() - self.audit('Start') + self.audit("Start") def audit(self, action): - processName = re.sub(r'^.*/', '', sys.argv[0]) - daemon = re.sub('.py$', '', processName) - audit('Shell.Daemon.' + action, daemon=daemon) + processName = re.sub(r"^.*/", "", sys.argv[0]) + daemon = re.sub(".py$", "", processName) + audit("Shell.Daemon." + action, daemon=daemon) def convertSocketOption(self, optString): """ @@ -91,60 +94,62 @@ def convertSocketOption(self, optString): to a C-friendly command-line option for passing to zensocket. """ optString = optString.upper() - if '=' not in optString: # Assume boolean + if "=" not in optString: # Assume boolean flag = optString value = 1 else: - flag, value = optString.split('=', 1) + flag, value = optString.split("=", 1) try: value = int(value) except ValueError: self.log.warn( "The value %s for flag %s cound not be converted", - value, flag) + value, + flag, + ) return None # Check to see if we can find the option if flag not in dir(socket): - self.log.warn("The flag %s is not a valid socket option", - flag) + self.log.warn("The flag %s is not a valid socket option", flag) return None numericFlag = getattr(socket, flag) - return '--socketOpt=%s:%s' % (numericFlag, value) + return "--socketOpt=%s:%s" % (numericFlag, value) def openPrivilegedPort(self, *address): - """ - Execute under zensocket, providing the args to zensocket - """ + """Execute under zensocket, providing the args to zensocket.""" socketOptions = [] for optString in set(self.options.socketOption): arg = self.convertSocketOption(optString) if arg: socketOptions.append(arg) - zensocket = binPath('zensocket') - cmd = [zensocket, zensocket] + list(address) + socketOptions \ - + ['--', sys.executable] + sys.argv \ - + ['--useFileDescriptor=$privilegedSocket'] + zensocket = binPath("zensocket") + cmd = ( + [zensocket, zensocket] + + list(address) + + socketOptions + + ["--", sys.executable] + + sys.argv + + ["--useFileDescriptor=$privilegedSocket"] + ) self.log.debug(cmd) os.execlp(*cmd) def writePidFile(self): - """ - Write the PID file to disk - """ - pidfile = getattr(self.options, 'pidfile', '') + """Write the PID file to disk.""" + pidfile = getattr(self.options, "pidfile", "") if pidfile: myname = pidfile else: myname = sys.argv[0].split(os.sep)[-1] - if myname.endswith('.py'): + if myname.endswith(".py"): myname = myname[:-3] - monitor = getattr(self.options, 'monitor', 'localhost') + monitor = getattr(self.options, "monitor", "localhost") myname = "%s-%s.pid" % (myname, monitor) if self.options.watchdog and not self.options.watchdogPath: - self.pidfile = zenPath("var", 'watchdog-%s' % myname) + self.pidfile = zenPath("var", "watchdog-%s" % myname) else: self.pidfile = zenPath("var", myname) - fp = open(self.pidfile, 'w') + fp = open(self.pidfile, "w") mypid = str(os.getpid()) fp.write(mypid) fp.close() @@ -152,45 +157,52 @@ def writePidFile(self): @property def logname(self): - return getattr(self, 'mname', self.__class__.__name__) + return getattr(self, "mname", self.__class__.__name__) def setupLogging(self): - """ - Create formating for log entries and set default log level - """ - # Initialize twisted logging to go nowhere. (it may be re-enabled by SIGUSR1) - globalLogBeginner.beginLoggingTo([lambda x: None], redirectStandardIO=False, discardBuffer=True) + """Create formating for log entries and set default log level.""" + # Initialize twisted logging to go nowhere. + globalLogBeginner.beginLoggingTo( + [lambda x: None], redirectStandardIO=False, discardBuffer=True + ) # Setup python logging module rootLog = logging.getLogger() rootLog.setLevel(logging.WARN) - zenLog = logging.getLogger('zen') + zenLog = logging.getLogger("zen") zenLog.setLevel(self.options.logseverity) formatter = logging.Formatter( - '%(asctime)s %(levelname)s %(name)s: %(message)s') + "%(asctime)s %(levelname)s %(name)s: %(message)s" + ) if self.options.logfileonly: - #clear out existing handlers + # clear out existing handlers hdlrs = rootLog.handlers for hdlr in hdlrs: rootLog.removeHandler(hdlr) - if self.options.watchdogPath or self.options.daemon \ - or self.options.duallog or self.options.logfileonly: + if ( + self.options.watchdogPath + or self.options.daemon + or self.options.duallog + or self.options.logfileonly + ): logdir = self.checkLogpath() or zenPath("log") handler = logging.handlers.RotatingFileHandler( - filename=os.path.join( - logdir, '%s.log' % self.logname.lower()), - maxBytes=self.options.maxLogKiloBytes * 1024, - backupCount=self.options.maxBackupLogs + filename=os.path.join(logdir, "%s.log" % self.logname.lower()), + maxBytes=self.options.maxLogKiloBytes * 1024, + backupCount=self.options.maxBackupLogs, ) handler.setFormatter(formatter) rootLog.addHandler(handler) - if not (self.options.watchdogPath or self.options.daemon \ - or self.options.logfileonly): + if not ( + self.options.watchdogPath + or self.options.daemon + or self.options.logfileonly + ): # We are logging to the console # Find the stream handler and make it match our desired log level if self.options.weblog: @@ -201,15 +213,17 @@ def setupLogging(self): consoleHandler = logging.StreamHandler(sys.stderr) rootLog.addHandler(consoleHandler) - for handler in (h for h in rootLog.handlers - if isinstance(h, logging.StreamHandler)): + for handler in ( + h + for h in rootLog.handlers + if isinstance(h, logging.StreamHandler) + ): handler.setFormatter(formatter) - self.log = logging.getLogger('zen.%s' % self.logname) + self.log = logging.getLogger("zen.%s" % self.logname) # Allow the user to dynamically lower and raise the logging # level without restarts. - import signal try: signal.signal(signal.SIGUSR1, self.sighandler_USR1) except ValueError: @@ -223,11 +237,12 @@ def sighandler_USR1(self, signum, frame): Switch to debug level if signaled by the user, and to default when signaled again. """ + def getTwistedLogger(): loggerName = "zen.%s.twisted" % self.logname return twisted_log.PythonLoggingObserver(loggerName=loggerName) - log = logging.getLogger('zen') + log = logging.getLogger("zen") currentLevel = log.getEffectiveLevel() if currentLevel == logging.DEBUG: if self.options.logseverity == logging.DEBUG: @@ -236,13 +251,16 @@ def getTwistedLogger(): log.info( "Restoring logging level back to %s (%d)", logging.getLevelName(self.options.logseverity) or "unknown", - self.options.logseverity) + self.options.logseverity, + ) try: defer.setDebugging(False) getTwistedLogger().stop() except ValueError: # Twisted logging is somewhat broken - log.info("Unable to remove Twisted logger -- " - "expect Twisted logging to continue.") + log.info( + "Unable to remove Twisted logger -- " + "expect Twisted logging to continue." + ) else: setLogLevel(logging.DEBUG, "zen") log.info("Setting logging level to DEBUG") @@ -250,32 +268,30 @@ def getTwistedLogger(): getTwistedLogger().start() dump_threads(signum, frame) self._sigUSR1_called(signum, frame) - self.audit('Debug') + self.audit("Debug") def _sigUSR1_called(self, signum, frame): pass def changeUser(self): - """ - Switch identity to the appropriate Unix user - """ + """Switch identity to the appropriate Unix user.""" if not self.keeproot: try: cname = pwd.getpwuid(os.getuid())[0] pwrec = pwd.getpwnam(self.options.uid) os.setuid(pwrec.pw_uid) - os.environ['HOME'] = pwrec.pw_dir + os.environ["HOME"] = pwrec.pw_dir except (KeyError, OSError): - print >>sys.stderr, "WARN: user:%s not found running as:%s" \ - % (self.options.uid, cname) + print( + "WARN: user:%s not found running as:%s" + % (self.options.uid, cname), + file=sys.stderr, + ) def becomeDaemon(self): - """Code below comes from the excellent recipe by Chad J. Schroeder. - """ + """Code below comes from the excellent recipe by Chad J. Schroeder.""" # Workaround for http://bugs.python.org/issue9405 on Mac OS X - from platform import system - if system() == 'Darwin': - from urllib import getproxies + if system() == "Darwin": getproxies() try: pid = os.fork() @@ -307,14 +323,13 @@ def becomeDaemon(self): os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) # Duplicate standard input to standard output and standard error. - os.dup2(0, 1) # standard output (1) - os.dup2(0, 2) # standard error (2) + os.dup2(0, 1) # standard output (1) + os.dup2(0, 2) # standard error (2) def sigTerm(self, signum=None, frame=None): - """ - Signal handler for the SIGTERM signal. - """ + """Signal handler for the SIGTERM signal.""" from Products.ZenUtils.Utils import unused + unused(signum, frame) stop = getattr(self, "stop", None) if callable(stop): @@ -322,76 +337,78 @@ def sigTerm(self, signum=None, frame=None): if self.pidfile and os.path.exists(self.pidfile): self.log.info("Deleting PID file %s ...", self.pidfile) os.remove(self.pidfile) - self.log.info('Daemon %s shutting down', type(self).__name__) - self.audit('Stop') + self.log.info("Daemon %s shutting down", type(self).__name__) + self.audit("Stop") def watchdogCycleTime(self): """ - Return our cycle time (in minutes) + Return our cycle time (in minutes). @return: cycle time @rtype: integer """ # time between child reports: default to 2x the default cycle time default = 1200 - cycleTime = getattr(self.options, 'cycleTime', default) + cycleTime = getattr(self.options, "cycleTime", default) if not cycleTime: cycleTime = default return cycleTime def watchdogStartTimeout(self): """ - Return our watchdog start timeout (in minutes) + Return our watchdog start timeout (in minutes). @return: start timeout @rtype: integer """ # Default start timeout should be cycle time plus a couple of minutes default = self.watchdogCycleTime() + 120 - startTimeout = getattr(self.options, 'starttimeout', default) + startTimeout = getattr(self.options, "starttimeout", default) if not startTimeout: startTimeout = default return startTimeout def watchdogMaxRestartTime(self): """ - Return our watchdog max restart time (in minutes) + Return our watchdog max restart time (in minutes). @return: maximum restart time @rtype: integer """ default = 600 - maxTime = getattr(self.options, 'maxRestartTime', default) + maxTime = getattr(self.options, "maxRestartTime", default) if not maxTime: maxTime = default return default def becomeWatchdog(self): - """ - Watch the specified daemon and restart it if necessary. - """ + """Watch the specified daemon and restart it if necessary.""" from Products.ZenUtils.Watchdog import Watcher, log + log.setLevel(self.options.logseverity) cmd = sys.argv[:] - if '--watchdog' in cmd: - cmd.remove('--watchdog') - if '--daemon' in cmd: - cmd.remove('--daemon') + if "--watchdog" in cmd: + cmd.remove("--watchdog") + if "--daemon" in cmd: + cmd.remove("--daemon") - socketPath = '%s/.%s-watchdog-%d' % ( - zenPath('var'), self.__class__.__name__, os.getpid()) + socketPath = "%s/.%s-watchdog-%d" % ( + zenPath("var"), + self.__class__.__name__, + os.getpid(), + ) cycleTime = self.watchdogCycleTime() startTimeout = self.watchdogStartTimeout() maxTime = self.watchdogMaxRestartTime() - self.log.debug("Watchdog cycleTime=%d startTimeout=%d maxTime=%d", - cycleTime, startTimeout, maxTime) - - watchdog = Watcher(socketPath, - cmd, - startTimeout, - cycleTime, - maxTime) + self.log.debug( + "Watchdog cycleTime=%d startTimeout=%d maxTime=%d", + cycleTime, + startTimeout, + maxTime, + ) + + watchdog = Watcher(socketPath, cmd, startTimeout, cycleTime, maxTime) watchdog.run() sys.exit(0) @@ -405,45 +422,88 @@ def niceDoggie(self, timeout): self.reporter.niceDoggie(timeout) def buildOptions(self): - """ - Standard set of command-line options. - """ CmdBase.buildOptions(self) - self.parser.add_option('--uid', dest='uid', default="zenoss", - help='User to become when running default:zenoss') - self.parser.add_option('-c', '--cycle', dest='cycle', - action="store_true", default=False, - help="Cycle continuously on cycleInterval from Zope") - self.parser.add_option('-D', '--daemon', default=False, - dest='daemon', action="store_true", - help="Launch into the background") - self.parser.add_option('--duallog', default=False, - dest='duallog', action="store_true", - help="Log to console and log file") - self.parser.add_option('--logfileonly', default=False, - dest='logfileonly', action="store_true", - help="Log to log file and not console") - self.parser.add_option('--weblog', default=False, - dest='weblog', action="store_true", - help="output log info in HTML table format") - self.parser.add_option('--watchdog', default=False, - dest='watchdog', action="store_true", - help="Run under a supervisor which will restart it") - self.parser.add_option('--watchdogPath', default=None, - dest='watchdogPath', - help="The path to the watchdog reporting socket") - self.parser.add_option('--starttimeout', - dest='starttimeout', type="int", - help="Wait seconds for initial heartbeat") - self.parser.add_option('--socketOption', - dest='socketOption', default=[], action='append', - help="Set listener socket options. " - "For option details: man 7 socket") - self.parser.add_option('--heartbeattimeout', - dest='heartbeatTimeout', - type='int', - help="Set a heartbeat timeout in seconds for a daemon", - default=900) - self.parser.add_option('--pidfile', dest='pidfile', default="", - help='pidfile to save a pid number of a process') - + self.parser.add_option( + "--uid", + dest="uid", + default="zenoss", + help="User to become when running; default %default", + ) + self.parser.add_option( + "-c", + "--cycle", + dest="cycle", + action="store_true", + default=False, + help="Cycle continuously on cycleInterval from Zope", + ) + self.parser.add_option( + "-D", + "--daemon", + default=False, + dest="daemon", + action="store_true", + help="Launch into the background", + ) + self.parser.add_option( + "--duallog", + default=False, + dest="duallog", + action="store_true", + help="Log to console and log file", + ) + self.parser.add_option( + "--logfileonly", + default=False, + dest="logfileonly", + action="store_true", + help="Log to log file and not console", + ) + self.parser.add_option( + "--weblog", + default=False, + dest="weblog", + action="store_true", + help="output log info in HTML table format", + ) + self.parser.add_option( + "--watchdog", + default=False, + dest="watchdog", + action="store_true", + help="Run under a supervisor which will restart it", + ) + self.parser.add_option( + "--watchdogPath", + default=None, + dest="watchdogPath", + help="The path to the watchdog reporting socket", + ) + self.parser.add_option( + "--starttimeout", + dest="starttimeout", + type="int", + help="Wait seconds for initial heartbeat", + ) + self.parser.add_option( + "--socketOption", + dest="socketOption", + default=[], + action="append", + help="Set listener socket options. " + "For option details: man 7 socket", + ) + self.parser.add_option( + "--heartbeattimeout", + dest="heartbeatTimeout", + type="int", + default=900, + help="Set a heartbeat timeout in seconds for a daemon; " + "default %default", + ) + self.parser.add_option( + "--pidfile", + dest="pidfile", + default="", + help="pidfile to save a pid number of a process", + ) diff --git a/Products/ZenUtils/ZenScriptBase.py b/Products/ZenUtils/ZenScriptBase.py index 5493c7e0e4..ac7b2049e4 100644 --- a/Products/ZenUtils/ZenScriptBase.py +++ b/Products/ZenUtils/ZenScriptBase.py @@ -1,13 +1,12 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - """ZenScriptBase Scripts with classes who extend ZenScriptBase have a zope instance with a @@ -16,22 +15,27 @@ from zope.component import getUtility -from AccessControl.SecurityManagement import newSecurityManager -from AccessControl.SecurityManagement import noSecurityManager +from AccessControl.SecurityManagement import ( + newSecurityManager, + noSecurityManager, +) from transaction import commit -from Products.ZenUtils.Utils import getObjByPath, zenPath, set_context -from Products.ZenUtils.CmdBase import CmdBase -from Products.ZenUtils.ZodbFactory import IZodbFactoryLookup from Products.ZenRelations.ZenPropertyManager import setDescriptors -from Products.ZenUtils.Exceptions import ZentinelException -defaultCacheDir = zenPath('var') +from .CmdBase import CmdBase +from .Exceptions import ZentinelException +from .Utils import getObjByPath, set_context, zenPath +from .ZodbFactory import IZodbFactoryLookup -class DataRootError(Exception):pass +defaultCacheDir = zenPath("var") -class ZenScriptBase(CmdBase): +class DataRootError(Exception): + pass + + +class ZenScriptBase(CmdBase): def __init__(self, noopts=0, app=None, connect=False, should_log=True): CmdBase.__init__(self, noopts, should_log=should_log) self.dataroot = None @@ -43,93 +47,86 @@ def __init__(self, noopts=0, app=None, connect=False, should_log=True): def connect(self): if not self.app: connectionFactory = getUtility(IZodbFactoryLookup).get() - self.db, self.storage = connectionFactory.getConnection(**self.options.__dict__) + self.db, self.storage = connectionFactory.getConnection( + **self.options.__dict__ + ) self.getDataRoot() self.login() - if getattr(self.dmd, 'propertyTransformers', None) is None: + if getattr(self.dmd, "propertyTransformers", None) is None: self.dmd.propertyTransformers = {} commit() setDescriptors(self.dmd) - - def login(self, name='admin', userfolder=None): + def login(self, name="admin", userfolder=None): """Logs in.""" if userfolder is None: userfolder = self.app.acl_users user = userfolder.getUserById(name) - if user is None: return - if not hasattr(user, 'aq_base'): + if user is None: + return + if not hasattr(user, "aq_base"): user = user.__of__(userfolder) newSecurityManager(None, user) - def logout(self): """Logs out.""" noSecurityManager() - def getConnection(self): - """Return a wrapped app connection from the connection pool. - """ + """Return a wrapped app connection from the connection pool.""" if not self.db: raise ZentinelException( - "running inside zope can't open connections.") + "running inside zope can't open connections." + ) with self.poollock: - connection=self.db.open() - root=connection.root() - app=root['Application'] + connection = self.db.open() + root = connection.root() + app = root["Application"] app = set_context(app) app._p_jar.sync() return app - def closeAll(self): - """Close all connections in both free an inuse pools. - """ + """Close all connections in both free an inuse pools.""" self.db.close() - def opendb(self): - if self.app: return - self.connection=self.db.open() - root=self.connection.root() - app = root['Application'] + if self.app: + return + self.connection = self.db.open() + root = self.connection.root() + app = root["Application"] self.app = set_context(app) self.app._p_jar.sync() - def syncdb(self): self.connection.sync() - def closedb(self): self.connection.close() - #self.db.close() + # self.db.close() self.app = None self.dataroot = None self.dmd = None - def getDataRoot(self): - if not self.app: self.opendb() + if not self.app: + self.opendb() if not self.dataroot: self.dataroot = getObjByPath(self.app, self.options.dataroot) self.dmd = self.dataroot - def getDmdObj(self, path): - """return an object based on a path starting from the dmd""" - return getObjByPath(self.app, self.options.dataroot+path) - + """Return an object based on a path starting from the dmd""" + return getObjByPath(self.app, self.options.dataroot + path) def findDevice(self, name): - """return a device based on its FQDN""" + """Return a device based on its FQDN""" devices = self.dataroot.getDmdRoot("Devices") return devices.findDevice(name) - def buildOptions(self): - """basic options setup sub classes can add more options here""" + """Basic options setup sub classes can add more options here""" CmdBase.buildOptions(self) connectionFactory = getUtility(IZodbFactoryLookup).get() diff --git a/Products/ZenUtils/ZodbFactory.py b/Products/ZenUtils/ZodbFactory.py index 1ec780b0e0..1ee8f9bfa1 100644 --- a/Products/ZenUtils/ZodbFactory.py +++ b/Products/ZenUtils/ZodbFactory.py @@ -1,38 +1,41 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2011, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## +from zope.interface import implementer, Interface +from zope.component import queryUtility + +from .GlobalConfig import globalConfToDict -__doc__="""ZodbConnection -""" -from Products.ZenUtils.GlobalConfig import globalConfToDict -from zope.interface import Interface -from zope.interface import implements -from zope.component import queryUtility class IZodbFactoryLookup(Interface): def get(name=None): - """Return the a ZODB connection Factory by name or look up in global.conf.""" + """ + Return the a ZODB connection Factory by name or look up in + global.conf. + """ + +@implementer(IZodbFactoryLookup) class ZodbFactoryLookup(object): - implements(IZodbFactoryLookup) def get(self, name=None): - """Return the ZODB connection factory by name or look up in global.conf.""" + """ + Return the ZODB connection factory by name or look up in global.conf. + """ if name is None: settings = globalConfToDict() - name = settings.get('zodb-db-type', 'mysql') + name = settings.get("zodb-db-type", "mysql") connectionFactory = queryUtility(IZodbFactory, name) return connectionFactory class IZodbFactory(Interface): - def getZopeZodbConf(): """Return a zope.conf style stanza for the zodb connection.""" diff --git a/Products/ZenUtils/config.py b/Products/ZenUtils/config.py index 833ce238fe..8c42e44449 100644 --- a/Products/ZenUtils/config.py +++ b/Products/ZenUtils/config.py @@ -1,51 +1,52 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2010, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -__doc__ = """ +""" Zenoss config parsers. -There are mutiple stages to config parsing. Parsing is split into stages so that -we can validate a whole config file and possibly rebuild it to correct errors. +There are mutiple stages to config parsing. Parsing is split into stages so +that we can validate a whole config file and possibly rebuild it to correct +errors. The stages are: -* Parse - Split the config file in to ConfigLine types while maintaining line order, comments, and new lines +* Parse - Split the config file in to ConfigLine types while maintaining line + order, comments, and new lines * Validate - Check that all lines are valid * Report - Investigate why a line might be invalid (ex: invalid key format) * Load - Get a config object back * Write - An optional stage to write the config back to a file """ +from __future__ import absolute_import, print_function + import re + class ConfigError(Exception): - """ - Error for problems parsing config files. - """ - pass + """Error for problems parsing config files.""" + class ConfigLineError(ConfigError): - """ - Error for problems parsing config files with line context. - """ + """Error for problems parsing config files with line context.""" + def __init__(self, message, lineno): super(ConfigLineError, self).__init__(message) self.lineno = lineno def __str__(self): - return '%s on line %d' % (self.message, self.lineno) + return "%s on line %d" % (self.message, self.lineno) + class ConfigErrors(ConfigError): - """ - A group of errors while parsing config. - """ + """A group of errors while parsing config.""" + def __init__(self, message, errors): super(ConfigErrors, self).__init__(message) self.errors = errors @@ -55,14 +56,17 @@ def __str__(self): for error in self.errors: output.append(str(error)) - return '\n - '.join(output) + return "\n - ".join(output) + class InvalidKey(ConfigError): pass + class ConfigLineKeyError(ConfigLineError): pass + class Config(dict): """ A bunch of configuration settings. Uses dictionary access, @@ -70,6 +74,7 @@ class Config(dict): Provides some Convenience functions for different types. """ + def __getattr__(self, attr): return self[attr] @@ -82,7 +87,7 @@ def getbool(self, key, default=None): If key doesn't exist, returns `default`. """ try: - return self[key].lower() in ('true', 'yes', 'y', '1') + return self[key].lower() in ("true", "yes", "y", "1") except KeyError: return default @@ -112,10 +117,10 @@ def getfloat(self, key, default=None): except (KeyError, ValueError): return default + class ConfigLine(object): - """ - Abstract class that represents a single line in the config. - """ + """Abstract class that represents a single line in the config.""" + def __init__(self, line): self.line = line @@ -126,52 +131,48 @@ def __str__(self): def setting(self): """ Return a key, value tuple if this line represents a setting. - Implemented in base classes. """ - return None + return NotImplemented @classmethod def parse(cls, line): """ - Returns an instance of cls if this class can parse this line. Otherwise returns None. - Implemented in base classes. + Returns an instance of cls if this class can parse this line. + Otherwise returns None. """ - return None + return NotImplemented @classmethod def checkError(cls, line, lineno): """ - Checks the string for possible matches, considers why it doesn't match exactly if it's close - and returns a ConfigLineError. - Implemented in base classes. + Checks the string for possible matches, considers why it doesn't match + exactly if it's close and returns a ConfigLineError. """ - return None + return NotImplemented + class SettingLine(ConfigLine): - """ - Represents a config line with a `key = value` pair. - """ - _regexp = re.compile(r'^(?P[a-z]+([a-z\d_]|-[a-z\d_])*)\s*(?P(=|:|\s)*)\s*(?P.*)$', re.I) + """Represents a config line with a `key = value` pair.""" + + _regexp = re.compile( + r"^(?P[a-z]+([a-z\d_]|-[a-z\d_])*)" # key + r"\s*(?P(=|:|\s)*)" # delimiter + r"\s*(?P.*)$", # value + re.I, + ) - def __init__(self, key, value=None, delim='='): + def __init__(self, key, value=None, delim="="): self.key = key self.value = value self.delim = delim + def __str__(self): + return "{key} {delim} {value}".format(**self.__dict__) + @property def setting(self): return self.key, self.value - def __str__(self): - return '{key} {delim} {value}'.format(**self.__dict__) - - @classmethod - def checkError(cls, line, lineno): - match = re.match(r'^(?P.+?)\s*(?P(=|:|\s)+)\s*(?P.+)$', line, re.I) - if match and not cls._regexp.match(line): - return ConfigLineKeyError('Invalid key "%s"' % match.groupdict()['key'], lineno) - - @classmethod def parse(cls, line): match = cls._regexp.match(line) @@ -179,33 +180,75 @@ def parse(cls, line): data = match.groupdict() return cls(**data) + @classmethod + def checkError(cls, line, lineno): + match = re.match( + r"^(?P.+?)\s*(?P(=|:|\s)+)\s*(?P.+)$", + line, + re.I, + ) + if match and not cls._regexp.match(line): + return ConfigLineKeyError( + 'Invalid key "%s"' % match.groupdict()["key"], lineno + ) + + class CommentLine(ConfigLine): + @property + def setting(self): + return None + @classmethod def parse(cls, line): - if line.startswith('#'): + if line.startswith("#"): return cls(line[1:].strip()) + @classmethod + def checkError(cls, line, lineno): + return None + def __str__(self): - return '# %s' % self.line + return "# %s" % self.line + class EmptyLine(ConfigLine): def __init__(self): pass + @property + def setting(self): + return None + @classmethod def parse(cls, line): - if line == '': + if line == "": return cls() + @classmethod + def checkError(cls, line, lineno): + return None + def __str__(self): - return '' + return "" + class InvalidLine(ConfigLine): """ Default line if no other ConfigLines matched. Assumed to be invalid input. """ - pass + @property + def setting(self): + return None + + @classmethod + def parse(cls, line): + return None + + @classmethod + def checkError(cls, line, lineno): + return None + class ConfigFile(object): """ @@ -236,7 +279,9 @@ def __init__(self, file): @param file file-like-object """ self.file = file - self.filename = self.file.name if hasattr(self.file, 'name') else 'Unknown' + self.filename = ( + self.file.name if hasattr(self.file, "name") else "Unknown" + ) self._lines = None def _parseLine(self, line): @@ -248,7 +293,6 @@ def _parseLine(self, line): return self._invalidLineType(cleanedLine) - def _checkLine(self, line, lineno): cleanedLine = line.strip() for type in self._lineTypes: @@ -259,8 +303,8 @@ def _checkLine(self, line, lineno): def parse(self): """ Parse a config file which has key-value pairs.Returns a list of config - line information. This line information can be used to accuratly recreate - the config without losing comments or invalid data. + line information. This line information can be used to accuratly + recreate the config without losing comments or invalid data. """ if self._lines is None: self._lines = [] @@ -277,7 +321,7 @@ def write(self, file): @param file file-like-object """ for line in self: - file.write(str(line) + '\n') + file.write(str(line) + "\n") def validate(self): """ @@ -293,10 +337,18 @@ def validate(self): if error: errors.append(error) else: - errors.append(ConfigLineError('Unexpected config line "%s"' % line.line, lineno + 1)) + errors.append( + ConfigLineError( + 'Unexpected config line "%s"' % line.line, + lineno + 1, + ) + ) if errors: - raise ConfigErrors('There were errors parsing the config "%s".' % self.filename, errors) + raise ConfigErrors( + 'There were errors parsing the config "%s".' % self.filename, + errors, + ) def __iter__(self): for line in self.parse(): @@ -307,6 +359,7 @@ def items(self): if line.setting: yield line.setting + class Parser(object): def __call__(self, file): configFile = ConfigFile(file) @@ -315,14 +368,16 @@ def __call__(self, file): class ConfigLoader(object): - """ - Lazily load the config when requested. - """ + """Lazily load the config when requested.""" + def __init__(self, config_files, config=Config, parser=Parser()): """ - @param config Config The config instance or class to load data into. Must support update which accepts an iterable of (key, value). - @param parser Parser The parser to use to parse the config files. Must be a callable and return an iterable of (key, value). - @param config_files list A list of config file names to parse in order. + :param config Config The config instance or class to load data into. + Must support update which accepts an iterable of (key, value). + :param parser Parser The parser to use to parse the config files. + Must be a callable and return an iterable of (key, value). + :param config_files list A list of config file names to + parse in order. """ if not isinstance(config_files, list): config_files = [config_files] @@ -333,32 +388,29 @@ def __init__(self, config_files, config=Config, parser=Parser()): self._config = None def load(self): - """ - Load the config_files into an instance of config_class - """ + """Load the config_files into an instance of config_class.""" if isinstance(self.config, type): self._config = self.config() else: self._config = self.config if not self.config_files: - raise ConfigError('Config loader has no config files to load.') + raise ConfigError("Config loader has no config files to load.") for file in self.config_files: - if not hasattr(file, 'read') and isinstance(file, basestring): + if not hasattr(file, "read") and isinstance(file, basestring): # Look like a file name, open it - with open(file, 'r') as fp: + with open(file, "r") as fp: options = self.parser(fp) else: options = self.parser(file) - self._config.update(options) - + # self._config.update(options) + for k, v in options: + self._config[k] = v def __call__(self): - """ - Lazily load the config file. - """ + """Lazily load the config file.""" if self._config is None: self.load() From 0a49ddad0ef0460c0828ccb0a8e5755721bb553f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 14 Aug 2023 11:48:05 -0500 Subject: [PATCH 022/147] Fixed up PBDaemon help, refactored conf file creation. --- Products/ZenHub/PBDaemon.py | 62 ++++------ Products/ZenUtils/CmdBase.py | 228 ++++++++++++----------------------- 2 files changed, 100 insertions(+), 190 deletions(-) diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index a832fa39d9..77d27f9e06 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -1294,48 +1294,53 @@ def _signalZenHubAnswering(self, answering): def buildOptions(self): ZenDaemon.buildOptions(self) - self.parser.add_option( "--hubhost", dest="hubhost", default=DEFAULT_HUB_HOST, - help="Host of zenhub daemon." " Default is %s." % DEFAULT_HUB_HOST, + help="Host of zenhub daemon; default %default", ) self.parser.add_option( "--hubport", dest="hubport", type="int", default=DEFAULT_HUB_PORT, - help="Port zenhub listens on." "Default is %s." % DEFAULT_HUB_PORT, + help="Port zenhub listens on; default %default", ) self.parser.add_option( "--hubusername", dest="hubusername", default=DEFAULT_HUB_USERNAME, - help="Username for zenhub login." - " Default is %s." % DEFAULT_HUB_USERNAME, + help="Username for zenhub login; default %default", ) self.parser.add_option( "--hubpassword", dest="hubpassword", default=DEFAULT_HUB_PASSWORD, - help="Password for zenhub login." - " Default is %s." % DEFAULT_HUB_PASSWORD, + help="Password for zenhub login; default %default", ) self.parser.add_option( "--monitor", dest="monitor", default=DEFAULT_HUB_MONITOR, help="Name of monitor instance to use for" - " configuration. Default is %s." % DEFAULT_HUB_MONITOR, + " configuration; default %default", ) self.parser.add_option( "--initialHubTimeout", dest="hubtimeout", type="int", default=30, - help="Initial time to wait for a ZenHub " "connection", + help="Initial time to wait for a ZenHub connection", + ) + self.parser.add_option( + "--zenhubpinginterval", + dest="zhPingInterval", + default=120, + type="int", + help="How often to ping zenhub", ) + self.parser.add_option( "--allowduplicateclears", dest="allowduplicateclears", @@ -1344,33 +1349,27 @@ def buildOptions(self): help="Send clear events even when the most " "recent event was also a clear event.", ) - self.parser.add_option( "--duplicateclearinterval", dest="duplicateclearinterval", default=0, type="int", - help=( - "Send a clear event every [DUPLICATECLEARINTEVAL] " "events." - ), + help="Send a clear event every DUPLICATECLEARINTEVAL events.", ) - self.parser.add_option( "--eventflushseconds", dest="eventflushseconds", default=5.0, type="float", - help="Seconds between attempts to flush " "events to ZenHub.", + help="Seconds between attempts to flush events to ZenHub.", ) - self.parser.add_option( "--eventflushchunksize", dest="eventflushchunksize", default=50, type="int", - help="Number of events to send to ZenHub" "at one time", + help="Number of events to send to ZenHub at one time", ) - self.parser.add_option( "--maxqueuelen", dest="maxqueuelen", @@ -1378,7 +1377,6 @@ def buildOptions(self): type="int", help="Maximum number of events to queue", ) - self.parser.add_option( "--queuehighwatermark", dest="queueHighWaterMark", @@ -1387,14 +1385,6 @@ def buildOptions(self): help="The size, in percent, of the event queue " "when event pushback starts", ) - self.parser.add_option( - "--zenhubpinginterval", - dest="zhPingInterval", - default=120, - type="int", - help="How often to ping zenhub", - ) - self.parser.add_option( "--disable-event-deduplication", dest="deduplicate_events", @@ -1411,9 +1401,8 @@ def buildOptions(self): default=publisher.defaultRedisPort ), help="redis connection string: " - "redis://[hostname]:[port]/[db], default: %default", + "redis://[hostname]:[port]/[db]; default: %default", ) - self.parser.add_option( "--metricBufferSize", dest="metricBufferSize", @@ -1435,13 +1424,6 @@ def buildOptions(self): default=publisher.defaultMaxOutstandingMetrics, help="Max Number of metrics to allow in redis", ) - self.parser.add_option( - "--disable-ping-perspective", - dest="pingPerspective", - help="Enable or disable ping perspective", - default=True, - action="store_false", - ) self.parser.add_option( "--writeStatistics", dest="writeStatistics", @@ -1449,3 +1431,11 @@ def buildOptions(self): default=30, help="How often to write internal statistics value in seconds", ) + + self.parser.add_option( + "--disable-ping-perspective", + dest="pingPerspective", + default=True, + action="store_false", + help="Enable or disable ping perspective", + ) diff --git a/Products/ZenUtils/CmdBase.py b/Products/ZenUtils/CmdBase.py index a0505d2923..b3bd621271 100644 --- a/Products/ZenUtils/CmdBase.py +++ b/Products/ZenUtils/CmdBase.py @@ -15,6 +15,7 @@ import os.path import re import sys +import textwrap from copy import copy from optparse import ( @@ -45,6 +46,18 @@ ) +# List of options to not include when generating a config file. +_OPTIONS_TO_IGNORE = ( + "", + "configfile", + "genconf", + "genxmlconfigs", + "genxmltable", + "help", + "version", +) + + class DMDError: pass @@ -387,7 +400,6 @@ def validateConfigFile( commented out. @type correctErrors: boolean """ - output = [] errors = [] validLines = [] @@ -426,11 +438,7 @@ def validateConfigFile( for lineno, message in errors: print( "INFO: Commenting out %s on line %d in %s" - % ( - message, - lineno, - filename, - ), + % (message, lineno, filename), file=sys.stderr, ) @@ -441,11 +449,7 @@ def validateConfigFile( for lineno, message in errors: print( "WARN: %s on line %d in %s" - % ( - message, - lineno, - filename, - ), + % (message, lineno, filename), file=sys.stderr, ) @@ -491,9 +495,7 @@ def setupLogging(self): logging.basicConfig() def checkLogpath(self): - """ - Validate the logpath is valid. - """ + """Validate the logpath is valid.""" if not self.options.logpath: return None else: @@ -518,65 +520,30 @@ def pretty_print_config_comment(self, comment): Quick and dirty pretty printer for comments that happen to be longer than can comfortably be seen on the display. """ - max_size = 40 - # - # As a heuristic we'll accept strings that are +- text_window - # size in length. - # - text_window = 5 - - if len(comment) <= max_size + text_window: - return comment - - # - # First, take care of embedded newlines and expand them out to - # array entries. - # - new_comment = [] - all_lines = comment.split("\n") - for line in all_lines: - if len(line) <= max_size + text_window: - new_comment.append(line) - continue - - start_position = max_size - text_window - while len(line) > max_size + text_window: - index = line.find(" ", start_position) - if index > 0: - new_comment.append(line[0:index]) - line = line[index:] - - else: - if start_position == 0: - # - # If we get here it means that the line is just one - # big string with no spaces in it. There's nothing - # that we can do except print it out. Doh! - # - new_comment.append(line) - break - - # - # Okay, haven't found anything to split on - # -- go back and try again - # - start_position = start_position - text_window - if start_position < 0: - start_position = 0 - - else: - new_comment.append(line) - - return "\n# ".join(new_comment) + new_comment = textwrap.wrap(comment, width=75) + return "# " + "\n# ".join(new_comment) + + def _get_default_value(self, parser, opt): + default_value = parser.defaults.get(opt.dest) + if default_value is NO_DEFAULT or default_value is None: + return "" + return str(default_value) + + def _get_help_text(self, opt, default_value): + if "%default" in opt.help: + return opt.help.replace("%default", default_value) + default_text = "" + if default_value != "": + default_text = " [default %s]" % (default_value,) + return opt.help + default_text def generate_configs(self, parser, options): """ - Create a configuration file based on the long-form of the option names + Create a configuration file based on the long-form of the option names. :param parser: an optparse parser object which contains defaults, help :param options: parsed options list containing actual values """ - # # Header for the configuration file # @@ -589,72 +556,54 @@ def generate_configs(self, parser, options): # Configuration file for %s # # To enable a particular option, uncomment the desired entry. -# -# Parameter Setting -# --------- -------""" - % (daemon_name) +#""" + % (daemon_name,) ) - options_to_ignore = ("help", "version", "", "genconf", "genxmltable") - # # Create an entry for each of the command line flags # # NB: Ideally, this should print out only the option parser dest # entries, rather than the command line options. # - import re - for opt in getAllParserOptionsGen(parser): if opt.help is SUPPRESS_HELP: continue # - # Get rid of the short version of the command + # Don't include items in the ignore list # option_name = re.sub(r".*/--", "", "%s" % opt) - - # - # And what if there's no short version? - # option_name = re.sub(r"^--", "", "%s" % option_name) - - # - # Don't display anything we shouldn't be displaying - # - if option_name in options_to_ignore: + if option_name in _OPTIONS_TO_IGNORE: continue # # Find the actual value specified on the command line, if any, # and display it # + default_value = self._get_default_value(parser, opt) + help_text = self._get_help_text(opt, default_value) + description = self.pretty_print_config_comment(help_text) value = getattr(parser.values, opt.dest) + if value is None: + value = default_value - default_value = parser.defaults.get(opt.dest) - if default_value is NO_DEFAULT or default_value is None: - default_value = "" - - if "%default" in opt.help: - help_txt = opt.help.replace("%default", str(default_value)) - else: - default_string = "" - if default_value != "": - default_string = ", default: " + str(default_value) - help_txt = opt.help + default_string - - comment = self.pretty_print_config_comment(help_txt) + comment_char = "#" if str(value) == str(default_value) else "" # # NB: I would prefer to use tabs to separate the parameter name # and value, but I don't know that this would work. # print( - """# -# %s -#%s %s""" - % (comment, option_name, value) + "\n".join( + ( + "#", + description, + "%s%s %s" % (comment_char, option_name, value), + ) + ) ) # @@ -689,13 +638,13 @@ def generate_xml_table(self, parser, options): xmlns:html="http://www.w3.org/1999/xhtml" xmlns:db="http://docbook.org/ns/docbook" - xml:id="%s.options" + xml:id="{name}.options" > -%s Options +{name} Options - + @@ -706,17 +655,11 @@ def generate_xml_table(self, parser, options): -""" # noqa E501 - % ( - daemon_name, - daemon_name, - daemon_name, - daemon_name, +""".format( # noqa E501 + name=daemon_name ) ) - options_to_ignore = ("help", "version", "", "genconf", "genxmltable") - # # Create an entry for each of the command line flags # @@ -747,12 +690,16 @@ def generate_xml_table(self, parser, options): # option_name = re.sub(r".*/--", "", "%s" % opt) option_name = re.sub(r"^--", "", "%s" % option_name) - if option_name in options_to_ignore: + if option_name in _OPTIONS_TO_IGNORE: continue - default_value = parser.defaults.get(opt.dest) - if default_value is NO_DEFAULT or default_value is None: - default_value = "" + default_value = self._get_default_value(parser, opt) + + if "%default" in opt.help: + comment = opt.help.replace("%default", default_value) + else: + comment = opt.help + default_string = "" if default_value != "": default_string = ( @@ -761,7 +708,7 @@ def generate_xml_table(self, parser, options): + "\n" ) - comment = self.pretty_print_config_comment(opt.help) + # comment = self.pretty_print_config_comment(opt.help) # # TODO: Determine the variable name used and display the @@ -774,13 +721,8 @@ def generate_xml_table(self, parser, options): %s %s - -""" - % ( - all_options, - comment, - default_string, - ) +""" + % (all_options, comment, default_string) ) else: @@ -793,13 +735,8 @@ def generate_xml_table(self, parser, options): %s %s - -""" - % ( - all_options, - comment, - default_string, - ) +""" + % (all_options, comment, default_string) ) # @@ -839,15 +776,6 @@ def generate_xml_configs(self, parser, options): % (export_date, daemon_name) ) - options_to_ignore = ( - "help", - "version", - "", - "genconf", - "genxmltable", - "genxmlconfigs", - ) - # # Create an entry for each of the command line flags # @@ -863,14 +791,11 @@ def generate_xml_configs(self, parser, options): # option_name = re.sub(r".*/--", "", "%s" % opt) option_name = re.sub(r"^--", "", "%s" % option_name) - if option_name in options_to_ignore: + if option_name in _OPTIONS_TO_IGNORE: continue - default_value = parser.defaults.get(opt.dest) - if default_value is NO_DEFAULT or default_value is None: - default_string = "" - else: - default_string = str(default_value) + default_value = self._get_default_value(parser, opt) + help_text = quote(self._get_help_text(opt, default_value)) # # TODO: Determine the variable name used and display the @@ -880,12 +805,7 @@ def generate_xml_configs(self, parser, options): print( """
%s Daemons%s options{name} Daemons{name} options
") html.append("") + def evsummarycell(ev): - if ev[1]-ev[2]>=0: klass = '%s empty thin' % ev[0] - else: klass = '%s thin' % ev[0] + if ev[1] - ev[2] >= 0: + klass = "%s empty thin" % ev[0] + else: + klass = "%s thin" % ev[0] h = '' % ( - klass, ev[1], ev[2]) + klass, + ev[1], + ev[2], + ) return h + info = self.getEventSummary(severity) html += map(evsummarycell, info) - html.append('
%s/%s
') - return '\n'.join(html) + html.append("") + return "\n".join(html) def getDataForJSON(self, minSeverity=0): """ Returns data ready for serialization """ - url, classurl = map(urlquote, - (self.getDeviceUrl(), self.getDeviceClassPath())) + url, classurl = map( + urlquote, (self.getDeviceUrl(), self.getDeviceClassPath()) + ) id = '%s' % ( - url, self.titleOrId()) + url, + self.titleOrId(), + ) ip = self.getDeviceIp() if self.checkRemotePerm(ZEN_VIEW, self.deviceClass()): - path = '%s' % (classurl,classurl) + path = '%s' % ( + classurl, + classurl, + ) else: path = classurl prod = self.getProdState() @@ -2088,20 +2450,22 @@ def zenPropertyOptions(self, propname): """ Returns a list of possible options for a given zProperty """ - if propname == 'zCollectorPlugins': + if propname == "zCollectorPlugins": from Products.DataCollector.Plugins import loadPlugins + return sorted(ldr.pluginName for ldr in loadPlugins(self.dmd)) - if propname == 'zCommandProtocol': - return ['ssh', 'telnet'] - if propname == 'zSnmpVer': - return ['v1', 'v2c', 'v3'] - if propname == 'zSnmpAuthType': - return ['', 'MD5', 'SHA'] - if propname == 'zSnmpPrivType': - return ['', 'DES', 'AES'] + if propname == "zCommandProtocol": + return ["ssh", "telnet"] + if propname == "zSnmpVer": + return ["v1", "v2c", "v3"] + if propname == "zSnmpAuthType": + return ["", "MD5", "SHA"] + if propname == "zSnmpPrivType": + return ["", "DES", "AES"] return ManagedEntity.zenPropertyOptions(self, propname) - security.declareProtected(ZEN_MANAGE_DEVICE, 'pushConfig') + security.declareProtected(ZEN_MANAGE_DEVICE, "pushConfig") + def pushConfig(self, REQUEST=None): """ This will result in a push of all the devices to live collectors @@ -2111,25 +2475,29 @@ def pushConfig(self, REQUEST=None): self._p_changed = True if REQUEST: messaging.IMessageSender(self).sendToBrowser( - 'Changes Pushed', - 'Changes to %s pushed to collectors.' % self.id + "Changes Pushed", + "Changes to %s pushed to collectors." % self.id, ) - audit('UI.Device.PushChanges', self) + audit("UI.Device.PushChanges", self) return self.callZenScreen(REQUEST) - security.declareProtected(ZEN_EDIT_LOCAL_TEMPLATES, 'bindTemplates') + security.declareProtected(ZEN_EDIT_LOCAL_TEMPLATES, "bindTemplates") + def bindTemplates(self, ids=(), REQUEST=None): """ This will bind available templates to the zDeviceTemplates @permission: ZEN_EDIT_LOCAL_TEMPLATES """ - result = self.setZenProperty('zDeviceTemplates', ids, REQUEST) + result = self.setZenProperty("zDeviceTemplates", ids, REQUEST) if REQUEST: - audit('UI.Device.BindTemplates', self, templates=ids) + audit("UI.Device.BindTemplates", self, templates=ids) return result - security.declareProtected(ZEN_EDIT_LOCAL_TEMPLATES, 'removeZDeviceTemplates') + security.declareProtected( + ZEN_EDIT_LOCAL_TEMPLATES, "removeZDeviceTemplates" + ) + def removeZDeviceTemplates(self, REQUEST=None): """ Deletes the local zProperty, zDeviceTemplates @@ -2139,14 +2507,19 @@ def removeZDeviceTemplates(self, REQUEST=None): for id in self.zDeviceTemplates: self.removeLocalRRDTemplate(id) if REQUEST: - audit('UI.Device.RemoveLocalTemplate', self, template=id) - from Products.ZenRelations.ZenPropertyManager import ZenPropertyDoesNotExist + audit("UI.Device.RemoveLocalTemplate", self, template=id) + from Products.ZenRelations.ZenPropertyManager import ( + ZenPropertyDoesNotExist, + ) + try: - return self.deleteZenProperty('zDeviceTemplates', REQUEST) + return self.deleteZenProperty("zDeviceTemplates", REQUEST) except ZenPropertyDoesNotExist: - if REQUEST: return self.callZenScreen(REQUEST) + if REQUEST: + return self.callZenScreen(REQUEST) + + security.declareProtected(ZEN_EDIT_LOCAL_TEMPLATES, "addLocalTemplate") - security.declareProtected(ZEN_EDIT_LOCAL_TEMPLATES, 'addLocalTemplate') def addLocalTemplate(self, id, REQUEST=None): """ Create a local template on a device @@ -2154,15 +2527,16 @@ def addLocalTemplate(self, id, REQUEST=None): @permission: ZEN_EDIT_LOCAL_TEMPLATES """ from Products.ZenModel.RRDTemplate import manage_addRRDTemplate + manage_addRRDTemplate(self, id) if id not in self.zDeviceTemplates: - self.bindTemplates(self.zDeviceTemplates+[id]) + self.bindTemplates(self.zDeviceTemplates + [id]) if REQUEST: messaging.IMessageSender(self).sendToBrowser( - 'Local Template Added', - 'Added template %s to %s.' % (id, self.id) + "Local Template Added", + "Added template %s to %s." % (id, self.id), ) - audit('UI.Device.AddLocalTemplate', self, template=id) + audit("UI.Device.AddLocalTemplate", self, template=id) return self.callZenScreen(REQUEST) def getAvailableTemplates(self): @@ -2170,32 +2544,39 @@ def getAvailableTemplates(self): Returns all available templates for this device """ # All templates defined on this device are available - templates = self.objectValues('RRDTemplate') + templates = self.objectValues("RRDTemplate") # Any templates available to the class that aren't overridden locally # are also available device_template_ids = set(t.id for t in templates) - templates.extend(t for t in self.deviceClass().getRRDTemplates() - if t.id not in device_template_ids) + templates.extend( + t + for t in self.deviceClass().getRRDTemplates() + if t.id not in device_template_ids + ) # filter before sorting - templates = filter(lambda t: isinstance(self, t.getTargetPythonClass()), templates) + templates = filter( + lambda t: isinstance(self, t.getTargetPythonClass()), templates + ) return sorted(templates, key=lambda x: x.id.lower()) def getSnmpV3EngineId(self): - return self.getProperty('zSnmpEngineId') + return self.getProperty("zSnmpEngineId") def setSnmpV3EngineId(self, value): - self.setZenProperty('zSnmpEngineId', value) + self.setZenProperty("zSnmpEngineId", value) + + security.declareProtected(ZEN_VIEW, "getLinks") - security.declareProtected(ZEN_VIEW, 'getLinks') - def getLinks(self, OSI_layer='3'): + def getLinks(self, OSI_layer="3"): """ Returns all Links on this Device's interfaces @permission: ZEN_VIEW """ - if OSI_layer=='3': + if OSI_layer == "3": from Products.ZenUtils.NetworkTree import getDeviceNetworkLinks + for link in getDeviceNetworkLinks(self): yield link else: @@ -2203,17 +2584,21 @@ def getLinks(self, OSI_layer='3'): for link in iface.links.objectValuesGen(): yield link - security.declareProtected(ZEN_VIEW, 'getXMLEdges') + security.declareProtected(ZEN_VIEW, "getXMLEdges") + def getXMLEdges(self, depth=3, filter="/", start=()): """ Gets XML """ - if not start: start=self.id - edges = NetworkTree.get_edges(self, depth, - withIcons=True, filter=filter) + if not start: + start = self.id + edges = NetworkTree.get_edges( + self, depth, withIcons=True, filter=filter + ) return edgesToXML(edges, start) - security.declareProtected(ZEN_VIEW, 'getPrettyLink') + security.declareProtected(ZEN_VIEW, "getPrettyLink") + @unpublished def getPrettyLink(self, target=None, altHref=""): """ @@ -2222,9 +2607,11 @@ def getPrettyLink(self, target=None, altHref=""): @rtype: HTML text @permission: ZEN_VIEW """ - template = ("
" - " " - "
%s") + template = ( + "
" + " " + "
%s" + ) icon = self.getIconPath() href = altHref if altHref else self.getPrimaryUrlPath() name = self.titleOrId() @@ -2234,8 +2621,11 @@ def getPrettyLink(self, target=None, altHref=""): if not self.checkRemotePerm(ZEN_VIEW, self): return rendered else: - return "%s" % \ - ('target=' + target if target else '', href, rendered) + return "%s" % ( + "target=" + target if target else "", + href, + rendered, + ) def osProcessClassMatchData(self): """ @@ -2244,14 +2634,16 @@ def osProcessClassMatchData(self): """ matchers = [] for pc in self.getDmdRoot("Processes").getSubOSProcessClassesSorted(): - matchers.append({ - 'includeRegex': pc.includeRegex, - 'excludeRegex': pc.excludeRegex, - 'replaceRegex': pc.replaceRegex, - 'replacement': pc.replacement, - 'primaryUrlPath': pc.getPrimaryUrlPath(), - 'primaryDmdId': pc.getPrimaryDmdId(), - }) + matchers.append( + { + "includeRegex": pc.includeRegex, + "excludeRegex": pc.excludeRegex, + "replaceRegex": pc.replaceRegex, + "replacement": pc.replacement, + "primaryUrlPath": pc.getPrimaryUrlPath(), + "primaryDmdId": pc.getPrimaryDmdId(), + } + ) return matchers @@ -2261,6 +2653,7 @@ def manageIpVersion(self): of the manageIp ip adddress """ from ipaddr import IPAddress + try: ip = self.getManageIp() return IPAddress(ip).version @@ -2272,8 +2665,9 @@ def manageIpVersion(self): def snmpwalkPrefix(self): """ - This method gets the ip address prefix used for this device when running - snmpwalk. + This method gets the ip address prefix used for this device + when running snmpwalk. + @rtype: string @return: Prefix used for snmwalk for this device """ @@ -2297,7 +2691,8 @@ def tracerouteCommand(self): Used by the user commands this returns which traceroute command this device should use. @rtype: string - @return "traceroute" or "traceroute6" depending on if the manageIp is ipv6 or not + @return "traceroute" or "traceroute6" depending on if the + manageIp is ipv6 or not. """ if self.manageIpVersion() == 6: return "traceroute6" @@ -2313,17 +2708,22 @@ def getStatus(self, statusclass=None, **kwargs): if statusclass is None: statusclass = self.zStatusEventClass - zep = getFacade('zep', self) + zep = getFacade("zep", self) try: event_filter = zep.createEventFilter( tags=[self.getUUID()], element_sub_identifier=[""], severity=[SEVERITY_CRITICAL], - status=[STATUS_NEW, STATUS_ACKNOWLEDGED, STATUS_SUPPRESSED], - event_class=filter(None, [self.zStatusEventClass])) + status=[ + STATUS_NEW, + STATUS_ACKNOWLEDGED, + STATUS_SUPPRESSED, + ], + event_class=filter(None, [self.zStatusEventClass]), + ) result = zep.getEventSummaries(0, filter=event_filter, limit=0) - return int(result['total']) + return int(result["total"]) except Exception: return None @@ -2336,32 +2736,38 @@ def _getPingStatus(self, statusclass): if not self.zPingMonitorIgnore and self.getManageIp(): # Override normal behavior - we only care if the manage IP is down - # need to add the ipinterface component id to search since we may be - # pinging interfaces and only care about status of the one that + # Need to add the ipinterface component id to search since we may + # be pinging interfaces and only care about status of the one that # matches the manage ip. This is potentially expensive element_sub_identifier = [""] - ifaces = self.getDeviceComponents(type='IpInterface') + ifaces = self.getDeviceComponents(type="IpInterface") for iface in ifaces: - if self.manageIp in [ip.partition("/")[0] for ip in iface.getIpAddresses()]: + if self.manageIp in [ + ip.partition("/")[0] for ip in iface.getIpAddresses() + ]: element_sub_identifier.append(iface.id) break - zep = getFacade('zep', self) - event_filter = zep.createEventFilter(tags=[self.getUUID()], - severity=[SEVERITY_WARNING,SEVERITY_ERROR,SEVERITY_CRITICAL], - status=[STATUS_NEW,STATUS_ACKNOWLEDGED, STATUS_SUPPRESSED], - element_sub_identifier=element_sub_identifier, - event_class=filter(None, [statusclass]), - details={EventProxy.DEVICE_IP_ADDRESS_DETAIL_KEY: self.getManageIp()}) + zep = getFacade("zep", self) + event_filter = zep.createEventFilter( + tags=[self.getUUID()], + severity=[SEVERITY_WARNING, SEVERITY_ERROR, SEVERITY_CRITICAL], + status=[STATUS_NEW, STATUS_ACKNOWLEDGED, STATUS_SUPPRESSED], + element_sub_identifier=element_sub_identifier, + event_class=filter(None, [statusclass]), + details={ + EventProxy.DEVICE_IP_ADDRESS_DETAIL_KEY: self.getManageIp() + }, + ) result = zep.getEventSummaries(0, filter=event_filter, limit=0) - return int(result['total']) + return int(result["total"]) else: return None def ipAddressAsInt(self): ip = self.getManageIp() if ip: - ip = ip.partition('/')[0] + ip = ip.partition("/")[0] if ip: return str(numbip(ip)) From 30801ff0ea3e43f92f45f57b349b456a6e597663 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 21 Sep 2023 15:08:55 -0500 Subject: [PATCH 027/147] Replace 'implements' with '@implementer' --- Products/ZenModel/Device.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Products/ZenModel/Device.py b/Products/ZenModel/Device.py index 0b91256ab7..29a55288f7 100644 --- a/Products/ZenModel/Device.py +++ b/Products/ZenModel/Device.py @@ -38,7 +38,7 @@ from ZODB.POSException import POSError from zope.component import subscribers from zope.event import notify -from zope.interface import implements +from zope.interface import implementer from Products.Jobber.jobs import FacadeMethodJob from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex @@ -262,6 +262,7 @@ class NoNetMask(Exception): pass +@implementer(IEventView, IGloballyIdentifiable) class Device( ManagedEntity, Commandable, @@ -277,8 +278,6 @@ class Device( enabled but maybe this will change. """ - implements(IEventView, IGloballyIdentifiable) - event_key = portal_type = meta_type = "Device" default_catalog = "deviceSearch" From dd713d63bea68decbb5777717c952bd331d1dd66 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 2 Oct 2023 15:11:54 -0500 Subject: [PATCH 028/147] Use load_zenpacks istead of ZenossStartup side-effects --- bin/zenglobalconf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/zenglobalconf b/bin/zenglobalconf index 19e597310b..83b8718983 100755 --- a/bin/zenglobalconf +++ b/bin/zenglobalconf @@ -143,8 +143,9 @@ def main(): print "--sync-zope-conf only valid with global.conf" sys.exit(1) # load zcml for the product - import Products.ZenossStartup + from Products.ZenUtils.zenpackload import load_zenpacks from Products.Five import zcml + load_zenpacks() zcml.load_site() # look up the utility from zope.component import getUtility From ceb07011a77373013c6439b2eba7190918098fd5 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 23 Oct 2023 14:02:12 -0500 Subject: [PATCH 029/147] Remove @deprecated decorators on Device methods that haven't been deprecated. --- Products/ZenModel/Device.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/Products/ZenModel/Device.py b/Products/ZenModel/Device.py index 29a55288f7..c2371b8eb3 100644 --- a/Products/ZenModel/Device.py +++ b/Products/ZenModel/Device.py @@ -758,11 +758,8 @@ def getSnmpConnInfo(self): return SnmpConnInfo(self) - @deprecated def getHWManufacturerName(self): """ - DEPRECATED - Return the hardware manufacturer name of this device. - @rtype: string @todo: Remove this method and remove the call from testDevice.py """ @@ -786,41 +783,29 @@ def getHWProductClass(self): if cls: return cls.titleOrId() - @deprecated def getHWProductKey(self): """ - DEPRECATED - Return the productKey of the device hardware. - @rtype: string @todo: Remove this method and remove the call from testDevice.py """ return self.hw.getProductKey() - @deprecated def getOSManufacturerName(self): """ - DEPRECATED - Return the OS manufacturer name of this device. - @rtype: string @todo: Remove this method and remove the call from testDevice.py """ return self.os.getManufacturerName() - @deprecated def getOSProductName(self): """ - DEPRECATED - Return the OS product name of this device. - @rtype: string @todo: Remove this method and remove the call from testDevice.py """ return self.os.getProductName() - @deprecated def getOSProductKey(self): """ - DEPRECATED - Return the productKey of the device OS. - @rtype: string @todo: Remove this method and remove the call from testDevice.py """ @@ -834,11 +819,8 @@ def setOSProductKey(self, prodKey, manufacturer=None): """ self.os.setProductKey(prodKey, manufacturer) - @deprecated def getHWTag(self): """ - DEPRECATED - Return the tag of the device HW. - @rtype: string @todo: remove this method and remove the call from testDevice.py """ @@ -868,11 +850,8 @@ def setHWSerialNumber(self, number): """ self.hw.serialNumber = number - @deprecated def getHWSerialNumber(self): """ - DEPRECATED - Return the hardware serial number. - @rtype: string @todo: Remove this method and remove the call from testDevice.py """ @@ -1622,7 +1601,6 @@ def setSnmpLastCollection(self, value=None): security.declareProtected(ZEN_CHANGE_DEVICE, "addManufacturer") - @deprecated def addManufacturer( self, newHWManufacturerName=None, @@ -1630,9 +1608,6 @@ def addManufacturer( REQUEST=None, ): """ - DEPRECATED - - Add either a hardware or software manufacturer to the database. - @permission: ZEN_CHANGE_DEVICE @todo: Doesn't really do work on a device object. Already exists on ZDeviceLoader @@ -1654,14 +1629,10 @@ def addManufacturer( security.declareProtected(ZEN_CHANGE_DEVICE, "setHWProduct") - @deprecated def setHWProduct( self, newHWProductName=None, hwManufacturer=None, REQUEST=None ): """ - DEPRECATED - - Adds a new hardware product - @permission: ZEN_CHANGE_DEVICE @todo: Doesn't really do work on a device object. Already exists on ZDeviceLoader @@ -1696,14 +1667,10 @@ def setHWProduct( security.declareProtected(ZEN_CHANGE_DEVICE, "setOSProduct") - @deprecated def setOSProduct( self, newOSProductName=None, osManufacturer=None, REQUEST=None ): """ - DEPRECATED - Adds a new os product - @permission: ZEN_CHANGE_DEVICE @todo: Doesn't really do work on a device object. Already exists on ZDeviceLoader @@ -1755,12 +1722,8 @@ def setLocation(self, locationPath, REQUEST=None): security.declareProtected(ZEN_CHANGE_DEVICE, "addLocation") - @deprecated def addLocation(self, newLocationPath, REQUEST=None): """ - DEPRECATED - Add a new location and relate it to this device - @todo: Doesn't really do work on a device object. Already exists on ZDeviceLoader """ From dda1ae614736597dbb4db4c8b0bc6f6e072c7e77 Mon Sep 17 00:00:00 2001 From: vsaliieva Date: Thu, 30 Nov 2023 16:24:09 +0000 Subject: [PATCH 030/147] ZEN-34587: cyberark load certificate test fails Fixes ZEN-34587. Back-port for ZEN-34468 from develop branch. --- Products/ZenCollector/tests/test_cyberark.py | 156 ++++++++++++++----- 1 file changed, 113 insertions(+), 43 deletions(-) diff --git a/Products/ZenCollector/tests/test_cyberark.py b/Products/ZenCollector/tests/test_cyberark.py index 2890543bcc..f198aa716b 100644 --- a/Products/ZenCollector/tests/test_cyberark.py +++ b/Products/ZenCollector/tests/test_cyberark.py @@ -495,61 +495,131 @@ def test_init(t): t.assertIsNone(prop.value) +# openssl genrsa -aes256 -passout pass:qwerty -out ca.pass.key 4096 +# openssl rsa -passin pass:qwerty -in ca.pass.key -out ca.key +# openssl req -new -x509 -days 3650 -key ca.key -out ca.crt rootCA_crt = """ -----BEGIN CERTIFICATE----- -MIIC1jCCAj+gAwIBAgIUDmS1sDHq5ZxY2xMz3OVPbT/LjfgwDQYJKoZIhvcNAQEL -BQAwfTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRleGFzMQ8wDQYDVQQHDAZBdXN0 -aW4xDzANBgNVBAoMBlplbm9zczEMMAoGA1UECwwDRGV2MQ8wDQYDVQQDDAZaZW5v -c3MxHTAbBgkqhkiG9w0BCQEWDmRldkB6ZW5vc3MuY29tMB4XDTIyMDEyNjE3MDc0 -NFoXDTI2MTIzMTE3MDc0NFowfTELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRleGFz -MQ8wDQYDVQQHDAZBdXN0aW4xDzANBgNVBAoMBlplbm9zczEMMAoGA1UECwwDRGV2 -MQ8wDQYDVQQDDAZaZW5vc3MxHTAbBgkqhkiG9w0BCQEWDmRldkB6ZW5vc3MuY29t -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQD5NUfsKhsfkDOQfiuJCzdk3GHD -A6J2ISD0cCRyhfqLWbu6Gz6yjmLMSrwqzp9xSPqbHTo3uC916aRdOREnOLeeNgMD -eHTQKbtEooNMXaeU0WwTHbWmsT6XI8tifAiMFsALsuZtXrObr1NFWPMSxOdrqnjg -FycFdbZB6Rvys1hiaQIDAQABo1MwUTAdBgNVHQ4EFgQU3RLbuadNNemGXzwMtv+P -+PytrgswHwYDVR0jBBgwFoAU3RLbuadNNemGXzwMtv+P+PytrgswDwYDVR0TAQH/ -BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQBXRCxYTdityAm0zK+MvpETpWZxNOdV -ZBFIohave+TAnpTyb8YpC1fCK/8dY4Q53yL/MNW9XosKI+5eQa+8X/FNEXv1TwNs -gHbYHHO7onDPDzkQoXBC0K65m8fSTsdbxazjG2UddyfWkI9wjESkE6yZjgtN52T3 -90Q7rR7mG9d9cA== +MIIFazCCA1OgAwIBAgIUbQjJ7ZePquLJ74Wi+WFWJTqChM8wDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA4MzAwNzAzNTdaFw0zMzA4 +MjcwNzAzNTdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCZlHDhq9m+eBuOgMAqREL3HQk0KJzJatYFSlj1zuFF +zJwDd6Q4LRxEu89PVf3l8xa09WY7Wa1t0NXXqndIzvh5gpha9uJ0I2hiq4IHBjql +8vEMLYeYWVdr9yePeOPF/OHnVRW3YWkj+G+cHgYweMMbWuLoxZHXyM+md5t2SmvH +4YrcgdcPD7Actl9GqIq4AMvqtu+X3W9jDyX6+S5TCgtcKHaBq0r3vSZ4BOn+zlgA +DejIjMyp8ws75vGrrP6aiP++Am4lVHnXV0EB3d1rx/WAH3Kf36uDwuD2+KRNwTVW +KJWkCUHUhp6GyZQT/OkuObROaar1DH3lPale0ka45JJlngFZQxXeHdpG4CSRD8pQ +j3WRGg1bmHx47m9lOaTqtmktjRXzGYNG/0eDwOQEs013unxBUw11gzL44W/AWqC8 +Hp3qZp2ZyzSLl+yrkKHcmgj3PpAcWtm/Vu0rMddjtkIIcXXf6nLOkpmDC+S6xIbc +Ksgd6ewy2tyxE5s3eNgKqPj4LJK0ANpDan/pVRpdQb0T5UUeNKCl4EeoZpwsHly6 +inltPqqZjwKOxqO037uBbc3gc/qacHBfb8yThm98PPR7A2C4BOwxNMvQ5e2Ey9+o +/w6qJNHOfX6W7YZhuf4OXBBMWj5LKoO/uVImg1fRATpGCwBK4kNqtVszWx6zy4bb +pwIDAQABo1MwUTAdBgNVHQ4EFgQU9lu+gYAb++EtfoAP3djV++c/SrYwHwYDVR0j +BBgwFoAU9lu+gYAb++EtfoAP3djV++c/SrYwDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAgEAKLtUhk7tP0f9fjzc8qnseG+QsmdObJuMd/x7m/h7GOjn +0VqOkE8hRJJXVIIDv8ZssK3d+MNhHKIHuH3bRfFtqopXXOxnLR5FvI4Z9t88po8V +75IreWyqBW2u3fCNzYTgxkqKR2aOc4TN6qTRtAS17BJybbpT8GMu8lQ3ubSAjVY7 +CWh3RxXalUw9vGQ9LIzeyASiOWRDXeeEIeuNcwPzXGjssGPQGjbR2Cbes88A3Sf5 +By8da/dlZMxQOtlryOgaLKmXg/P3x6DzCmhS2tWfBMQ23ifuegYylFPecqpJw8L7 +Atz8TmULt2raWk+rrzcBwcBnx+t5WtFT7SrhEOiBtA5BwWNprLi9XyFqXQBsMpog +t/vSlCT8MmnleCmaXvHk/+xqasHDxaowSibOjJHxrkQzhkSC7atfKc9Qqw8kABAL +ZlaSBRGFs9MIGCIO0crMzYkHlxH9BuORnJDKGYRFzPVgov/QlnrZWoz67G7foj6U +Dt/HXx+taPY1WXjl5f+njgVXnQaEiH6kSfc3GP7zHVW8G/KYXGLdoKjYIn3iOFYr +mZtK7sQdO4g/RVH9arKj6JHlPo6l7b/RybamY0pny4ptiVPv0qq2cgxXMj8s7XeO +Tj65afHGguoEE61o/QbH5I+KDgcCOrIHq7vyjBfH/kQzViqTIBSpajVZ46V99Xc= -----END CERTIFICATE----- """ +# openssl req -new -key server.key -out server.csr +# openssl x509 -CAcreateserial -req -days 3650 -in server.csr -CA ca.crt -CAkey ca.key -out server.crt client_crt = """ -----BEGIN CERTIFICATE----- -MIICfDCCAeUCFCIyzicXHM920mzh6McYBfIKAmeQMA0GCSqGSIb3DQEBCwUAMH0x -CzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMQ8w -DQYDVQQKDAZaZW5vc3MxDDAKBgNVBAsMA0RldjEPMA0GA1UEAwwGWmVub3NzMR0w -GwYJKoZIhvcNAQkBFg5kZXZAemVub3NzLmNvbTAeFw0yMjAxMjYxNzEwMDBaFw0y -NjEyMzExNzEwMDBaMH0xCzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0G -A1UEBwwGQXVzdGluMQ8wDQYDVQQKDAZaZW5vc3MxDDAKBgNVBAsMA0RldjEPMA0G -A1UEAwwGWmVub3NzMR0wGwYJKoZIhvcNAQkBFg5kZXZAemVub3NzLmNvbTCBnzAN -BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwYaLu8f8Hd9yTqGCfFXb1P60LEzlGUom -mStO06zfk3FFz9MBEbVHX53+92R/xhKVfUiRa967COM4y6XJHnfPD/sFBCir4z4+ -ApLRV8jEsWYP/sDG59nZDZm+IUqOwqWfYlvJWpbOlFC5s1q4xeECemM88c9poKAZ -AW3H9oM/pR0CAwEAATANBgkqhkiG9w0BAQsFAAOBgQDDH+LvhUfdLTGF2L/KwHxw -KdWs1KEoFUqI2kD9nUVDj0WoX6pSE8/txRS3Pw2PsA2KahAPTAOZJcLVy5rbUCvF -+DgiPegUZ/btgGrrT5NfTPtkb1E8wNsz+XOEwzlzNakA08Lec6q/vBewJVm2duMd -bqCsKPJj+yBv0nMqFWgVmQ== +MIIFETCCAvkCFHHU+QLzVaIAlzUoRGeFjk9PtLsRMA0GCSqGSIb3DQEBCwUAMEUx +CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwODMwMDcwNzIyWhcNMzMwODI3MDcw +NzIyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE +CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEApZbM2FwREXdRztX/WqIwr3fK1BA5juT/t3aNuT5BVgYtPrJW +7FqoYTUVAcbK662A/yMMpNYTP5lejpEhwuFQXDi3rhNAI6oamhIKfEaZcEvj1VwS +O2JF+FNWl6t0y+gwd6RQTGJcpd5i2aszpc7BTBYjRxG1RQch4Uue5/fx9eD2XgxS +YgbLq6ODa1KVdm2yoWT+Tp8soaWPNFfeTDQrrQcsb5IT+oCclwsMK49z4hK2uxko +Pg7NsvrpLUNNlQCiFoiGdlswdfEumGolKDkS893vLF2pIGhykCGjKbVl+xVAQ6ch +TlkuOH2q91CDnojZwd6pkkPCiSG0v/crIROrGAu8mhKQkkzQ7IMixNLiBEdk9tum +2fCeqVhz/mGp6W5xlKXEIdoLPxtFRHpOI96PGGyHo5GhAIlrqqzUmuRSDnr4Iiyo +vwlX1pmVl1y4WK25oxhExf2qMAOSHH0eA8RhqIulKhbQCYnfwxbgSIBFyWGJR4bD +pyIpPum99RzUUQ5ez848cvK7LiNASeCHoBAeutG0YpXSBTHL+ql/UVLQ7e1yvdV/ +sDi1rnztLBPwUK6+To7Kn67eErdbi/50q/rXryF6YGbPsBZ5xVMmX9nleLkqSTSp +PPPWniHm6GBcZPWmAUWauBCa7XmJmdREoRZ1afA+efvpTgTm7/S4+GVvs+0CAwEA +ATANBgkqhkiG9w0BAQsFAAOCAgEAg2XMP4PAM96soOXNKFeihT4ccQvzzVOtMBV4 +Cc2OP1Ak5BUGRHRfd9AbjYLyRr5L9EyKJmckwZZtFEcRtZLAabUGYiu2L76wwgw3 +uZ3rtkt860N8bj7id9+P8139sIuKnBfnGf7YBp3fbH0GarWFQYyRBehCdsy6FS6o +gV9jmO3NYfXYHX6cR0uYKW9Bv7+xZrJf0+ZHtjwxDNGy3WRyVnTxDChbRFgN5irR +chiRkuOLI4sDa8dc0xrD9WiZ28+PeIiwYfjjHZqresSkOTPB0mzqYMLEDTClrf+M +Q8vW41k0rRvcudousTyrKuMfHxmdC8Ei1OuXjhVQ3nvJnRfWlAqlfSLAcfab0mNb +nptMwzyKDNWJB4aNUDT5RZ4Wbt2Khy9ol9F8STINQ6B71l8/1ORQrZol3QMR/hdu +SCEGu9rpKI+Yti+s0C6IDXU6qDP0kDAzITGrjeBkYSoIj7Tk6L4g1oaZQtQD9Pb8 +xF4SAAt+tc7/U7mvLolHTHpvZzxclHLoyyyRDJ2PkMNhcwQcYag/GCm31K42dKU/ +Zcsd5ZhlNhxmzipuJjzhqrICXCt+WGEiz0/KlxtHhJYEBrjimPoc6Mhm7Byo+4To +1HHU0t80+sRir1j+waNepTsdEdaNVKVweBu0lem2CU2IMzjkSQRu4UOssvJNTw0Y +hk+hfo0= -----END CERTIFICATE----- """ +# openssl genrsa -aes256 -passout pass:ytrewq -out server.pass.key 4096 +# openssl rsa -passin pass:ytrewq -in server.pass.key -out server.key client_pem = """ -----BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDBhou7x/wd33JOoYJ8VdvU/rQsTOUZSiaZK07TrN+TcUXP0wER -tUdfnf73ZH/GEpV9SJFr3rsI4zjLpcked88P+wUEKKvjPj4CktFXyMSxZg/+wMbn -2dkNmb4hSo7CpZ9iW8lals6UULmzWrjF4QJ6Yzzxz2mgoBkBbcf2gz+lHQIDAQAB -AoGANo+lY7rdVNrDknGspTtbsDBjQb4oNToXqcVxAvLRUfN0mERIH+L5DXcxBDS8 -ZW6l4N2NyljQaJAPWjMSgdmLcdrhzABsicKQ1/gkjsfNK8Rz0IzlfR/MuljrFC6s -ZUeWuBsd5wp8/RrFXZVcNypV7mvJ/iJGZnoZqrAwn5bNS20CQQD06C8inRZRjhZC -wtvl/XQaiPsgoez8J8VU2lHMhIvFiJWp4dztoPsBi74MQcF/TgoItn2AmOtZOtph -3H7y1GMLAkEAykqRvYv6eoNbmMZFvDvyVJu2rbVAn0qWJz988oLrmduWWAMy5L2Z -pShUAnkBRT7FbXWYAjeSUEX8PITflQGxdwJAe6ic3CpbMZS/0rfXFprSO++8dW6t -XWirb7vIn66xcG0VvLCJwAaPluk7ba7qB+CcmmeimQMdmnFoAQ+3nd71nwJAA5Yy -41N6C3YMx7asQdwmPc3M/WN7U9e0tdlwU7RyjPXRwpm760ZZVQ5T/v86QIoOYhR1 -r4Rgub+j60bH2BKBnQJAIDZfnhUFv6eYy6I/yPcQ+sIOjC7X1/6XjjPNMAeOheVn -zUc7lh/1HLm55dzLz2Csrosc5YX3ZV99h58Mm5k+Vw== +MIIJKwIBAAKCAgEApZbM2FwREXdRztX/WqIwr3fK1BA5juT/t3aNuT5BVgYtPrJW +7FqoYTUVAcbK662A/yMMpNYTP5lejpEhwuFQXDi3rhNAI6oamhIKfEaZcEvj1VwS +O2JF+FNWl6t0y+gwd6RQTGJcpd5i2aszpc7BTBYjRxG1RQch4Uue5/fx9eD2XgxS +YgbLq6ODa1KVdm2yoWT+Tp8soaWPNFfeTDQrrQcsb5IT+oCclwsMK49z4hK2uxko +Pg7NsvrpLUNNlQCiFoiGdlswdfEumGolKDkS893vLF2pIGhykCGjKbVl+xVAQ6ch +TlkuOH2q91CDnojZwd6pkkPCiSG0v/crIROrGAu8mhKQkkzQ7IMixNLiBEdk9tum +2fCeqVhz/mGp6W5xlKXEIdoLPxtFRHpOI96PGGyHo5GhAIlrqqzUmuRSDnr4Iiyo +vwlX1pmVl1y4WK25oxhExf2qMAOSHH0eA8RhqIulKhbQCYnfwxbgSIBFyWGJR4bD +pyIpPum99RzUUQ5ez848cvK7LiNASeCHoBAeutG0YpXSBTHL+ql/UVLQ7e1yvdV/ +sDi1rnztLBPwUK6+To7Kn67eErdbi/50q/rXryF6YGbPsBZ5xVMmX9nleLkqSTSp +PPPWniHm6GBcZPWmAUWauBCa7XmJmdREoRZ1afA+efvpTgTm7/S4+GVvs+0CAwEA +AQKCAgEAiL4HW4Rr8+h8/jlqLgZR/hUGwijD32TsZyzXzGnEuq1PH79WWMhk1CFp +v5XSbN1S8V6YSmcebh7RHxpqruwx2HZd+Lqc9Na8MQ9E6WvDuiBxfPgTdkapUXBA +ye8k/F456BMg3HM93xvOtcHTXNFoftSpPT86Wk6Rg+NWzmjKvymPSgsS3TCPcKYP +GMmR88KTCQTFnVeFG9gEck09neBXUQPjhh8zsGIU7gaJfk9wevjJPaiAuv6uj2b0 +uBQkNS/YqpMDtymG017gA61kEdtP82MK57BQwhp+wNeGTiMmnDnoX/XcYz7yFGRy +ktlCV+DbMmYV0ltygpv7D6ulSiNb3aNFb0uk83xjoXKjx7YQW9bI1Uz2eFPoQAPo +mfgNW/Zp9P7z4WZJEbQPTQP/hNHMfRo+1Gx4J5Dm62I8hqByvoVoP2UTOZpKUmG7 +XOQEMqNei+fKuJbMOBoZ2qoEneSMw7RQfxDmD9xv7vDpb6XzWgiIzTCoo2Ikk/Cd +X2YBgYNP5VP7pYZ8FzOI9unvnOx5Zwxx15GYrYXV/pEOsmikkllvi1/wGtyyZUjb +s2BdSpP86tqBT+kB/hS8JxTIEN7HR61TQt5NB1WxPom4yUTatd8MqSzdrmJqC4z5 +DDurZPkvxFS5Kf34e1VG7SJrcxdegO9s/mtTn3eCqtmxR1GkL2ECggEBANYDNlK2 +0A6XVWTX24kMeV5wRsYgVv533KgefeRwdzKzRQ/0xKmPK02RftX8faUo3AF9YTl8 +XclHVovMsTCfDwNdXzvdb3w9r5SURmPWFjZjnHRdHOafwQS4GAzhX95N2KbmO1Lb +dknwiASKI0vrSzuZn0D81IDq2Ulq+NcjOeKS9SXCvF1igZDiB25NwT+5lnd/cXMH +0/ziHv9nIzLq+bfVE9KWTGYG7LzabqVUI6klDG5UqgKv0n3yae3fFUkaEC15zmdU +zcVXYf9uFnq3aPVyach61M3zpFkoZfdOc5FutSAENHEkMXPjaFmENPnYSN7O46dZ +CbjmzCS3HTzuiu8CggEBAMYThEnNRmsSW8pfX2JRRsLKBVB3FbkfBf/f5kfVLyjo +c1dnmZzQbUZzZeLs/HWjKqNxktsq762RmHdF2xv8HPutqOFXom/BlJcfhUXF9dan +bVnRLj+L7vwKxd4YWa91hLwLxgDRkWI4XO+iPht6KpzlwOaTrej99h6iTUaOzt7R +wSJlsBCDj1Jgews6QRHoP05R/Ehw9yBi+Azf5JneB+WIme5Y29WltAlBMJDHMCgc +4K/S40U96/TeWVjnc57RrzQN9yRM9Is4A3WqzP8xuCcA1KsUbpSly4vMDRQQP40Q +ETSy3R94lSGahs50fEt0VZCEwEHvig/5KnO6tWqFnuMCggEBAIeQnVaj6wNzJWqt +uakEt9T0tkBGuBSVhLcSKZkNDNSW7oZ+/ByUTk/ifD+8ozJ9wW9IJtAtUZNwlwgT +b6Jm/zGYcf0P9dDzmkc57aTMNmHZk3+6g9YrGC+PFd0C3qGJGlYOvUFtN2766I5H +mrg6ofttAo4+GbZYDbAODPbqn35ArP1wb7WP8pb+NsrOgj2FqCSmHA1LxiMIca5D +fO6CHhEu7lGVV2vBszCmBTTBKZ25lDhHdTIigem6JxPBHlCiK+FCqVaXR4lcIv2U +lLTDfb8M7KlL9YVIcrDvgDe6AEb9o8pWH4oT7SeFw9IAhzZEpVROJbMaGaiAuov/ +WowAZw0CggEBALFJ2LdB/8xoQzZQxQw4GTDSJ42M+SmX5gPPQMt8udhQrqRF+01L +lPNg6IoDejhE0i42wq5esOZXEfN32BUlRD/UgPspOB/1UW0ublg0RsVZWFvzCgUg +18hKUC5o9yU/941ksFYdPZZ/QlfOjO6FG00Rq+X1usx3O2rR9H655dm0Przt7Xfq +eUbPSnKTMpi3mqocYcXpLpiTXNgRMgiynbjJ2pVmfWWuCgXajoCXeLf+mPFmvbtF +IEQtHCWiDG/T2JCsC1A3fQ57FUWlmhS0SNLIQJHcGNn9x8EZ437YyDkXb38OtTKs ++DZ6nDyAMJxMxSU0XOznXVjMuT2amTR94ucCggEBAMNxXMOvtsy5s0SFk1jZOUAC +bEt9SvYYMzUIlNZZYTu69qU38J+ScPmQY2bTMM5oKu0y8RI8C7RtFUbgH6MqJfSM +pMa0uDNjVP1fEQbI5oatjsEJzyjqBVRgOSJODrgBSx2A4J9nfmXxkuv7U1Wo7CtN +0AG3p4wO5JH/IB/ex+ZevaoTYtBDnSagbJYsvWIfv9NYelL2zVG4lKJ9bBDnF0Xk +wZNb4mJtu3FtiJIXXdYDdrM+ARiLfn4t5HPccgFohFD/Ks3nQPXjydXruHkNPrdb +y979XN5woKDgO0sMzktFM0VssRC+bc4GuRMYLaPHI39q8a18q6Q0MBN0xaH+a5c= -----END RSA PRIVATE KEY----- """ From cb16c39aa2f1049120abcc2d890261393fd874b2 Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Thu, 30 Nov 2023 20:55:57 +0200 Subject: [PATCH 031/147] zendisc used ip assigned to netapp interface (#4225) * zendisc used ip assigned to netapp interface Fixes ZEN-30583. zendisc creating devices for ip addresses assigned to a netapp components. There is because of relation ipaddress <-> manageDevice. So when we are checking for the device existence we check manageDevice for that ipaddress. Since we no longer check device existence from ipaddress<->interface relation when creating a device that is why zendisc is discovering devices with ip that are already assigned to the components. Additional check was added to remove such IPs from the discovering list. --- Products/DataCollector/zendisc.py | 16 ++++++++++++++- Products/ZenHub/services/DiscoverService.py | 22 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Products/DataCollector/zendisc.py b/Products/DataCollector/zendisc.py index 7900a24a48..840bc4bfd8 100755 --- a/Products/DataCollector/zendisc.py +++ b/Products/DataCollector/zendisc.py @@ -138,7 +138,13 @@ def discoverIps(self, nets): ) continue self.log.info("Discover network '%s'", net.getNetworkName()) - results = yield self.pingMany(net.fullIpList()) + + full_ip_list = net.fullIpList() + if self.options.removeInterfaceIps: + full_ip_list = yield self.config().callRemote( + "removeInterfaces", net) + + results = yield self.pingMany(full_ip_list) goodips, badips = _partitionPingResults(results) self.log.debug( "Found %d good IPs and %d bad IPs", len(goodips), len(badips) @@ -872,6 +878,14 @@ def buildOptions(self): default=False, help="Prefer SNMP name to DNS name when modeling via SNMP.", ) + self.parser.add_option( + "--remove-interface-ips", + dest="removeInterfaceIps", + action="store_true", + default=False, + help="Skip discovery on IPs already assigned to interfaces (device components).", + ) + # --job: a development-only option that jobs will use to communicate # their existence to zendisc. Not for users, so help is suppressed. self.parser.add_option("--job", dest="job", help=SUPPRESS_HELP) diff --git a/Products/ZenHub/services/DiscoverService.py b/Products/ZenHub/services/DiscoverService.py index a759455415..f91c87cb77 100644 --- a/Products/ZenHub/services/DiscoverService.py +++ b/Products/ZenHub/services/DiscoverService.py @@ -303,3 +303,25 @@ def remote_moveDevice(self, dev, path): def remote_getDefaultNetworks(self): monitor = self.dmd.Monitors.Performance._getOb(self.instance) return [net for net in monitor.discoveryNetworks] + + @translateError + def remote_removeInterfaces(self, net): + """ + Remove IPs for particular network + already assigned to interfaces (device components) + + @param net - network to discover + @return: a list of IPs without addresses assigned for interfaces + @rtype: list + """ + + full_ip_list = net.fullIpList() + + for d in self.dmd.Devices.getSubDevicesGen(): + for interface in d.os.interfaces(): + for addr in interface.ipaddresses(): + ip = addr.getIp() + if net.netmask == addr.netmask and ip in full_ip_list: + full_ip_list.remove(ip) + + return full_ip_list From f15e0699219bb6c57b4a165a9c6cd65cf946fa0f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 14 Dec 2023 18:04:21 -0600 Subject: [PATCH 032/147] Modernize number literals. --- Products/DataCollector/plugins/CollectorPlugin.py | 4 ++-- Products/ZenModel/DeviceHW.py | 2 +- Products/ZenModel/FileSystem.py | 4 ++-- Products/ZenModel/OperatingSystem.py | 2 +- Products/ZenModel/ZenPack.py | 4 ++-- Products/ZenModel/ZenPackLoader.py | 6 +++--- Products/ZenModel/migrate/import_export_filesystem.py | 2 +- Products/ZenModel/tests/IpUtilTest.py | 6 +++--- Products/ZenModel/tests/testMaintenanceWindow.py | 2 +- Products/ZenUtils/IpUtil.py | 4 ++-- Products/ZenUtils/ZenBackup.py | 4 ++-- Products/ZenUtils/ZenDaemon.py | 2 +- Products/ZenUtils/ZenPackCmd.py | 4 ++-- Products/ZenUtils/zenpack.py | 6 +++--- 14 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Products/DataCollector/plugins/CollectorPlugin.py b/Products/DataCollector/plugins/CollectorPlugin.py index 477224fc59..644345b96e 100644 --- a/Products/DataCollector/plugins/CollectorPlugin.py +++ b/Products/DataCollector/plugins/CollectorPlugin.py @@ -128,13 +128,13 @@ def copyDataToProxy(self, device, proxy): def asdate(self, val): """Convert a byte string to the date string 'YYYY/MM/DD HH:MM:SS'""" - datear = (1968, 1, 8, 10, 15, 00) + datear = (1968, 1, 8, 10, 15, 0) try: datear = struct.unpack("!h5B", val[0:7]) except Exception: pass if datear[0] == 0: - datear = (1968, 1, 8, 10, 15, 00) + datear = (1968, 1, 8, 10, 15, 0) return "%d/%02d/%02d %02d:%02d:%02d" % datear[:6] diff --git a/Products/ZenModel/DeviceHW.py b/Products/ZenModel/DeviceHW.py index c64c876cb5..f4b566e5cb 100644 --- a/Products/ZenModel/DeviceHW.py +++ b/Products/ZenModel/DeviceHW.py @@ -25,7 +25,7 @@ class DeviceHW(Hardware): meta_type = "DeviceHW" - totalMemory = 0L + totalMemory = 0 _properties = Hardware._properties + ( {'id':'totalMemory', 'type':'long', 'mode':'w'}, diff --git a/Products/ZenModel/FileSystem.py b/Products/ZenModel/FileSystem.py index 705fd83181..304504ba05 100644 --- a/Products/ZenModel/FileSystem.py +++ b/Products/ZenModel/FileSystem.py @@ -57,8 +57,8 @@ class FileSystem(OSComponent): storageDevice = "" type = "" blockSize = 0 - totalBlocks = 0L - totalFiles = 0L + totalBlocks = 0 + totalFiles = 0 capacity = 0 inodeCapacity = 0 maxNameLen = 0 diff --git a/Products/ZenModel/OperatingSystem.py b/Products/ZenModel/OperatingSystem.py index 59d4dc540d..32303d9836 100644 --- a/Products/ZenModel/OperatingSystem.py +++ b/Products/ZenModel/OperatingSystem.py @@ -34,7 +34,7 @@ class OperatingSystem(Software): - totalSwap = 0L + totalSwap = 0 uname = "" _properties = Software._properties + ( diff --git a/Products/ZenModel/ZenPack.py b/Products/ZenModel/ZenPack.py index b305c97437..6ee246fd29 100644 --- a/Products/ZenModel/ZenPack.py +++ b/Products/ZenModel/ZenPack.py @@ -77,7 +77,7 @@ def __init__(self, *args, **kw): VersionBase.__init__(self, 'Zenoss', *args, **kw) -def needDir(path, perms=0750): +def needDir(path, perms=0o750): if not os.path.isdir(path): os.mkdir(path, perms) return path @@ -343,7 +343,7 @@ def storeBackup(self): """ backupDir = zenPath(".ZenPacks") if not os.path.isdir(backupDir): - os.makedirs(backupDir, 0750) + os.makedirs(backupDir, 0o750) src = self.eggPath() filename = "" diff --git a/Products/ZenModel/ZenPackLoader.py b/Products/ZenModel/ZenPackLoader.py index 06a89d4c3b..0cfb21612a 100644 --- a/Products/ZenModel/ZenPackLoader.py +++ b/Products/ZenModel/ZenPackLoader.py @@ -240,7 +240,7 @@ def _updateConfFile(self, pack): def load(self, pack, unused): for fs in findFiles(pack, 'daemons', filter=self.filter): - os.chmod(fs, 0755) + os.chmod(fs, 0o755) path = self.binPath(fs) if os.path.lexists(path): os.remove(path) @@ -286,7 +286,7 @@ def binPath(self, bin_file): def load(self, pack, unused): for fs in findFiles(pack, 'bin', filter=self.filter): - os.chmod(fs, 0755) + os.chmod(fs, 0o755) path = self.binPath(fs) if os.path.lexists(path): os.remove(path) @@ -314,7 +314,7 @@ def filter(self, f): def load(self, pack, unused): for fs in findFiles(pack, 'libexec', filter=self.filter): - os.chmod(fs, 0755) + os.chmod(fs, 0o755) def upgrade(self, pack, app): self.unload(pack, app) diff --git a/Products/ZenModel/migrate/import_export_filesystem.py b/Products/ZenModel/migrate/import_export_filesystem.py index f472ae45ab..fee4adbee9 100644 --- a/Products/ZenModel/migrate/import_export_filesystem.py +++ b/Products/ZenModel/migrate/import_export_filesystem.py @@ -30,6 +30,6 @@ def cutover(self, unused): for directory in ['import', 'export']: path = zenPath(directory) if not os.path.exists(path): - os.mkdir(path, 0750) + os.mkdir(path, 0o750) ImportExportFilesystem() diff --git a/Products/ZenModel/tests/IpUtilTest.py b/Products/ZenModel/tests/IpUtilTest.py index caaa8b0312..3d46d14d11 100644 --- a/Products/ZenModel/tests/IpUtilTest.py +++ b/Products/ZenModel/tests/IpUtilTest.py @@ -137,14 +137,14 @@ def testNumbIpGood(self): self.assertEqual( IpUtil.numbip( '192.168.2.3'), - 3232236035L) + 3232236035) def testStripGood(self): '''check that the strip function can convert a number back into an IP''' self.assertEqual( IpUtil.strip( - 3232236035L), + 3232236035), '192.168.2.3') def testGetNetBadIp(self): @@ -173,7 +173,7 @@ def testGetNetAllGood(self): IpUtil.getnet( '192.168.2.3', '255.255.240.0'), - 3232235520L) + 3232235520) def testGetNetStr(self): '''check to make sure getnetstr works fine, diff --git a/Products/ZenModel/tests/testMaintenanceWindow.py b/Products/ZenModel/tests/testMaintenanceWindow.py index d0cf77a85b..40b0f05225 100644 --- a/Products/ZenModel/tests/testMaintenanceWindow.py +++ b/Products/ZenModel/tests/testMaintenanceWindow.py @@ -339,7 +339,7 @@ class multiWindow: multiWin.dev.setGroups(multiWin.grp.id) multiWin.startDateTime = '1138531500' - startDate_time = [ 2006, 1, 31, 10, 00, 12, 0, 0, 0 ] + startDate_time = [ 2006, 1, 31, 10, 0, 12, 0, 0, 0 ] multiWin.tn = range(1,maxWindows) multiWin.time_tn = [] multiWin.mwIds = [] diff --git a/Products/ZenUtils/IpUtil.py b/Products/ZenUtils/IpUtil.py index 750141e30f..93c499e0f4 100644 --- a/Products/ZenUtils/IpUtil.py +++ b/Products/ZenUtils/IpUtil.py @@ -339,10 +339,10 @@ def bitsToDecimalMask(netbits): >>> bitsToDecimalMask(0) 0L """ - masknumb = 0L + masknumb = 0 netbits=int(netbits) for i in range(32-netbits, 32): - masknumb += 2L ** i + masknumb += 2 ** i return masknumb diff --git a/Products/ZenUtils/ZenBackup.py b/Products/ZenUtils/ZenBackup.py index 428981d5f3..b0508f26eb 100755 --- a/Products/ZenUtils/ZenBackup.py +++ b/Products/ZenUtils/ZenBackup.py @@ -153,7 +153,7 @@ def getName(index=0): (index and '_%s' % index) or '') backupDir = zenPath('backups') if not os.path.exists(backupDir): - os.mkdir(backupDir, 0750) + os.mkdir(backupDir, 0o750) for i in range(MAX_UNIQUE_NAME_ATTEMPTS): name = os.path.join(backupDir, getName(i)) if not os.path.exists(name): @@ -455,7 +455,7 @@ def makeBackup(self): self.rootTempDir = self.getTempDir() self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR) self.log.debug("Use %s as a staging directory for the backup", self.tempDir) - os.mkdir(self.tempDir, 0750) + os.mkdir(self.tempDir, 0o750) if self.options.collector: self.options.noEventsDb = True diff --git a/Products/ZenUtils/ZenDaemon.py b/Products/ZenUtils/ZenDaemon.py index 5b73443802..60ca76db1f 100755 --- a/Products/ZenUtils/ZenDaemon.py +++ b/Products/ZenUtils/ZenDaemon.py @@ -34,7 +34,7 @@ # Daemon creation code below based on Recipe by Chad J. Schroeder # File mode creation mask of the daemon. -UMASK = 0022 +UMASK = 0o022 # Default working directory for the daemon. WORKDIR = "/" diff --git a/Products/ZenUtils/ZenPackCmd.py b/Products/ZenUtils/ZenPackCmd.py index de1d4d759c..b4c4553803 100644 --- a/Products/ZenUtils/ZenPackCmd.py +++ b/Products/ZenUtils/ZenPackCmd.py @@ -77,7 +77,7 @@ def CreateZenPack(zpId, prevZenPackName='', devDir=None): if not devDir: devDir = zenPath('ZenPacks') if not os.path.exists(devDir): - os.mkdir(devDir, 0750) + os.mkdir(devDir, 0o750) destDir = os.path.join(devDir, zpId) shutil.copytree(srcDir, destDir, symlinks=False) os.system('find %s -name .svn | xargs rm -rf' % destDir) @@ -695,7 +695,7 @@ def CreateZenPacksDir(): """ zpDir = zenPath('ZenPacks') if not os.path.isdir(zpDir): - os.mkdir(zpDir, 0750) + os.mkdir(zpDir, 0o750) def DoEasyInstall(eggPath): diff --git a/Products/ZenUtils/zenpack.py b/Products/ZenUtils/zenpack.py index 0abecf6c56..d6b31227d1 100644 --- a/Products/ZenUtils/zenpack.py +++ b/Products/ZenUtils/zenpack.py @@ -278,7 +278,7 @@ def path(self, *parts): # by ZPLSkins loader. skinsSubdir = zenPath('Products', packName, 'skins', packName) if not os.path.exists(skinsSubdir): - os.makedirs(skinsSubdir, 0750) + os.makedirs(skinsSubdir, 0o750) self.install(packName) elif self.options.fetch: @@ -761,11 +761,11 @@ def extract(self, fname): if name.endswith('~'): continue if name.endswith('/'): if not os.path.exists(fullname): - os.makedirs(fullname, 0750) + os.makedirs(fullname, 0o750) else: base = os.path.dirname(fullname) if not os.path.isdir(base): - os.makedirs(base, 0750) + os.makedirs(base, 0o750) file(fullname, 'wb').write(zf.read(name)) return packName From 08c55be09c23276df3b31f9021058ae34ebd65b0 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 15 Dec 2023 15:28:25 -0600 Subject: [PATCH 033/147] Fix unit tests for modernized number literals. --- Products/ZenModel/IpNetwork.py | 2 +- Products/ZenUtils/IpUtil.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Products/ZenModel/IpNetwork.py b/Products/ZenModel/IpNetwork.py index 0809a197b7..4f77e575cb 100644 --- a/Products/ZenModel/IpNetwork.py +++ b/Products/ZenModel/IpNetwork.py @@ -624,7 +624,7 @@ def primarySortKey(self): >>> net = dmd.Networks.addSubNetwork('1.2.3.0', 24) >>> net.primarySortKey() - 16909056L + 16909056 """ return numbip(self.id) diff --git a/Products/ZenUtils/IpUtil.py b/Products/ZenUtils/IpUtil.py index 93c499e0f4..8dffcbc59e 100644 --- a/Products/ZenUtils/IpUtil.py +++ b/Products/ZenUtils/IpUtil.py @@ -195,20 +195,18 @@ def ipToDecimal(ip): calculating netmasks etc. >>> ipToDecimal('10.10.20.5') - 168432645L + 168432645 >>> try: ipToDecimal('10.10.20.500') ... except IpAddressError as ex: print ex 10.10.20.500 is an invalid address """ checkip(ip) - # The unit tests expect to always get a long, while the - # ipaddr.IPaddress class doesn't provide a direct "to long" capability unwrapped = ipunwrap(ip) if '%' in unwrapped: address = unwrapped[:unwrapped.index('%')] else: address = unwrapped - return long(int(IPAddress(address))) + return int(IPAddress(address)) def ipFromIpMask(ipmask): """ @@ -309,7 +307,7 @@ def maskToBits(netmask): 0 """ if isinstance(netmask, basestring) and '.' in netmask: - test = 0xffffffffL + test = 0xffffffff if netmask[0]=='0': return 0 masknumb = ipToDecimal(netmask) for i in range(32): @@ -333,11 +331,11 @@ def bitsToDecimalMask(netbits): Convert integer number of netbits to a decimal number >>> bitsToDecimalMask(32) - 4294967295L + 4294967295 >>> bitsToDecimalMask(19) - 4294959104L + 4294959104 >>> bitsToDecimalMask(0) - 0L + 0 """ masknumb = 0 netbits=int(netbits) @@ -371,12 +369,12 @@ def decimalNetFromIpAndNet(ip, netmask): Get network address of IP as string netmask as in the form 255.255.255.0 >>> getnet('10.12.25.33', 24) - 168564992L + 168564992 >>> getnet('10.12.25.33', '255.255.255.0') - 168564992L + 168564992 """ checkip(ip) - return long(int(IPNetwork( ipunwrap(ip) + '/' + str(netmask)).network)) + return int(IPNetwork(ipunwrap(ip) + '/' + str(netmask)).network) def getnetstr(ip, netmask): """ From a9f54d9d89762d3ab2409c742a8b54a22a559e73 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 7 Dec 2023 16:02:57 -0600 Subject: [PATCH 034/147] Implement the Configuration Cache for device configurations. ZEN-34531 --- Products/DataCollector/SnmpClient.py | 12 +- Products/DataCollector/zendisc.py | 3 +- Products/DataCollector/zenmodeler.py | 112 +- Products/Jobber/config.py | 10 +- Products/Jobber/log.py | 197 +-- Products/Jobber/task/event.py | 23 +- Products/Jobber/task/utils.py | 9 +- Products/Jobber/tests/test_log.py | 8 - Products/ZenCollector/CollectorCmdBase.py | 46 - Products/ZenCollector/DeviceConfigCache.py | 50 - Products/ZenCollector/__init__.py | 37 - Products/ZenCollector/collector.zcml | 20 + Products/ZenCollector/config.py | 378 ---- Products/ZenCollector/config/__init__.py | 17 + Products/ZenCollector/config/proxy.py | 142 ++ Products/ZenCollector/config/task.py | 207 +++ Products/ZenCollector/configcache/__init__.py | 8 + Products/ZenCollector/configcache/__main__.py | 21 + .../ZenCollector/configcache/app/__init__.py | 17 + Products/ZenCollector/configcache/app/base.py | 139 ++ .../ZenCollector/configcache/app/config.py | 101 ++ .../ZenCollector/configcache/app/genconf.py | 60 + Products/ZenCollector/configcache/app/init.py | 27 + .../ZenCollector/configcache/app/logger.py | 189 ++ .../ZenCollector/configcache/app/metrics.py | 81 + Products/ZenCollector/configcache/app/pid.py | 130 ++ Products/ZenCollector/configcache/app/zodb.py | 235 +++ .../configcache/cache/__init__.py | 24 + .../ZenCollector/configcache/cache/model.py | 147 ++ .../ZenCollector/configcache/cache/storage.py | 721 ++++++++ Products/ZenCollector/configcache/cli.py | 284 +++ .../ZenCollector/configcache/configcache.py | 32 + .../ZenCollector/configcache/configure.zcml | 17 + Products/ZenCollector/configcache/debug.py | 45 + .../ZenCollector/configcache/invalidator.py | 226 +++ Products/ZenCollector/configcache/manager.py | 193 ++ Products/ZenCollector/configcache/meta.zcml | 14 + .../ZenCollector/configcache/misc/__init__.py | 37 + .../ZenCollector/configcache/misc/args.py | 53 + .../ZenCollector/configcache/modelchange.zcml | 85 + .../configcache/modelchange/__init__.py | 16 + .../configcache/modelchange/filters.py | 222 +++ .../configcache/modelchange/invalidation.py | 52 + .../configcache/modelchange/oids.py | 109 ++ .../configcache/modelchange/pipeline.py | 126 ++ .../configcache/modelchange/processor.py | 227 +++ .../modelchange/tests/mock_interface.py | 29 + .../modelchange/tests/test_filters.py | 303 ++++ .../modelchange/tests/test_oids.py | 70 + Products/ZenCollector/configcache/task.py | 106 ++ .../configcache/tests/__init__.py | 0 .../configcache/tests/test_bestmatchmap.py | 70 + .../configcache/tests/test_storage.py | 667 +++++++ .../configcache/utils/__init__.py | 31 + .../configcache/utils/dispatcher.py | 66 + .../ZenCollector/configcache/utils/pollers.py | 88 + .../configcache/utils/propertymap.py | 72 + .../configcache/utils/services.py | 107 ++ Products/ZenCollector/configcache/version.py | 33 + Products/ZenCollector/configure.zcml | 25 +- Products/ZenCollector/daemon.py | 774 ++++---- Products/ZenCollector/frameworkfactory.py | 39 + Products/ZenCollector/interfaces.py | 144 +- Products/ZenCollector/listeners.py | 98 + Products/ZenCollector/services/config.py | 480 +---- Products/ZenCollector/services/error.py | 46 + .../ZenCollector/services/optionsfilter.py | 30 + Products/ZenCollector/services/push.py | 303 ++++ Products/ZenCollector/statistics.py | 72 + Products/ZenCollector/tasks.py | 215 +-- Products/ZenCollector/tests/testConfig.py | 46 +- Products/ZenCollector/tests/testFactory.py | 7 +- Products/ZenCollector/tests/test_daemon.py | 14 +- Products/ZenCollector/utils/maintenance.py | 39 +- Products/ZenHub/PBDaemon.py | 1314 +++----------- Products/ZenHub/configure.zcml | 10 +- Products/ZenHub/errors.py | 85 + Products/ZenHub/events/__init__.py | 13 + Products/ZenHub/events/client.py | 149 ++ Products/ZenHub/events/queue/__init__.py | 4 + Products/ZenHub/events/queue/base.py | 62 + Products/ZenHub/events/queue/deduping.py | 117 ++ Products/ZenHub/events/queue/deque.py | 60 + Products/ZenHub/events/queue/fingerprint.py | 25 + Products/ZenHub/events/queue/manager.py | 255 +++ Products/ZenHub/events/queue/misc.py | 21 + .../ZenHub/events/queue/tests/__init__.py | 0 .../ZenHub/events/queue/tests/test_base.py | 32 + .../events/queue/tests/test_deduping.py | 234 +++ .../ZenHub/events/queue/tests/test_deque.py | 102 ++ .../events/queue/tests/test_fingerprint.py | 55 + .../ZenHub/events/queue/tests/test_manager.py | 328 ++++ .../ZenHub/events/queue/tests/test_misc.py | 29 + Products/ZenHub/events/tests/__init__.py | 0 Products/ZenHub/hub.zcml | 26 +- Products/ZenHub/interfaces.py | 18 +- Products/ZenHub/invalidationfilter.py | 1 + Products/ZenHub/invalidationoid.py | 18 - Products/ZenHub/server/avatar.py | 2 +- Products/ZenHub/server/executors/workers.py | 2 +- Products/ZenHub/server/service.py | 2 +- Products/ZenHub/server/tests/test_service.py | 5 +- Products/ZenHub/server/workerpool.py | 2 +- Products/ZenHub/services/ConfigCache.py | 182 ++ Products/ZenHub/services/EventService.py | 2 +- Products/ZenHub/services/ModelerService.py | 4 - Products/ZenHub/services/PerformanceConfig.py | 87 +- Products/ZenHub/services/SnmpTrapConfig.py | 8 +- Products/ZenHub/services/ThresholdMixin.py | 41 +- Products/ZenHub/tests/testPBDaemon.py | 34 +- Products/ZenHub/tests/test_PBDaemon.py | 1572 ++--------------- Products/ZenHub/tests/test_errors.py | 57 + .../ZenHub/tests/test_invalidationfilter.py | 10 +- .../ZenHub/tests/test_invalidationmanager.py | 6 +- Products/ZenHub/tests/test_invalidationoid.py | 46 +- Products/ZenHub/tests/test_invalidations.py | 7 +- Products/ZenHub/tests/test_zenhub.py | 6 +- Products/ZenHub/zenhub.py | 14 +- Products/ZenHub/zenhubclient.py | 342 ++++ Products/ZenHub/zenhubworker.py | 2 +- Products/ZenModel/ThresholdInstance.py | 55 +- Products/ZenModel/ValueChangeThreshold.py | 55 +- .../migrate/addConfigCacheProperties.py | 70 + Products/ZenRRD/RRDDaemon.py | 15 +- Products/ZenRelations/ZenPropertyManager.py | 40 +- Products/ZenRelations/zPropertyCategory.py | 6 + Products/ZenUtils/DaemonStats.py | 68 +- Products/ZenUtils/MetricServiceRequest.py | 4 +- Products/ZenUtils/PBUtil.py | 78 +- Products/ZenUtils/ZCmdBase.py | 2 +- Products/ZenUtils/ZenDaemon.py | 6 +- Products/ZenUtils/metricwriter.py | 90 +- Products/ZenUtils/terminal_size.py | 125 ++ setup.py | 21 +- 134 files changed, 10124 insertions(+), 4785 deletions(-) delete mode 100644 Products/ZenCollector/CollectorCmdBase.py delete mode 100644 Products/ZenCollector/DeviceConfigCache.py create mode 100644 Products/ZenCollector/collector.zcml delete mode 100644 Products/ZenCollector/config.py create mode 100644 Products/ZenCollector/config/__init__.py create mode 100644 Products/ZenCollector/config/proxy.py create mode 100644 Products/ZenCollector/config/task.py create mode 100644 Products/ZenCollector/configcache/__init__.py create mode 100644 Products/ZenCollector/configcache/__main__.py create mode 100644 Products/ZenCollector/configcache/app/__init__.py create mode 100644 Products/ZenCollector/configcache/app/base.py create mode 100644 Products/ZenCollector/configcache/app/config.py create mode 100644 Products/ZenCollector/configcache/app/genconf.py create mode 100644 Products/ZenCollector/configcache/app/init.py create mode 100644 Products/ZenCollector/configcache/app/logger.py create mode 100644 Products/ZenCollector/configcache/app/metrics.py create mode 100644 Products/ZenCollector/configcache/app/pid.py create mode 100644 Products/ZenCollector/configcache/app/zodb.py create mode 100644 Products/ZenCollector/configcache/cache/__init__.py create mode 100644 Products/ZenCollector/configcache/cache/model.py create mode 100644 Products/ZenCollector/configcache/cache/storage.py create mode 100644 Products/ZenCollector/configcache/cli.py create mode 100644 Products/ZenCollector/configcache/configcache.py create mode 100644 Products/ZenCollector/configcache/configure.zcml create mode 100644 Products/ZenCollector/configcache/debug.py create mode 100644 Products/ZenCollector/configcache/invalidator.py create mode 100644 Products/ZenCollector/configcache/manager.py create mode 100644 Products/ZenCollector/configcache/meta.zcml create mode 100644 Products/ZenCollector/configcache/misc/__init__.py create mode 100644 Products/ZenCollector/configcache/misc/args.py create mode 100644 Products/ZenCollector/configcache/modelchange.zcml create mode 100644 Products/ZenCollector/configcache/modelchange/__init__.py create mode 100644 Products/ZenCollector/configcache/modelchange/filters.py create mode 100644 Products/ZenCollector/configcache/modelchange/invalidation.py create mode 100644 Products/ZenCollector/configcache/modelchange/oids.py create mode 100644 Products/ZenCollector/configcache/modelchange/pipeline.py create mode 100644 Products/ZenCollector/configcache/modelchange/processor.py create mode 100644 Products/ZenCollector/configcache/modelchange/tests/mock_interface.py create mode 100644 Products/ZenCollector/configcache/modelchange/tests/test_filters.py create mode 100644 Products/ZenCollector/configcache/modelchange/tests/test_oids.py create mode 100644 Products/ZenCollector/configcache/task.py create mode 100644 Products/ZenCollector/configcache/tests/__init__.py create mode 100644 Products/ZenCollector/configcache/tests/test_bestmatchmap.py create mode 100644 Products/ZenCollector/configcache/tests/test_storage.py create mode 100644 Products/ZenCollector/configcache/utils/__init__.py create mode 100644 Products/ZenCollector/configcache/utils/dispatcher.py create mode 100644 Products/ZenCollector/configcache/utils/pollers.py create mode 100644 Products/ZenCollector/configcache/utils/propertymap.py create mode 100644 Products/ZenCollector/configcache/utils/services.py create mode 100644 Products/ZenCollector/configcache/version.py create mode 100644 Products/ZenCollector/frameworkfactory.py create mode 100644 Products/ZenCollector/listeners.py create mode 100644 Products/ZenCollector/services/error.py create mode 100644 Products/ZenCollector/services/optionsfilter.py create mode 100644 Products/ZenCollector/services/push.py create mode 100644 Products/ZenCollector/statistics.py create mode 100644 Products/ZenHub/errors.py create mode 100644 Products/ZenHub/events/__init__.py create mode 100644 Products/ZenHub/events/client.py create mode 100644 Products/ZenHub/events/queue/__init__.py create mode 100644 Products/ZenHub/events/queue/base.py create mode 100644 Products/ZenHub/events/queue/deduping.py create mode 100644 Products/ZenHub/events/queue/deque.py create mode 100644 Products/ZenHub/events/queue/fingerprint.py create mode 100644 Products/ZenHub/events/queue/manager.py create mode 100644 Products/ZenHub/events/queue/misc.py create mode 100644 Products/ZenHub/events/queue/tests/__init__.py create mode 100644 Products/ZenHub/events/queue/tests/test_base.py create mode 100644 Products/ZenHub/events/queue/tests/test_deduping.py create mode 100644 Products/ZenHub/events/queue/tests/test_deque.py create mode 100644 Products/ZenHub/events/queue/tests/test_fingerprint.py create mode 100644 Products/ZenHub/events/queue/tests/test_manager.py create mode 100644 Products/ZenHub/events/queue/tests/test_misc.py create mode 100644 Products/ZenHub/events/tests/__init__.py create mode 100644 Products/ZenHub/services/ConfigCache.py create mode 100644 Products/ZenHub/tests/test_errors.py create mode 100644 Products/ZenHub/zenhubclient.py create mode 100644 Products/ZenModel/migrate/addConfigCacheProperties.py create mode 100644 Products/ZenUtils/terminal_size.py diff --git a/Products/DataCollector/SnmpClient.py b/Products/DataCollector/SnmpClient.py index 7dfb6ff528..f5397dbd66 100644 --- a/Products/DataCollector/SnmpClient.py +++ b/Products/DataCollector/SnmpClient.py @@ -50,7 +50,7 @@ def __init__( datacollector=None, plugins=[], ): - BaseClient.__init__(self, device, datacollector) + super(SnmpClient, self).__init__(device, datacollector) global defaultTries, defaultTimeout self.hostname = hostname self.device = device @@ -266,7 +266,8 @@ def clientFinished(self, result): if isinstance(result.value, error.TimeoutError): log.error( - "Device %s timed out: are " "your SNMP settings correct?", + "Device %s timed out: are " + "your SNMP settings correct?", self.hostname, ) summary = "SNMP agent down - no response received" @@ -286,13 +287,14 @@ def clientFinished(self, result): self._sendStatusEvent(summary, eventKey="agent_down") else: self._sendStatusEvent( - "SNMP agent up", eventKey="agent_down", severity=Event.Clear + "SNMP agent up", + eventKey="agent_down", + severity=Event.Clear, ) try: self.proxy.close() except AttributeError: - log.info("Caught AttributeError closing SNMP connection.") - """tell the datacollector that we are all done""" + log.info("caught AttributeError closing SNMP connection.") if self.datacollector: self.datacollector.clientFinished(self) else: diff --git a/Products/DataCollector/zendisc.py b/Products/DataCollector/zendisc.py index 840bc4bfd8..ae22f11f29 100755 --- a/Products/DataCollector/zendisc.py +++ b/Products/DataCollector/zendisc.py @@ -883,7 +883,8 @@ def buildOptions(self): dest="removeInterfaceIps", action="store_true", default=False, - help="Skip discovery on IPs already assigned to interfaces (device components).", + help="Skip discovery on IPs already assigned to interfaces " + "(device components).", ) # --job: a development-only option that jobs will use to communicate diff --git a/Products/DataCollector/zenmodeler.py b/Products/DataCollector/zenmodeler.py index 795c109397..bd6762eca6 100755 --- a/Products/DataCollector/zenmodeler.py +++ b/Products/DataCollector/zenmodeler.py @@ -23,7 +23,6 @@ except ImportError: USE_WMI = False -import collections import cPickle as pickle import gzip import os @@ -39,8 +38,8 @@ import zope.component from metrology import Metrology +from twisted.internet import reactor, defer from twisted.internet.defer import succeed -from twisted.internet import reactor from twisted.python.failure import Failure from Products.DataCollector import Classifier @@ -57,9 +56,7 @@ from Products.ZenCollector.interfaces import IEventService from Products.ZenEvents.ZenEventClasses import Heartbeat, Error from Products.ZenHub.PBDaemon import FakeRemote, PBDaemon, HubDown -from Products.ZenUtils.DaemonStats import DaemonStats from Products.ZenUtils.Driver import drive, driveLater -from Products.ZenUtils.metricwriter import ThresholdNotifier from Products.ZenUtils.Utils import unused, zenPath from Products.Zuul.utils import safe_hasattr as hasattr @@ -89,7 +86,7 @@ class ZenModeler(PBDaemon): metrics. """ - name = "zenmodeler" + mname = name = "zenmodeler" initialServices = PBDaemon.initialServices + ["ModelerService"] generateEvents = True @@ -104,15 +101,12 @@ def __init__(self, single=False): @param single: collect from a single device? @type single: boolean """ - PBDaemon.__init__(self) + super(ZenModeler, self).__init__() # FIXME: cleanup --force option #2660 self.options.force = True self.start = None self.startat = None - self.rrdStats = DaemonStats() - self.single = single - if self.options.device: - self.single = True + self.single = single if not self.options.device else True self.modelerCycleInterval = self.options.cycletime # get the minutes and convert to fraction of a day self.collage = float(self.options.collage) / 1440.0 @@ -120,7 +114,6 @@ def __init__(self, single=False): self.clients = [] self.finished = [] self.devicegen = None - self.counters = collections.Counter() self.configFilter = None self.configLoaded = False @@ -166,14 +159,15 @@ def reportError(self, error): """ self.log.error("Error occured: %s", error) + @defer.inlineCallbacks def connected(self): - """ - Called after connected to the zenhub service - """ + """Invoked after connected to ZenHub.""" reactor.callLater(_CONFIG_PULLING_TIMEOUT, self._checkConfigLoad) - d = self.configure() - d.addCallback(self.heartbeat) - d.addErrback(self.reportError) + try: + yield self.configure() + self.heartbeat() + except Exception: + self.log.exception("failed to configure") def _checkConfigLoad(self): """ @@ -187,6 +181,7 @@ def _checkConfigLoad(self): ) reactor.callLater(_CONFIG_PULLING_TIMEOUT, self._checkConfigLoad) + @defer.inlineCallbacks def configure(self): """ Get our configuration from zenhub @@ -194,51 +189,36 @@ def configure(self): # add in the code to fetch cycle time, etc. self.log.info("Getting configuration from ZenHub...") - def inner(driver): - """ - Generator function to gather our configuration + svc = self.config() - @param driver: driver object - @type driver: driver object - """ - self.log.debug("fetching monitor properties") - yield self.config().callRemote("propertyItems") - items = dict(driver.next()) - # If the cycletime option is not specified or zero, then use the - # modelerCycleInterval value in the database. - if not self.options.cycletime: - self.modelerCycleInterval = items.get( - "modelerCycleInterval", _DEFAULT_CYCLE_INTERVAL - ) - self.configCycleInterval = items.get( - "configCycleInterval", self.configCycleInterval + self.log.debug("fetching monitor properties") + items = yield svc.callRemote("propertyItems") + items = dict(items) + # If the cycletime option is not specified or zero, then use the + # modelerCycleInterval value in the database. + if not self.options.cycletime: + self.modelerCycleInterval = items.get( + "modelerCycleInterval", _DEFAULT_CYCLE_INTERVAL ) - reactor.callLater(self.configCycleInterval * 60, self.configure) - - self.log.debug("Getting threshold classes...") - yield self.config().callRemote("getThresholdClasses") - self.remote_updateThresholdClasses(driver.next()) - - self.log.debug("Getting collector thresholds...") - yield self.config().callRemote("getCollectorThresholds") - thresholds = driver.next() - threshold_notifier = ThresholdNotifier(self.sendEvent, thresholds) + self.configCycleInterval = items.get( + "configCycleInterval", self.configCycleInterval + ) + reactor.callLater(self.configCycleInterval * 60, self.configure) - self.rrdStats.config( - self.name, - self.options.monitor, - self.metricWriter(), - threshold_notifier, - self.derivativeTracker(), - ) + self.log.debug("Getting threshold classes...") + classes = yield svc.callRemote("getThresholdClasses") + self.remote_updateThresholdClasses(classes) - self.log.debug("Getting collector plugins for each DeviceClass") - yield self.config().callRemote("getClassCollectorPlugins") - self.classCollectorPlugins = driver.next() + self.log.debug("Getting collector thresholds...") + thresholds = yield svc.callRemote("getCollectorThresholds") + self.getThresholds().updateList(thresholds) - self.configLoaded = True + self.log.debug("Getting collector plugins for each DeviceClass") + self.classCollectorPlugins = yield svc.callRemote( + "getClassCollectorPlugins" + ) - return drive(inner) + self.configLoaded = True def config(self): """ @@ -601,13 +581,13 @@ def snmpCollect(self, device, ip, timeout): # # return drive(inner) - def addClient(self, device, timeout, clientType, name): + def addClient(self, client, timeout, clientType, name): """ If device is not None, schedule the device to be collected. Otherwise log an error. - @param device: device to collect against - @type device: string + @param client: modelling client + @type client: object @param timeout: timeout before failing the connection @type timeout: integer @param clientType: description of the plugin type @@ -615,11 +595,11 @@ def addClient(self, device, timeout, clientType, name): @param name: plugin name @type name: string """ - if device: - device.timeout = timeout - device.timedOut = False - self.clients.append(device) - device.run() + if client: + client.timeout = timeout + client.timedOut = False + self.clients.append(client) + client.run() else: self.log.warn( "Unable to create a %s collector for %s", clientType, name @@ -690,7 +670,7 @@ def clientFinished(self, collectorClient): @type: Twisted deferred object """ device = collectorClient.device - self.log.debug("Client for %s finished collecting", device.id) + self.log.info("Client for %s finished collecting", device.id) def processClient(driver): try: @@ -1265,7 +1245,7 @@ def processOptions(self): if USE_WMI: setNTLMv2Auth(self.options) - configFilter = parseWorkerOptions(self.options.__dict__) + configFilter = parseWorkerOptions(self.options.__dict__, self.log) if configFilter: self.configFilter = configFilter diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 7f79295d39..9a2515ceb6 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -95,7 +95,15 @@ class Celery(object): CELERY_ACCEPT_CONTENT = ["without-unicode"] # List of modules to import when the Celery worker starts - CELERY_IMPORTS = ("Products.Jobber.jobs",) + CELERY_IMPORTS = ( + "Products.Jobber.jobs", + "Products.ZenCollector.configcache.task", + ) + + # Job/Task routing + CELERY_ROUTES = { + "configcache.build_device_config": {"queue": "configcache"} + } # Result backend (redis) CELERY_RESULT_BACKEND = ZenJobs.get("redis-url") diff --git a/Products/Jobber/log.py b/Products/Jobber/log.py index 28070a41b4..7566b4168c 100644 --- a/Products/Jobber/log.py +++ b/Products/Jobber/log.py @@ -40,107 +40,90 @@ _default_log_level = logging.getLevelName(ZenJobs.get("logseverity")) _default_config = { - "worker": { - "version": 1, - "disable_existing_loggers": False, - "filters": { - "main": { - "()": "Products.Jobber.utils.log.WorkerFilter", - }, + "version": 1, + "disable_existing_loggers": False, + "filters": { + "main": { + "()": "Products.Jobber.utils.log.WorkerFilter", }, - "formatters": { - "main": { - "()": "Products.Jobber.utils.log.TaskFormatter", - "base": ( - "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " - "worker=%(instance)s/%(processName)s: %(message)s" - ), - "task": ( - "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " - "worker=%(instance)s/%(processName)s " - "task=%(taskname)s taskid=%(taskid)s: %(message)s " - ), - "datefmt": "%Y-%m-%d %H:%M:%S", - }, - }, - "handlers": { - "main": { - "formatter": "main", - "class": "cloghandler.ConcurrentRotatingFileHandler", - "filename": os.path.join( - ZenJobs.get("logpath"), "zenjobs.log" - ), - "maxBytes": ZenJobs.get("maxlogsize") * 1024, - "backupCount": ZenJobs.get("maxbackuplogs"), - "mode": "a", - "filters": ["main"], - }, - }, - "loggers": { - "STDOUT": { - "level": _default_log_level, - }, - "zen": { - "level": _default_log_level, - }, - "zen.zenjobs": { - "level": _default_log_level, - "propagate": False, - "handlers": ["main"], - }, - "zen.zenjobs.job": { - "level": _default_log_level, - "propagate": False, - }, - "celery": { - "level": _default_log_level, - }, + }, + "formatters": { + "main": { + "()": "Products.Jobber.utils.log.TaskFormatter", + "base": ( + "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " + "worker=%(instance)s/%(processName)s: %(message)s" + ), + "task": ( + "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " + "worker=%(instance)s/%(processName)s " + "task=%(taskname)s taskid=%(taskid)s: %(message)s " + ), + "datefmt": "%Y-%m-%d %H:%M:%S", }, - "root": { - "handlers": ["main"], + "beat": { + "format": ( + "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " + "%(message)s" + ), + "datefmt": "%Y-%m-%d %H:%M:%S", }, }, - "beat": { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "beat": { - "format": ( - "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " - "%(message)s" - ), - "datefmt": "%Y-%m-%d %H:%M:%S", - }, + "handlers": {}, + "loggers": { + "STDOUT": { + "level": _default_log_level, }, - "handlers": { - "beat": { - "formatter": "beat", - "class": "cloghandler.ConcurrentRotatingFileHandler", - "filename": os.path.join( - ZenJobs.get("logpath"), "zenjobs-scheduler.log" - ), - "maxBytes": ZenJobs.get("maxlogsize") * 1024, - "backupCount": ZenJobs.get("maxbackuplogs"), - "mode": "a", - }, + "zen": { + "level": _default_log_level, }, - "loggers": { - "STDOUT": { - "level": _default_log_level, - }, - "zen": { - "level": _default_log_level, - }, - "celery": { - "level": _default_log_level, - }, - }, - "root": { - "handlers": ["beat"], + "celery": { + "level": _default_log_level, }, }, + "root": { + "handlers": [], + }, +} + +_main_loggers = { + "zen.zenjobs": { + "level": _default_log_level, + "propagate": False, + "handlers": ["main"], + }, + "zen.zenjobs.job": { + "level": _default_log_level, + "propagate": False, + }, +} +_configcache_loggers = { +} + +_main_handler = { + "formatter": "main", + "class": "cloghandler.ConcurrentRotatingFileHandler", + "filename": None, + "maxBytes": ZenJobs.get("maxlogsize") * 1024, + "backupCount": ZenJobs.get("maxbackuplogs"), + "mode": "a", + "filters": ["main"], +} +_beat_handler = { + "formatter": "beat", + "class": "cloghandler.ConcurrentRotatingFileHandler", + "filename": None, + "maxBytes": ZenJobs.get("maxlogsize") * 1024, + "backupCount": ZenJobs.get("maxbackuplogs"), + "mode": "a", } +_main_filename = os.path.join(ZenJobs.get("logpath"), "zenjobs.log") +_beat_filename = os.path.join(ZenJobs.get("logpath"), "zenjobs-scheduler.log") +_configcache_filename = os.path.join( + ZenJobs.get("logpath"), "configcache_builder.log" +) + _loglevelconf_filepath = zenPath("etc", "zenjobs_log_levels.conf") @@ -152,20 +135,24 @@ def _get_logger(name=None): return get_logger(name) -def get_default_config(name): - """Return the default logging configuration for the given name. - - :rtype: dict - """ - return _default_config[name] - - -def configure_logging(logfile=None, **kw): +def configure_logging(logfile="main", **kw): """Configure logging for zenjobs.""" - # Sketchy hack. Determine the logging config based on whether - # the logfile parameter is not None. - config_name = "worker" if logfile is None else "beat" - logging.config.dictConfig(get_default_config(config_name)) + # NOTE: Cleverly using the `-f` command line argument to specify + # which logging configuration to use. + if logfile in ("main", "configcache"): + _default_config["loggers"].update(**_main_loggers) + _default_config["root"]["handlers"].append("main") + if logfile == "main": + _main_handler["filename"] = _main_filename + else: + _main_handler["filename"] = _configcache_filename + _default_config["handlers"]["main"] = _main_handler + elif logfile == "beat": + _default_config["root"]["handlers"].append("beat") + _beat_handler["filename"] = _beat_filename + _default_config["handlers"]["beat"] = _beat_handler + + logging.config.dictConfig(_default_config) if os.path.exists(_loglevelconf_filepath): levelconfig = load_log_level_config(_loglevelconf_filepath) @@ -181,7 +168,7 @@ def configure_logging(logfile=None, **kw): sys.__stderr__ = errproxy sys.stderr = errproxy - if config_name == "beat": + if logfile == "beat": # The celery.beat module has a novel approach to getting its # logger, so fixing things so log messages can get sent where # we see them. diff --git a/Products/Jobber/task/event.py b/Products/Jobber/task/event.py index d5896ddd7b..30ca0d9115 100644 --- a/Products/Jobber/task/event.py +++ b/Products/Jobber/task/event.py @@ -41,7 +41,7 @@ def on_failure(self, exc, task_id, args, kwargs, einfo): def _send_event(task, exc, task_id, args, kwargs): - classkey, summary = _getErrorInfo(task.app, exc) + classkey, summary = _getErrorInfo(task, exc) name = task.getJobType() if hasattr(task, "getJobType") else task.name publisher = getUtility(IEventPublisher) event = Event.Event( @@ -67,21 +67,20 @@ def _send_event(task, exc, task_id, args, kwargs): mlog.info(*log_message) -def _getTimeoutSummary(app, ex): - return "Job killed after {}.".format( - humanize_timedelta( - timedelta( - seconds=app.conf.get("CELERYD_TASK_SOFT_TIME_LIMIT"), - ), - ), +def _getTimeoutSummary(task, ex): + _, soft_limit = task.request.timelimit or (None, None) + if soft_limit is None: + soft_limit = task.app.conf.get("CELERYD_TASK_SOFT_TIME_LIMIT") + return "Job timed out after {}.".format( + humanize_timedelta(timedelta(seconds=soft_limit)) ) -def _getAbortedSummary(app, ex): +def _getAbortedSummary(task, ex): return "Job aborted by user" -def _getErrorSummary(app, ex): +def _getErrorSummary(task, ex): return "{0.__class__.__name__}: {0}".format(ex) @@ -91,9 +90,9 @@ def _getErrorSummary(app, ex): } -def _getErrorInfo(app, ex): +def _getErrorInfo(task, ex): """Returns (eventkey, summary).""" key, summary_fn = _error_eventkey_map.get( type(ex).__name__, ("zenjobs-failure", _getErrorSummary) ) - return key, summary_fn(app, ex) + return key, summary_fn(task, ex) diff --git a/Products/Jobber/task/utils.py b/Products/Jobber/task/utils.py index 9307d40bbf..9a3e14df45 100644 --- a/Products/Jobber/task/utils.py +++ b/Products/Jobber/task/utils.py @@ -11,6 +11,8 @@ import inspect +from itertools import chain + from zope.component import getUtility from ..interfaces import IJobStore @@ -31,7 +33,12 @@ def requires(*features): if cls not in culled: culled.insert(0, cls) name = "".join(t.__name__ for t in features) + "Task" - basetask = type(name, tuple(culled), {"abstract": True}) + throws = set( + chain.from_iterable(getattr(cls, "throws", ()) for cls in culled) + ) + basetask = type( + name, tuple(culled), {"abstract": True, "throws": tuple(throws)} + ) return basetask diff --git a/Products/Jobber/tests/test_log.py b/Products/Jobber/tests/test_log.py index a00f71d113..86f17e4391 100644 --- a/Products/Jobber/tests/test_log.py +++ b/Products/Jobber/tests/test_log.py @@ -39,10 +39,8 @@ class ConfigureLoggingTest(TestCase): @patch("{src}.LoggingProxy".format(**PATH), autospec=True) @patch("{src}.apply_levels".format(**PATH), autospec=True) @patch("{src}.load_log_level_config".format(**PATH), autospec=True) - @patch("{src}.get_default_config".format(**PATH), autospec=True) def test_nominal( t, - _get_default_config, _load_log_level_config, _apply_levels, _LoggingProxy, @@ -52,7 +50,6 @@ def test_nominal( _sys, _os, ): - dictConfig = _logging.config.dictConfig exists = _os.path.exists getLogger = _logging.getLogger levelConfig = _load_log_level_config.return_value @@ -78,7 +75,6 @@ def test_nominal( configure_logging() - dictConfig.assert_called_once_with(_get_default_config.return_value) exists.assert_called_once_with(_loglevelconf_filepath) _load_log_level_config.assert_called_once_with(_loglevelconf_filepath) _apply_levels.assert_called_once_with(levelConfig) @@ -99,10 +95,8 @@ def test_nominal( @patch("{src}.LoggingProxy".format(**PATH), autospec=True) @patch("{src}.apply_levels".format(**PATH), autospec=True) @patch("{src}.load_log_level_config".format(**PATH), autospec=True) - @patch("{src}.get_default_config".format(**PATH), autospec=True) def test_missing_loglevel_file( t, - _get_default_config, _load_log_level_config, _apply_levels, _LoggingProxy, @@ -112,7 +106,6 @@ def test_missing_loglevel_file( _sys, _os, ): - dictConfig = _logging.config.dictConfig exists = _os.path.exists getLogger = _logging.getLogger logs = { @@ -137,7 +130,6 @@ def test_missing_loglevel_file( configure_logging() - dictConfig.assert_called_once_with(_get_default_config.return_value) exists.assert_called_once_with(_loglevelconf_filepath) _load_log_level_config.assert_has_calls([]) _apply_levels.assert_has_calls([]) diff --git a/Products/ZenCollector/CollectorCmdBase.py b/Products/ZenCollector/CollectorCmdBase.py deleted file mode 100644 index 2382e43eb5..0000000000 --- a/Products/ZenCollector/CollectorCmdBase.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2012, all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -import sys - -import zope.component - -from Products.ZenUtils.CmdBase import CmdBase - -from .daemon import CollectorDaemon -from .interfaces import IWorkerExecutor, IWorkerTaskFactory -from .tasks import SimpleTaskSplitter - - -class CollectorCmdBase(CmdBase): - def __init__( - self, - iCollectorWorkerClass, - iCollectorPreferencesClass, - noopts=0, - args=None, - ): - super(CollectorCmdBase, self).__init__(noopts, args) - self.workerClass = iCollectorWorkerClass - self.prefsClass = iCollectorPreferencesClass - - def run(self): - if "--worker" in sys.argv: - executor = zope.component.getUtility(IWorkerExecutor) - executor.setWorkerClass(self.workerClass) - executor.run() - else: - myPreferences = self.prefsClass() - myTaskFactory = zope.component.getUtility(IWorkerTaskFactory) - myTaskFactory.setWorkerClass(self.workerClass) - myTaskSplitter = SimpleTaskSplitter(myTaskFactory) - daemon = CollectorDaemon(myPreferences, myTaskSplitter) - myTaskFactory.postInitialization() - self.log = daemon.log - daemon.run() diff --git a/Products/ZenCollector/DeviceConfigCache.py b/Products/ZenCollector/DeviceConfigCache.py deleted file mode 100644 index 558e6411e4..0000000000 --- a/Products/ZenCollector/DeviceConfigCache.py +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2011, all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -import os - -from Products.ZenUtils.FileCache import FileCache - - -class DeviceConfigCache(object): - def __init__(self, basepath): - self.basepath = basepath - - def _getFileCache(self, monitor): - return FileCache(os.path.join(self.basepath, monitor)) - - def cacheConfigProxies(self, prefs, configs): - for cfg in configs: - self.updateConfigProxy(prefs, cfg) - - def updateConfigProxy(self, prefs, config): - cache = self._getFileCache(prefs.options.monitor) - key = config.configId - cache[key] = config - - def deleteConfigProxy(self, prefs, deviceid): - cache = self._getFileCache(prefs.options.monitor) - key = deviceid - try: - del cache[key] - except KeyError: - pass - - def getConfigProxies(self, prefs, cfgids): - cache = self._getFileCache(prefs.options.monitor) - if cfgids: - ret = [] - for cfgid in cfgids: - if cfgid in cache: - config = cache[cfgid] - if config: - ret.append(config) - return ret - else: - return filter(None, cache.values()) diff --git a/Products/ZenCollector/__init__.py b/Products/ZenCollector/__init__.py index 6f5ce1f780..d5c7c44b79 100644 --- a/Products/ZenCollector/__init__.py +++ b/Products/ZenCollector/__init__.py @@ -70,40 +70,3 @@ daemon = CollectorDaemon(myPreferences, myTaskSplitter) daemon.run() """ - -import zope.component -import zope.interface - -from .config import ConfigurationLoaderTask, ConfigurationProxy -from .interfaces import IFrameworkFactory -from .scheduler import Scheduler - - -@zope.interface.implementer(IFrameworkFactory) -class CoreCollectorFrameworkFactory(object): - def __init__(self): - self._configProxy = ConfigurationProxy() - self._scheduler = None - self._configurationLoader = ConfigurationLoaderTask - - def getConfigurationProxy(self): - return self._configProxy - - def getScheduler(self): - if self._scheduler is None: - self._scheduler = Scheduler() - return self._scheduler - - def getConfigurationLoaderTask(self): - return self._configurationLoader - - def getFrameworkBuildOptions(self): - return None - - -# Install the core collector framework factory as a Zope utility so it is -# available to all, and replaceable if necessary. -__factory__ = CoreCollectorFrameworkFactory() -zope.component.provideUtility(__factory__, IFrameworkFactory) -zope.component.provideUtility(__factory__, IFrameworkFactory, "core") -zope.component.provideUtility(__factory__, IFrameworkFactory, "nosip") diff --git a/Products/ZenCollector/collector.zcml b/Products/ZenCollector/collector.zcml new file mode 100644 index 0000000000..41f958fa4b --- /dev/null +++ b/Products/ZenCollector/collector.zcml @@ -0,0 +1,20 @@ + + + + + + + + diff --git a/Products/ZenCollector/config.py b/Products/ZenCollector/config.py deleted file mode 100644 index dc20210777..0000000000 --- a/Products/ZenCollector/config.py +++ /dev/null @@ -1,378 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2009, 2010, all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -""" -The config module provides the implementation of the IConfigurationProxy -interface used within Zenoss Core. This implementation provides basic -configuration retrieval services directly from a remote ZenHub service. -""" - -import logging -import time - -import zope.component -import zope.interface - -from cryptography.fernet import Fernet -from metrology import Metrology -from twisted.internet import defer -from twisted.python.failure import Failure - -from Products.ZenHub.PBDaemon import HubDown -from Products.ZenUtils.observable import ObservableMixin - -from .interfaces import ( - ICollector, - ICollectorPreferences, - IFrameworkFactory, - IConfigurationProxy, - IScheduledTask, - IDataService, - IEventService, -) -from .tasks import TaskStates - -log = logging.getLogger("zen.collector.config") - - -@zope.interface.implementer(IConfigurationProxy) -class ConfigurationProxy(object): - """ - This implementation of IConfigurationProxy provides basic configuration - retrieval from the remote ZenHub instance using the remote configuration - service proxy as specified by the collector's configuration. - """ - - _cipher_suite = None - - def getPropertyItems(self, prefs): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - self._collector = zope.component.queryUtility(ICollector) - serviceProxy = self._collector.getRemoteConfigServiceProxy() - - # Load any configuration properties for this daemon - log.debug("Fetching daemon configuration properties") - d = serviceProxy.callRemote("getConfigProperties") - d.addCallback(lambda result: dict(result)) - return d - - def getThresholdClasses(self, prefs): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - self._collector = zope.component.queryUtility(ICollector) - serviceProxy = self._collector.getRemoteConfigServiceProxy() - - log.debug("Fetching threshold classes") - d = serviceProxy.callRemote("getThresholdClasses") - return d - - def getThresholds(self, prefs): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - self._collector = zope.component.queryUtility(ICollector) - serviceProxy = self._collector.getRemoteConfigServiceProxy() - - log.debug("Fetching collector thresholds") - d = serviceProxy.callRemote("getCollectorThresholds") - return d - - def getConfigProxies(self, prefs, ids=[]): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - self._collector = zope.component.queryUtility(ICollector) - serviceProxy = self._collector.getRemoteConfigServiceProxy() - - log.debug("Fetching configurations") - # get options from prefs.options and send to remote - d = serviceProxy.callRemote( - "getDeviceConfigs", ids, options=prefs.options.__dict__ - ) - return d - - def deleteConfigProxy(self, prefs, id): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - # not implemented in the basic ConfigurationProxy - return defer.succeed(None) - - def updateConfigProxy(self, prefs, config): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - # not implemented in the basic ConfigurationProxy - return defer.succeed(None) - - def getConfigNames(self, result, prefs): - if not ICollectorPreferences.providedBy(prefs): - raise TypeError("config must provide ICollectorPreferences") - - self._collector = zope.component.queryUtility(ICollector) - serviceProxy = self._collector.getRemoteConfigServiceProxy() - - log.debug("Fetching device names") - d = serviceProxy.callRemote( - "getDeviceNames", options=prefs.options.__dict__ - ) - - def printNames(names): - log.debug( - "workerid %s Fetched Names %s %s", - prefs.options.workerid, - len(names), - names, - ) - return names - - d.addCallback(printNames) - return d - - @defer.inlineCallbacks - def _get_cipher_suite(self): - """ - Fetch the encryption key for this collector from zenhub. - """ - if self._cipher_suite is None: - self._collector = zope.component.queryUtility(ICollector) - proxy = self._collector.getRemoteConfigServiceProxy() - try: - key = yield proxy.callRemote("getEncryptionKey") - self._cipher_suite = Fernet(key) - except Exception as e: - log.warn("Remote exception: %s", e) - self._cipher_suite = None - defer.returnValue(self._cipher_suite) - - @defer.inlineCallbacks - def encrypt(self, data): - """ - Encrypt data using a key from zenhub. - """ - cipher_suite = yield self._get_cipher_suite() - encrypted_data = None - if cipher_suite: - try: - encrypted_data = yield cipher_suite.encrypt(data) - except Exception as e: - log.warn("Exception encrypting data %s", e) - defer.returnValue(encrypted_data) - - @defer.inlineCallbacks - def decrypt(self, data): - """ - Decrypt data using a key from zenhub. - """ - cipher_suite = yield self._get_cipher_suite() - decrypted_data = None - if cipher_suite: - try: - decrypted_data = yield cipher_suite.decrypt(data) - except Exception as e: - log.warn("Exception decrypting data %s", e) - defer.returnValue(decrypted_data) - - -@zope.interface.implementer(IScheduledTask) -class ConfigurationLoaderTask(ObservableMixin): - """ - A task that periodically retrieves collector configuration via the - IConfigurationProxy service. - """ - - STATE_CONNECTING = "CONNECTING" - STATE_FETCH_MISC_CONFIG = "FETCHING_MISC_CONFIG" - STATE_FETCH_DEVICE_CONFIG = "FETCHING_DEVICE_CONFIG" - STATE_PROCESS_DEVICE_CONFIG = "PROCESSING_DEVICE_CONFIG" - - _frameworkFactoryName = "core" - - def __init__( - self, - name, - configId=None, - scheduleIntervalSeconds=None, - taskConfig=None, - ): - super(ConfigurationLoaderTask, self).__init__() - self._fetchConfigTimer = Metrology.timer("collectordaemon.configs") - - # Needed for interface - self.name = name - self.configId = configId if configId else name - self.state = TaskStates.STATE_IDLE - - self._dataService = zope.component.queryUtility(IDataService) - self._eventService = zope.component.queryUtility(IEventService) - - if taskConfig is None: - raise TypeError("taskConfig cannot be None") - self._prefs = taskConfig - self.interval = self._prefs.configCycleInterval * 60 - self.options = self._prefs.options - - self._daemon = zope.component.getUtility(ICollector) - self._daemon.heartbeatTimeout = self.options.heartbeatTimeout - log.debug( - "Heartbeat timeout set to %ds", self._daemon.heartbeatTimeout - ) - - frameworkFactory = zope.component.queryUtility( - IFrameworkFactory, self._frameworkFactoryName - ) - self._configProxy = frameworkFactory.getConfigurationProxy() - - self.devices = [] - self.startDelay = 0 - - def doTask(self): - """ - Contact zenhub and gather configuration data. - - @return: A task to gather configs - @rtype: Twisted deferred object - """ - log.debug("%s gathering configuration", self.name) - self.startTime = time.time() - - # Were we given a command-line option to collect a single device? - if self.options.device: - self.devices = [self.options.device] - - d = self._baseConfigs() - self._deviceConfigs(d, self.devices) - d.addCallback(self._notifyConfigLoaded) - d.addErrback(self._handleError) - return d - - def _baseConfigs(self): - """ - Load the configuration that doesn't depend on loading devices. - """ - d = self._fetchPropertyItems() - d.addCallback(self._processPropertyItems) - d.addCallback(self._fetchThresholdClasses) - d.addCallback(self._processThresholdClasses) - d.addCallback(self._fetchThresholds) - d.addCallback(self._processThresholds) - return d - - def _deviceConfigs(self, d, devices): - """ - Load the device configuration - """ - d.addCallback(self._fetchConfig, devices) - d.addCallback(self._processConfig) - - def _notifyConfigLoaded(self, result): - # This method is prematuraly called in enterprise bc - # _splitConfiguration calls defer.succeed after creating - # a new task for incremental loading - self._daemon.runPostConfigTasks() - return defer.succeed("Configuration loaded") - - def _handleError(self, result): - if isinstance(result, Failure): - log.error( - "Task %s configure failed: %s", - self.name, - result.getErrorMessage(), - ) - - # stop if a single device was requested and nothing found - if self.options.device or not self.options.cycle: - self._daemon.stop() - - ex = result.value - if isinstance(ex, HubDown): - result = str(ex) - # Allow the loader to be reaped and re-added - self.state = TaskStates.STATE_COMPLETED - return result - - def _fetchPropertyItems(self, previous_cb_result=None): - return defer.maybeDeferred( - self._configProxy.getPropertyItems, self._prefs - ) - - def _fetchThresholdClasses(self, previous_cb_result): - return defer.maybeDeferred( - self._configProxy.getThresholdClasses, self._prefs - ) - - def _fetchThresholds(self, previous_cb_result): - return defer.maybeDeferred( - self._configProxy.getThresholds, self._prefs - ) - - def _fetchConfig(self, result, devices): - self.state = self.STATE_FETCH_DEVICE_CONFIG - start = time.time() - - def recordTime(result): - # get in milliseconds - duration = int((time.time() - start) * 1000) - self._fetchConfigTimer.update(duration) - return result - - d = defer.maybeDeferred( - self._configProxy.getConfigProxies, self._prefs, devices - ) - d.addCallback(recordTime) - return d - - def _processPropertyItems(self, propertyItems): - log.debug("Processing received property items") - self.state = self.STATE_FETCH_MISC_CONFIG - if propertyItems: - self._daemon._setCollectorPreferences(propertyItems) - - def _processThresholdClasses(self, thresholdClasses): - log.debug("Processing received threshold classes") - if thresholdClasses: - self._daemon._loadThresholdClasses(thresholdClasses) - - def _processThresholds(self, thresholds): - log.debug("Processing received thresholds") - if thresholds: - self._daemon._configureThresholds(thresholds) - - @defer.inlineCallbacks - def _processConfig(self, configs, purgeOmitted=True): - log.debug("Processing %s received device configs", len(configs)) - if self.options.device: - configs = [ - cfg - for cfg in configs - if self.options.device in (cfg.id, cfg.configId) - ] - if not configs: - log.error( - "Configuration for %s unavailable -- " - "is that the correct name?", - self.options.device, - ) - - if not configs: - # No devices (eg new install), -d name doesn't exist or - # device explicitly ignored by zenhub service. - if not self.options.cycle: - self._daemon.stop() - defer.returnValue(["No device configuration to load"]) - - self.state = self.STATE_PROCESS_DEVICE_CONFIG - yield self._daemon._updateDeviceConfigs(configs, purgeOmitted) - defer.returnValue(configs) - - def cleanup(self): - pass # Required by interface diff --git a/Products/ZenCollector/config/__init__.py b/Products/ZenCollector/config/__init__.py new file mode 100644 index 0000000000..04257133d5 --- /dev/null +++ b/Products/ZenCollector/config/__init__.py @@ -0,0 +1,17 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from .proxy import ConfigurationProxy +from .task import ConfigurationLoaderTask, DeviceConfigLoader + +__all__ = ( + "ConfigurationLoaderTask", + "ConfigurationProxy", + "DeviceConfigLoader", +) diff --git a/Products/ZenCollector/config/proxy.py b/Products/ZenCollector/config/proxy.py new file mode 100644 index 0000000000..1e049c4365 --- /dev/null +++ b/Products/ZenCollector/config/proxy.py @@ -0,0 +1,142 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2009, 2010, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +""" +The config module provides the implementation of the IConfigurationProxy +interface used within Zenoss Core. This implementation provides basic +configuration retrieval services directly from a remote ZenHub service. +""" + +import logging + +from cryptography.fernet import Fernet +from twisted.internet import defer +from zope.component import queryUtility +from zope.interface import implementer + +from ..interfaces import ICollector, IConfigurationProxy + +log = logging.getLogger("zen.collector.configurationproxy") + + +@implementer(IConfigurationProxy) +class ConfigurationProxy(object): + """ + This implementation of IConfigurationProxy provides basic configuration + retrieval from the remote ZenHub instance using the remote configuration + service proxy as specified by the collector's configuration. + """ + + _cipher_suite = None + + def __init__(self, prefs): + super(ConfigurationProxy, self).__init__() + self._prefs = prefs + self._collector = queryUtility(ICollector) + + @defer.inlineCallbacks + def getPropertyItems(self): + ref = yield self._collector.getRemoteConfigServiceProxy() + result = yield ref.callRemote("getConfigProperties") + log.info("fetched daemon configuration properties") + props = dict(result) + defer.returnValue(props) + + @defer.inlineCallbacks + def getThresholdClasses(self): + ref = yield self._collector.getRemoteConfigServiceProxy() + classes = yield ref.callRemote("getThresholdClasses") + log.info("fetched threshold classes") + defer.returnValue(classes) + + @defer.inlineCallbacks + def getThresholds(self): + ref = yield self._collector.getRemoteConfigServiceProxy() + try: + thresholds = yield ref.callRemote("getCollectorThresholds") + log.info("fetched collector thresholds") + defer.returnValue(thresholds) + except Exception: + log.exception("getThresholds failed") + + @defer.inlineCallbacks + def getConfigProxies(self, token, deviceIds): + ref = yield self._collector.getRemoteConfigCacheProxy() + + log.debug("fetching configurations") + # get options from prefs.options and send to remote + proxies = yield ref.callRemote( + "getDeviceConfigs", + self._prefs.configurationService, + token, + deviceIds, + options=self._prefs.options.__dict__, + ) + defer.returnValue(proxies) + + @defer.inlineCallbacks + def getConfigNames(self): + ref = yield self._collector.getRemoteConfigCacheProxy() + + # log.info("fetching device names") + names = yield ref.callRemote( + "getDeviceNames", + self._prefs.configurationService, + options=self._prefs.options.__dict__, + ) + log.info( + "workerid %s fetched names %s %s", + self._prefs.options.workerid, + len(names), + names, + ) + defer.returnValue(names) + + @defer.inlineCallbacks + def _get_cipher_suite(self): + """ + Fetch the encryption key for this collector from zenhub. + """ + if self._cipher_suite is None: + ref = yield self._collector.getRemoteConfigServiceProxy() + try: + key = yield ref.callRemote("getEncryptionKey") + self._cipher_suite = Fernet(key) + except Exception as e: + log.warn("remote exception: %s", e) + self._cipher_suite = None + defer.returnValue(self._cipher_suite) + + @defer.inlineCallbacks + def encrypt(self, data): + """ + Encrypt data using a key from zenhub. + """ + cipher_suite = yield self._get_cipher_suite() + encrypted_data = None + if cipher_suite: + try: + encrypted_data = yield cipher_suite.encrypt(data) + except Exception as e: + log.warn("exception encrypting data %s", e) + defer.returnValue(encrypted_data) + + @defer.inlineCallbacks + def decrypt(self, data): + """ + Decrypt data using a key from zenhub. + """ + cipher_suite = yield self._get_cipher_suite() + decrypted_data = None + if cipher_suite: + try: + decrypted_data = yield cipher_suite.decrypt(data) + except Exception as e: + log.warn("exception decrypting data %s", e) + defer.returnValue(decrypted_data) diff --git a/Products/ZenCollector/config/task.py b/Products/ZenCollector/config/task.py new file mode 100644 index 0000000000..aebc32d662 --- /dev/null +++ b/Products/ZenCollector/config/task.py @@ -0,0 +1,207 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2009, 2010, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +""" +The config module provides the implementation of the IConfigurationProxy +interface used within Zenoss Core. This implementation provides basic +configuration retrieval services directly from a remote ZenHub service. +""" + +import itertools +import logging +import time + +from metrology import Metrology +from twisted.internet import defer +from zope.component import getUtility, queryUtility +from zope.interface import implementer + +from Products.ZenHub.PBDaemon import HubDown +from Products.ZenUtils.observable import ObservableMixin + +from ..tasks import TaskStates +from ..interfaces import ( + ICollector, + IDataService, + IEventService, + IScheduledTask, + IFrameworkFactory, +) + +log = logging.getLogger("zen.collector.config") + + +@implementer(IScheduledTask) +class ConfigurationLoaderTask(ObservableMixin): + """ + Periodically retrieves collector configuration via the + IConfigurationProxy service. + """ + + STATE_CONNECTING = "CONNECTING" + STATE_FETCH_MISC_CONFIG = "FETCHING_MISC_CONFIG" + STATE_FETCH_DEVICE_CONFIG = "FETCHING_DEVICE_CONFIG" + STATE_PROCESS_DEVICE_CONFIG = "PROCESSING_DEVICE_CONFIG" + + def __init__( + self, + name, + configId=None, + scheduleIntervalSeconds=None, + taskConfig=None, + ): + if taskConfig is None: + raise TypeError("taskConfig cannot be None") + + super(ConfigurationLoaderTask, self).__init__() + + self._fetchConfigTimer = Metrology.timer("collectordaemon.configs") + + # Needed for interface + self.name = name + self.configId = configId if configId else name + self.state = TaskStates.STATE_IDLE + + self._dataService = queryUtility(IDataService) + self._eventService = queryUtility(IEventService) + + self._prefs = taskConfig + self.interval = self._prefs.configCycleInterval * 60 + self.options = self._prefs.options + + self._collector = getUtility(ICollector) + self._collector.heartbeatTimeout = self.options.heartbeatTimeout + log.debug( + "heartbeat timeout set to %ds", self._collector.heartbeatTimeout + ) + + frameworkFactory = queryUtility( + IFrameworkFactory, self._collector.frameworkFactoryName + ) + self._configProxy = frameworkFactory.getConfigurationProxy() + + self.startDelay = 0 + + @defer.inlineCallbacks + def doTask(self): + """ + Contact zenhub and gather configuration data. + + @return: A task to gather configs + @rtype: Twisted deferred object + """ + log.debug("%s gathering configuration", self.name) + self.startTime = time.time() + + proxy = self._configProxy + try: + propertyItems = yield proxy.getPropertyItems() + self._processPropertyItems(propertyItems) + + thresholdClasses = yield proxy.getThresholdClasses() + self._processThresholdClasses(thresholdClasses) + + thresholds = yield proxy.getThresholds() + self._processThresholds(thresholds) + + self._collector.runPostConfigTasks() + except Exception as ex: + log.exception("task '%s' failed", self.name) + + # stop if a single device was requested and nothing found + if self.options.device or not self.options.cycle: + self._collector.stop() + + if isinstance(ex, HubDown): + # Allow the loader to be reaped and re-added + self.state = TaskStates.STATE_COMPLETED + + def _processPropertyItems(self, propertyItems): + log.debug("processing received property items") + self.state = self.STATE_FETCH_MISC_CONFIG + if propertyItems: + self._collector._setCollectorPreferences(propertyItems) + + def _processThresholdClasses(self, thresholdClasses): + log.debug("processing received threshold classes") + if thresholdClasses: + self._collector._loadThresholdClasses(thresholdClasses) + + def _processThresholds(self, thresholds): + log.debug("processing received thresholds") + if thresholds: + self._collector._configureThresholds(thresholds) + + def cleanup(self): + pass # Required by interface + + +class DeviceConfigLoader(object): + """Handles retrieving devices from the ConfigCache service.""" + + def __init__(self, options, proxy, callback): + self._options = options + self._proxy = proxy + self._callback = callback + self._deviceIds = set([options.device] if options.device else []) + self._changes_since = 0 + + @property + def deviceIds(self): + return self._deviceIds + + @defer.inlineCallbacks + def __call__(self): + try: + next_time = time.time() + config_data = yield self._proxy.getConfigProxies( + self._changes_since, self._deviceIds + ) + yield self._processConfigs(config_data) + self._changes_since = next_time + except Exception: + log.exception("failed to retrieve device configs") + + @defer.inlineCallbacks + def _processConfigs(self, config_data): + new = config_data.get("new", []) + updated = config_data.get("updated", []) + removed = config_data.get("removed", []) + try: + if self._options.device: + configs = [ + cfg + for cfg in itertools.chain(new, updated) + if self._options.device in (cfg.id, cfg.configId) + ] + if not configs: + log.error( + "configuration for %s unavailable -- " + "is that the correct name?", + self._options.device, + ) + defer.returnValue(None) + + if not new and not updated: + defer.returnValue(None) + + # self.state = self.STATE_PROCESS_DEVICE_CONFIG + yield self._callback(new, updated, removed) + finally: + self._update_local_cache(new, updated, removed) + log.info( + "processed %d new, %d updated, and %d removed device configs", + len(new), + len(updated), + len(removed), + ) + + def _update_local_cache(self, new, updated, removed): + self._deviceIds.difference_update(removed) + self._deviceIds.update(cfg.id for cfg in new) diff --git a/Products/ZenCollector/configcache/__init__.py b/Products/ZenCollector/configcache/__init__.py new file mode 100644 index 0000000000..bf7884f0d0 --- /dev/null +++ b/Products/ZenCollector/configcache/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## diff --git a/Products/ZenCollector/configcache/__main__.py b/Products/ZenCollector/configcache/__main__.py new file mode 100644 index 0000000000..03365d61b7 --- /dev/null +++ b/Products/ZenCollector/configcache/__main__.py @@ -0,0 +1,21 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +__all__ = ("main",) + + +def main(): + from .configcache import main + main() + + +if __name__ == "__main__": + main() diff --git a/Products/ZenCollector/configcache/app/__init__.py b/Products/ZenCollector/configcache/app/__init__.py new file mode 100644 index 0000000000..c972e8d2cb --- /dev/null +++ b/Products/ZenCollector/configcache/app/__init__.py @@ -0,0 +1,17 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from .base import Application +from .init import initialize_environment +from .pid import pidfile + + +__all__ = ("Application", "initialize_environment", "pidfile") diff --git a/Products/ZenCollector/configcache/app/base.py b/Products/ZenCollector/configcache/app/base.py new file mode 100644 index 0000000000..56f6612f44 --- /dev/null +++ b/Products/ZenCollector/configcache/app/base.py @@ -0,0 +1,139 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging + +from signal import signal, SIGTERM, SIGHUP, SIGINT +from threading import Event + +import attr + +from MySQLdb import OperationalError + +# from ..misc import app_name + +from .config import add_config_arguments, getConfigFromArguments +from .init import initialize_environment +from .genconf import GenerateConfig +from .logger import add_logging_arguments, setup_logging, setup_debug_logging +from .metrics import MetricManager +from .pid import add_pidfile_arguments, pidfile +from .zodb import add_zodb_arguments, zodb + +_delay = 10 # seconds + + +class Application(object): + """Base class for applications.""" + + @classmethod + def from_args(cls, args): + config = getConfigFromArguments(args.parser, args) + return cls(config, args.task) + + def __init__(self, config, task): + self.config = config + self.task = task + + def run(self): + configs = getattr(self.task, "configs", ()) + overrides = getattr(self.task, "config_overrides", ()) + initialize_environment(configs=configs, overrides=overrides) + setup_logging(self.config) + setup_debug_logging(self.config) + with pidfile(self.config): + stop = Event() + set_shutdown_handler(lambda x, y: _handle_signal(stop, x, y)) + controller = _Controller(stop) + log = logging.getLogger( + "zen.{}".format(self.task.__module__.split(".", 2)[-1]) + ) + log.info("application has started") + try: + # Setup Metric Reporting + metric_manager = MetricManager( + daemon_tags={ + "zenoss_daemon": "configcache", + "internal": True, + } + ) + while not controller.shutdown: + try: + with zodb(self.config) as (db, session, dmd): + ctx = ApplicationContext( + controller, + db, + session, + dmd, + metric_manager, + ) + self.task(self.config, ctx).run() + except OperationalError as oe: + log.warn("Lost database connection: %s", oe) + except Exception: + log.exception("unhandled error") + controller.wait(_delay) + except BaseException as e: + log.warn("shutting down due to %s", e) + controller.quit() + finally: + log.info("application is quitting") + + @staticmethod + def add_genconf_command(subparsers, parsers): + GenerateConfig.add_command(subparsers, parsers) + pass + + @staticmethod + def add_all_arguments(parser): + add_config_arguments(parser) + add_pidfile_arguments(parser) + add_logging_arguments(parser) + add_zodb_arguments(parser) + + add_config_arguments = staticmethod(add_config_arguments) + add_pidfile_arguments = staticmethod(add_pidfile_arguments) + add_logging_arguments = staticmethod(add_logging_arguments) + add_zodb_arguments = staticmethod(add_zodb_arguments) + + +@attr.s(frozen=True, slots=True) +class ApplicationContext(object): + controller = attr.ib() + db = attr.ib() + session = attr.ib() + dmd = attr.ib() + metrics = attr.ib() + + +class _Controller(object): + def __init__(self, stop): + self.__stop = stop + + @property + def shutdown(self): + return self.__stop.is_set() + + def quit(self): + self.__stop.set() + + def wait(self, interval): + self.__stop.wait(interval) + + +def _handle_signal(stop, signum, frame): + stop.set() + + +def set_shutdown_handler(func): + signal(SIGTERM, func) + signal(SIGHUP, func) + signal(SIGINT, func) diff --git a/Products/ZenCollector/configcache/app/config.py b/Products/ZenCollector/configcache/app/config.py new file mode 100644 index 0000000000..b7cba77f85 --- /dev/null +++ b/Products/ZenCollector/configcache/app/config.py @@ -0,0 +1,101 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import os + +from Products.ZenUtils.config import Config, ConfigLoader +from Products.ZenUtils.GlobalConfig import getGlobalConfiguration +from Products.ZenUtils.Utils import zenPath + + +def add_config_arguments(parser): + filename = "-".join(parser.prog.split(" ")[:-1]) + ".conf" + parser.add_argument( + "-C", + "--configfile", + default=os.path.join(zenPath("etc"), filename), + help="Pathname of the configuration file" + ) + + +def getConfigFromArguments(parser, args): + """ + Return a dict containing the configuration. + + @type args: argparse.Namespace + """ + options = tuple( + (cfg_name, opt_name, xform, default) + for cfg_name, opt_name, xform, default in ( + ( + _long_name(act.option_strings), + act.dest, + act.type if act.type is not None else _identity, + act.default, + ) + for act in parser._actions + if act.dest not in ("help", "version", "configfile") + ) + if cfg_name is not None + ) + dest_names = { + long_name: dest_name + for long_name, dest_name, _, _ in options + } + xforms = { + long_name: xform + for long_name, _, xform, _ in options + } + defaults = { + long_name: default + for long_name, _, _, default in options + } + config = defaults.copy() + config.update( + (key, xforms[key](value)) + for key, value in getGlobalConfiguration().items() + if key in dest_names + ) + + configfile = getattr(args, "configfile", None) + if configfile: + app_config_loader = ConfigLoader(configfile, Config) + try: + config.update( + (key, xforms[key](value)) + for key, value in app_config_loader().items() + if key in dest_names + ) + except IOError as ex: + # Re-raise exception if the error is not "File not found" + if ex.errno != 2: + raise + + # Apply command-line overrides. An override is a value from the + # command line that differs from the default. This does mean that + # explicitely specified default values on the CLI are ignored. + config.update( + (cname, override) + for cname, default, override in ( + (cname, defaults[cname], getattr(args, oname, None)) + for cname, oname in dest_names.items() + ) + if override != default + ) + return config + + +def _long_name(names): + name = next((nm for nm in names if nm.startswith("--")), None) + if name: + return name[2:] + + +def _identity(value): + return value diff --git a/Products/ZenCollector/configcache/app/genconf.py b/Products/ZenCollector/configcache/app/genconf.py new file mode 100644 index 0000000000..9e5123619d --- /dev/null +++ b/Products/ZenCollector/configcache/app/genconf.py @@ -0,0 +1,60 @@ +from __future__ import print_function + +import textwrap + +from ..misc.args import get_subparser + +# List of options to not include when generating a config file. +_ARGS_TO_IGNORE = ( + "", + "configfile", + "help", +) + + +class GenerateConfig(object): + + description = "Write an example config file to stdout" + + @staticmethod + def add_command(subparsers, parsers): + subp_genconf = get_subparser( + subparsers, "genconf", GenerateConfig.description + ) + subp_genconf.set_defaults( + factory=GenerateConfig, + parsers=parsers, + ) + + def __init__(self, args): + self._parsers = args.parsers + + def run(self): + actions = [] + for parser in self._parsers: + for action in parser._actions: + if action.dest in _ARGS_TO_IGNORE: + continue + if any(action.dest == act[1] for act in actions): + continue + actions.append( + ( + action.help, + action.dest.replace("_", "-"), + action.default, + ) + ) + print(_item_as_text(actions[0])) + for act in actions[1:]: + print() + print(_item_as_text(act)) + + +def _item_as_text(item): + return "{}\n#{} {}".format( + "\n".join( + "# {}".format(line) for line in textwrap.wrap(item[0], width=75) + ), + item[1], + item[2], + ) diff --git a/Products/ZenCollector/configcache/app/init.py b/Products/ZenCollector/configcache/app/init.py new file mode 100644 index 0000000000..c5de57e180 --- /dev/null +++ b/Products/ZenCollector/configcache/app/init.py @@ -0,0 +1,27 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from OFS.Application import import_products +from Zope2.App import zcml + +import Products.ZenWidgets + +from Products.ZenUtils.Utils import load_config, load_config_override +from Products.ZenUtils.zenpackload import load_zenpacks + + +def initialize_environment(configs=(), overrides=()): + import_products() + load_zenpacks() + zcml.load_site() + load_config_override('scriptmessaging.zcml', Products.ZenWidgets) + for filepath, module in configs: + load_config(filepath, module) + for filepath, module in overrides: + load_config_override(filepath, module) diff --git a/Products/ZenCollector/configcache/app/logger.py b/Products/ZenCollector/configcache/app/logger.py new file mode 100644 index 0000000000..ef79e4b600 --- /dev/null +++ b/Products/ZenCollector/configcache/app/logger.py @@ -0,0 +1,189 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import argparse +import copy +import logging +import logging.config +import logging.handlers +import os +import signal + +from Products.ZenUtils.Utils import zenPath + +_default_config_template = { + "version": 1, + "disable_existing_loggers": True, + "filters": {}, + "formatters": { + "main": { + "format": ( + "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: " + "%(message)s" + ), + "datefmt": "%Y-%m-%d %H:%M:%S", + }, + }, + "handlers": { + "main": { + "formatter": "main", + "class": "cloghandler.ConcurrentRotatingFileHandler", + "filename": None, + "maxBytes": None, + "backupCount": None, + "mode": "a", + "filters": [], + } + }, + "loggers": { + "": {"level": logging.WARN}, + "zen": {"level": logging.NOTSET}, + }, + "root": { + "handlers": ["main"], + }, +} + + +def setup_logging(config): + """Create formatting for log entries and set default log level.""" + logconfig = copy.deepcopy(_default_config_template) + loglevel = config["log-level"] + logconfig["loggers"]["zen"]["level"] = loglevel + logconfig["handlers"]["main"]["filename"] = config["log-filename"] + logconfig["handlers"]["main"]["maxBytes"] = ( + config["log-max-file-size"] * 1024 + ) + logconfig["handlers"]["main"]["backupCount"] = config["log-max-file-count"] + logging.config.dictConfig(logconfig) + + +def setup_debug_logging(config): + # Allow the user to dynamically lower and raise the logging + # level without restarts. + try: + signal.signal( + signal.SIGUSR1, + lambda x, y: _debug_logging_switch(config["log-level"], x, y), + ) + except ValueError: + # If we get called multiple times, this will generate an exception: + # ValueError: signal only works in main thread + # Ignore it as we've already set up the signal handler. + pass + + +def _debug_logging_switch(default_level, signum, frame): + zenlog = logging.getLogger("zen") + currentlevel = zenlog.getEffectiveLevel() + if currentlevel == logging.DEBUG: + if currentlevel == default_level: + return + zenlog.setLevel(default_level) + logging.getLogger().setLevel(logging.WARN) + zenlog.info( + "restored logging level back to %s (%d)", + logging.getLevelName(default_level) or "unknown", + default_level, + ) + else: + zenlog.setLevel(logging.NOTSET) + logging.getLogger().setLevel(logging.DEBUG) + zenlog.info( + "logging level set to %s (%d)", + logging.getLevelName(logging.DEBUG), + logging.DEBUG, + ) + + +def _level_as_int(v): + try: + return int(v) + except ValueError: + return logging.getLevelName(v.upper()) + + +def _add_log_suffix(v): + if not v.endswith(".log"): + if not os.path.basename(v): + raise ValueError("no filename for log file given") + return v + ".log" + return v + + +class LogLevel(argparse.Action): + """Define a 'logging level' action for argparse.""" + + def __init__( + self, + option_strings, + dest, + nargs=None, + const=None, + default="info", + type=None, + choices=None, + help="Default logging severity level", + **kwargs + ): + if nargs is not None: + raise ValueError("'nargs' not supported for LogLevel action") + if type is not None: + raise ValueError("'type' not supported for LogLevel action") + if const is not None: + raise ValueError("'const' not supported for LogLevel action") + choices = tuple( + value + for pair in sorted( + (level_id, level_name.lower()) + for level_id, level_name in logging._levelNames.items() + if isinstance(level_id, int) and level_id != 0 + ) + for value in pair + ) + super(LogLevel, self).__init__( + option_strings, + dest, + default=default, + type=_level_as_int, + choices=choices, + help=help, + **kwargs + ) + + def __call__(self, parser, namespace, values=None, option_string=None): + setattr(namespace, self.dest, values) + + +def add_logging_arguments(parser): + group = parser.add_argument_group("Logging Options") + group.add_argument("-v", "--log-level", action=LogLevel) + filename = "-".join(parser.prog.split(" ")[:-1]) + ".log" + dirname = zenPath("log") + group.add_argument( + "--log-filename", + default=os.path.join(dirname, filename), + type=_add_log_suffix, + help="Pathname of the log file. If a directory path is not " + "specified, the log file is save to {}".format(dirname), + ) + group.add_argument( + "--log-max-file-size", + default=10240, + type=int, + help="Maximum size of log file in KB before starting a new file", + ) + group.add_argument( + "--log-max-file-count", + default=3, + type=int, + help="Maximum number of archival log files to keep", + ) diff --git a/Products/ZenCollector/configcache/app/metrics.py b/Products/ZenCollector/configcache/app/metrics.py new file mode 100644 index 0000000000..1fc4d0cfb3 --- /dev/null +++ b/Products/ZenCollector/configcache/app/metrics.py @@ -0,0 +1,81 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import os + +from Products.ZenUtils.MetricReporter import MetricReporter +from Products.ZenUtils.metricwriter import ( + AggregateMetricWriter, + FilteredMetricWriter, + MetricWriter, +) + +from Products.ZenHub.metricpublisher.publisher import ( + RedisListPublisher, + HttpPostPublisher, +) + + +class MetricManager(object): + """General interface for storing and reporting metrics + metric publisher: publishes metrics to an external system (redis, http) + metric writer: drives metric pulisher(s), calling their .put method + metric reporter: once its .start method is called, + periodically calls writer.write_metric, to publish stored metrics + """ + + def __init__(self, daemon_tags): + self.daemon_tags = daemon_tags + self._metric_writer = None + self._metric_reporter = None + + def start(self): + self.metricreporter.start() + + def stop(self): + self.metricreporter.stop() + + @property + def metricreporter(self): + if not self._metric_reporter: + self._metric_reporter = MetricReporter( + metricWriter=self.metric_writer, tags=self.daemon_tags + ) + + return self._metric_reporter + + @property + def metric_writer(self): + if not self._metric_writer: + self._metric_writer = _cc_metric_writer_factory() + + return self._metric_writer + + +def _cc_metric_writer_factory(): + metric_writer = MetricWriter(RedisListPublisher()) + cc = os.environ.get("CONTROLPLANE", "0") == "1" + internal_url = os.environ.get("CONTROLPLANE_CONSUMER_URL", None) + if cc and internal_url: + username = os.environ.get("CONTROLPLANE_CONSUMER_USERNAME", "") + password = os.environ.get("CONTROLPLANE_CONSUMER_PASSWORD", "") + _publisher = HttpPostPublisher(username, password, internal_url) + internal_metric_writer = FilteredMetricWriter( + _publisher, _internal_metric_filter + ) + metric_writer = AggregateMetricWriter( + [metric_writer, internal_metric_writer] + ) + return metric_writer + + +def _internal_metric_filter(metric, value, timestamp, tags): + return tags and tags.get("internal", False) diff --git a/Products/ZenCollector/configcache/app/pid.py b/Products/ZenCollector/configcache/app/pid.py new file mode 100644 index 0000000000..02effb5723 --- /dev/null +++ b/Products/ZenCollector/configcache/app/pid.py @@ -0,0 +1,130 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import atexit +import errno +import fcntl +import logging +import os + +from Products.ZenUtils.Utils import zenPath + +log = logging.getLogger("zen.pid") + + +def _add_pid_suffix(v): + if not v.endswith(".pid"): + if not os.path.basename(v): + raise ValueError("no filename for pid file given") + return v + ".pid" + return v + + +def add_pidfile_arguments(parser): + filename = "-".join(parser.prog.split(" ")[:-1]) + ".pid" + dirname = os.path.join(zenPath("var"), "run") + parser.add_argument( + "--pidfile", + default=os.path.join(dirname, filename), + type=_add_pid_suffix, + help="Pathname of the PID file. If a directory path is not " + "specified, the pidfile is save to {}".format(dirname), + ) + + +class pidfile(object): + """ + Write a file containing the current process's PID. + + The context manager yields the PID value to the caller. + """ + + def __init__(self, config): + pidfile = config["pidfile"] + filename = os.path.basename(pidfile) + dirname = os.path.dirname(pidfile) + if not os.path.isdir(dirname): + if not dirname: + dirname = os.path.join(zenPath("var"), "run") + else: + raise RuntimeError( + "not a directory direcory={}".format(dirname) + ) + self._dirname = dirname + self._filename = filename + self.pathname = os.path.join(self._dirname, self._filename) + + def __enter__(self): + self.create() + return self + + def __exit__(self, exc_type=None, exc_value=None, exc_tb=None): + self.close() + + def read(self): + with open(self.pathname, "r") as fp: + return _read_pidfile(fp) + + def create(self): + atexit.register(self.close) + self.pid = os.getpid() + self.fp = open(self.pathname, "a+") + try: + _flock(self.fp.fileno()) + except IOError as ex: + raise RuntimeError( + "pidfile already locked pidfile={} error={}".format( + self.pathname, ex + ) + ) + oldpid = _read_pidfile(self.fp) + if oldpid is not None and pid_exists(oldpid): + raise RuntimeError("PID is still running pid={}".format(oldpid)) + self.fp.seek(0) + self.fp.truncate() + self.fp.write("%d\n" % self.pid) + self.fp.flush() + self.fp.seek(0) + + def close(self): + if not self.fp: + return + try: + self.fp.close() + self.fp = None # so subsequent calls to `close` exit early + except IOError as ex: + if ex.errno != errno.EBADF: + raise + finally: + if os.path.isfile(self.pathname): + os.remove(self.pathname) + + +def pid_exists(pid): + try: + os.kill(pid, 0) + except OSError as ex: + if ex.errno == errno.ESRCH: + # This pid has no matching process + return False + return True + + +def _read_pidfile(fp): + fp.seek(0) + pid_str = fp.read(16).split("\n", 1)[0].strip() + if not pid_str: + return None + return int(pid_str) + + +def _flock(fileno): + fcntl.flock(fileno, fcntl.LOCK_EX | fcntl.LOCK_NB) diff --git a/Products/ZenCollector/configcache/app/zodb.py b/Products/ZenCollector/configcache/app/zodb.py new file mode 100644 index 0000000000..4026a21064 --- /dev/null +++ b/Products/ZenCollector/configcache/app/zodb.py @@ -0,0 +1,235 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import contextlib +import os + +import transaction +import ZODB.config + +from AccessControl.SecurityManagement import ( + newSecurityManager, + noSecurityManager, +) +from Products.CMFCore.utils import getToolByName +from ZPublisher.HTTPRequest import HTTPRequest +from ZPublisher.HTTPResponse import HTTPResponse +from ZPublisher.BaseRequest import RequestContainer + +from Products.ZenRelations.ZenPropertyManager import setDescriptors +from Products.ZenUtils.Utils import getObjByPath, zenPath + +_zodb_config_template = """\ +%import relstorage + + cache-size {zodb-cachesize} + + + cache-local-mb {zodb-max-cache-mb} + cache-local-object-max {zodb-cache-max-object-size} + keep-history false + + host {zodb-host} + port {zodb-port} + user {zodb-user} + passwd {zodb-password} + db {zodb-db} + + + +""" + +_default_config_file = os.path.join(zenPath("etc"), "zodb.conf") + + +class _ZODBConnectionDefaults: + host = "localhost" + user = "zenoss" + password = "zenoss" + db = "zodb" + port = 3306 + cachesize = 1000 + cache_max_object_size = 1048576 + commit_lock_timeout = 30 + max_cache_mb = 512 + + +def add_zodb_arguments(parser): + """Add ZODB CLI arguments to `parser`.""" + group = parser.add_argument_group("ZODB Options") + group.add_argument( + "--zodb-config-file", + default=_default_config_file, + help="ZODB connection config file" + ) + group.add_argument( + "--zodb-cachesize", + default=_ZODBConnectionDefaults.cachesize, + type=int, + help="Maximum number of objects kept in the cache", + ) + group.add_argument( + "--zodb-host", + default=_ZODBConnectionDefaults.host, + help="Hostname of the MySQL server for ZODB", + ) + group.add_argument( + "--zodb-port", + type=int, + default=_ZODBConnectionDefaults.port, + help="Port of the MySQL server for ZODB", + ) + group.add_argument( + "--zodb-user", + default=_ZODBConnectionDefaults.user, + help="User of the MySQL server for ZODB", + ) + group.add_argument( + "--zodb-password", + default=_ZODBConnectionDefaults.password, + help="Password of the MySQL server for ZODB", + ) + group.add_argument( + "--zodb-db", + default=_ZODBConnectionDefaults.db, + help="Name of database for MySQL object store", + ) + group.add_argument( + "--zodb-cache-max-object-size", + default=_ZODBConnectionDefaults.cache_max_object_size, + type=int, + help="Maximum size of an object stored in the cache (bytes)", + ) + group.add_argument( + "--zodb-commit-lock-timeout", + default=_ZODBConnectionDefaults.commit_lock_timeout, + type=float, + help=( + "Specify the number of seconds a database connection will " + "wait to acquire a database 'commit' lock before failing." + ), + ) + group.add_argument( + "--zodb-max-cache-mb", + default=_ZODBConnectionDefaults.max_cache_mb, + type=int, + help="Maximum size of the cache (megabytes)" + ) + + +@contextlib.contextmanager +def zodb(config): + """ + Context manager managing the connection to ZODB. + + @type config: dict + """ + with contextlib.closing(getDB(config)) as db: + with contextlib.closing(db.open()) as session: + try: + with dataroot(session) as dmd: + yield (db, session, dmd) + finally: + transaction.abort() + + +def getDB(config): + """ + Returns a connection to the ZODB database. + + If specified, the 'zodb-config-file' key in `config` should name a + file containing the ZODB connection configuration in the ZConfig format. + + :param config: Contains configuration data for ZODB connection + :type config: dict + :rtype: :class:`ZODB.DB.DB` + """ + configfile = config.get("zodb-config-file") + if configfile and os.path.isfile(configfile): + url = "file://%s" % configfile + return ZODB.config.databaseFromURL(url) + zodb_config = _getConfigString(config) + return ZODB.config.databaseFromString(zodb_config) + + +@contextlib.contextmanager +def dataroot(session): + """ + Context manager returning the root Zenoss ZODB object from the session. + + The data root is commonly known as the "dmd" object. + + :param session: An active ZODB connection (session) object. + :type session: :class:`ZODB.Connection.Connection` + :rtype: :class:`Products.ZenModel.DataRoot.DataRoot` + """ + root = session.root() + application = _getContext(root["Application"]) + dataroot = getObjByPath(application, "/zport/dmd") + _ = _login(dataroot) + setDescriptors(dataroot) + try: + yield dataroot + finally: + noSecurityManager() + + +def _getConfigString(config): + """ + Returns a ZConfig string to connect to ZODB. + + :rtype: str + """ + return _zodb_config_template.format(**config) + + +def _getContext(app): + resp = HTTPResponse(stdout=None) + env = { + "SERVER_NAME": "localhost", + "SERVER_PORT": "8080", + "REQUEST_METHOD": "GET", + } + req = HTTPRequest(None, env, resp) + return app.__of__(RequestContainer(REQUEST=req)) + + +_default_user = "zenoss_system" + + +def _login(context, userid=_default_user): + """Authenticate user and configure credentials.""" + if userid is None: + userid = _default_user + + user = _getUser(context, userid) + newSecurityManager(None, user) + return user + + +def _getUser(context, userid): + root = context.getPhysicalRoot() + tool = getToolByName(root, "acl_users") + + user = tool.getUserById(userid) + if user is None: + # Try a different tool. + tool = getToolByName(root.zport, "acl_users") + user = tool.getUserById(userid) + + if user is None: + user = tool.getUserById(_default_user) + + if not hasattr(user, "aq_base"): + user = user.__of__(tool) + + return user + + +__all__ = ("zodb", "getDB", "dataroot") diff --git a/Products/ZenCollector/configcache/cache/__init__.py b/Products/ZenCollector/configcache/cache/__init__.py new file mode 100644 index 0000000000..733d6a1805 --- /dev/null +++ b/Products/ZenCollector/configcache/cache/__init__.py @@ -0,0 +1,24 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from .model import ( + ConfigKey, + ConfigQuery, + ConfigRecord, + ConfigStatus, +) + +__all__ = ( + "ConfigKey", + "ConfigQuery", + "ConfigRecord", + "ConfigStatus", +) diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py new file mode 100644 index 0000000000..36c2d7f98f --- /dev/null +++ b/Products/ZenCollector/configcache/cache/model.py @@ -0,0 +1,147 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import time + +import attr + +from attr.validators import instance_of + +from Products.ZenCollector.services.config import DeviceProxy + + +@attr.s(frozen=True, slots=True) +class ConfigQuery(object): + service = attr.ib(validator=[instance_of(str)], default="*") + monitor = attr.ib(validator=[instance_of(str)], default="*") + device = attr.ib(validator=[instance_of(str)], default="*") + + +@attr.s(frozen=True, slots=True) +class ConfigKey(object): + service = attr.ib(validator=[instance_of(str)]) + monitor = attr.ib(validator=[instance_of(str)]) + device = attr.ib(validator=[instance_of(str)]) + + +@attr.s(frozen=True, slots=True) +class ConfigPending(object): + key = attr.ib(validator=[instance_of(ConfigKey)]) + started = attr.ib(validator=[instance_of(float)]) + + @classmethod + def make(cls, svc, mon, dev, started=None): + if started is None: + started = time.time() + return cls(ConfigKey(svc, mon, dev), started) + + @classmethod + def from_key(cls, key, started=None): + if started is None: + started = time.time() + return cls(key, started) + + def astuple(self): + return attr.astuple(self, recurse=False) + + +@attr.s(slots=True) +class ConfigRecord(object): + key = attr.ib( + validator=[instance_of(ConfigKey)], on_setattr=attr.setters.NO_OP + ) + uid = attr.ib(validator=[instance_of(str)], on_setattr=attr.setters.NO_OP) + updated = attr.ib(validator=[instance_of(float)]) + config = attr.ib(validator=[instance_of(DeviceProxy)]) + + @classmethod + def make(cls, svc, mon, dev, uid, updated, config): + return cls(ConfigKey(svc, mon, dev), uid, updated, config) + + @property + def service(self): + return self.key.service + + @property + def monitor(self): + return self.key.monitor + + @property + def device(self): + return self.key.device + + +class _ConfigStatus(object): + """ + Namespace class for Current, Building, Expired, and Pending types. + """ + + class Current(object): + """The configuration is current.""" + + def __init__(self, ts): + self.updated = ts + + def __eq__(self, other): + if not isinstance(other, _ConfigStatus.Current): + return NotImplemented + return self.updated == other.updated + + class Expired(object): + """The configuration has expired.""" + + def __eq__(self, other): + if not isinstance(other, _ConfigStatus.Expired): + return NotImplemented + return True + + class Pending(object): + """The configuration is waiting for a rebuild.""" + + def __init__(self, ts): + self.submitted = ts + + def __eq__(self, other): + if not isinstance(other, _ConfigStatus.Pending): + return NotImplemented + return self.submitted == other.submitted + + class Building(object): + """The configuration is rebuilding.""" + + def __init__(self, ts): + self.started = ts + + def __eq__(self, other): + if not isinstance(other, _ConfigStatus.Building): + return NotImplemented + return self.started == other.started + + def __contains__(self, value): + return isinstance( + value, + ( + _ConfigStatus.Building, + _ConfigStatus.Current, + _ConfigStatus.Expired, + _ConfigStatus.Pending, + ), + ) + + +ConfigStatus = _ConfigStatus() + +__all__ = ( + "ConfigKey", + "ConfigQuery", + "ConfigRecord", + "ConfigStatus", +) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py new file mode 100644 index 0000000000..1d594ee1d7 --- /dev/null +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -0,0 +1,721 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2019, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +# Key structure +# ============= +# modelchange:device:uid: +# modelchange:device:config::: +# modelchange:device:age:: [(, ), ...] +# modelchange:device:pending:: [(, ), ...] +# modelchange:device:building:: [(, ), ...] +# +# While "device" seems redundant, other values in this position could be +# "threshold" and "property". +# +# The "config" segment identifies a key storing a device configuration. +# The "age" segment identifies a key storing a set of device IDs sorted by +# a score that has an simple encoding. The score is segmented by value. +# Values above zero are timestamps of when the current configuration was +# stored in redis. A score of zero means the configuration is outdated +# and needs to be replaced with an updated version. A score less than zero +# is the negated timestamp of when a request was submitted for a device +# configuration build. +# +# names the configuration service class used to generate the +# configuration. +# names the monitor (collector) the device belongs to. +# is the ID of the device +# is the object path to the device in ZODB. +# + +from __future__ import absolute_import, print_function, division + +import ast +import logging +import itertools + +from functools import partial +from itertools import islice + +import attr + +from twisted.spread.jelly import jelly, unjelly +from zope.component.factory import Factory + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from .model import ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus + +_app = "configcache" +log = logging.getLogger("zen.modelchange.stores") + + +class ConfigStoreFactory(Factory): + """ + IFactory implementation for ConfigStore objects. + """ + + def __init__(self): + super(ConfigStoreFactory, self).__init__( + ConfigStore, "ConfigStore", "Configuration Cache Storage" + ) + + +class ConfigStore(object): + """ + A device config store for a single configuration class. + """ + + @classmethod + def make(cls): + """Create and return a ConfigStore object.""" + client = getRedisClient(url=getRedisUrl()) + return cls(client) + + def __init__(self, client): + """Initialize a ConfigStore instance.""" + self.__client = client + self.__uids = _DeviceUIDTable() + self.__config = _DeviceConfigTable() + self.__age = _ConfigMetadataTable("age") + self.__pending = _ConfigMetadataTable("pending") + self.__building = _ConfigMetadataTable("building") + self.__range = type( + "rangefuncs", + (object,), + { + "age": partial(_range, self.__client, self.__age), + "pending": partial(_range, self.__client, self.__pending), + "building": partial(_range, self.__client, self.__building), + }, + )() + + def search(self, query=ConfigQuery()): + """ + Returns the configuration keys matching the search criteria. + + @type query: ConfigQuery + @rtype: Iterator[ConfigKey] + @raises TypeError: Unsupported value given for a field + @raises AttributeError: Unknown field + """ + if not isinstance(query, ConfigQuery): + raise TypeError("'{!r} is not a ConfigQuery".format(query)) + return ( + ConfigKey(svc, mon, dvc) + for svc, mon, dvc in self.__config.scan( + self.__client, **attr.asdict(query) + ) + ) + + def add(self, record): + """ + @type record: ConfigRecord + """ + svc, mon, dvc, uid, updated, config = _from_record(record) + dead_key_parts = tuple( + (key.service, key.monitor, key.device) + for key in self.search(ConfigQuery(service=svc, device=dvc)) + if key.monitor != mon + ) + dead_keys = [self.__config.make_key(*dkp) for dkp in dead_key_parts] + watch_keys = dead_keys + [svc, mon, dvc] + add_uid = not self.__uids.exists(self.__client, dvc) + + def _add_impl(pipe): + pipe.multi() + # Remove configs for this device that exist with a different + # monitor. + # Note: configs produced by different configuration services + # may exist simultaneously. + for dkp in dead_key_parts: + self.__config.delete(pipe, *dkp) + self.__age.delete(pipe, *dkp) + self.__pending.delete(pipe, *dkp) + self.__building.delete(pipe, *dkp) + if add_uid: + self.__uids.set(pipe, dvc, uid) + self.__config.set(pipe, svc, mon, dvc, config) + self.__age.add(pipe, svc, mon, dvc, updated) + + self.__client.transaction(_add_impl, *watch_keys) + + def get_uid(self, device): + """ + Return the ZODB UID (path) for the given device. + + @type device: str + """ + return self.__uids.get(self.__client, device) + + def get(self, key, default=None): + """ + @type key: ConfigKey + @rtype: ConfigRecord + """ + conf = self.__config.get( + self.__client, key.service, key.monitor, key.device + ) + if conf is None: + return default + score = self.__age.score( + self.__client, key.service, key.monitor, key.device + ) + score = 0 if score < 0 else score + uid = self.__uids.get(self.__client, key.device) + return _to_record( + key.service, key.monitor, key.device, uid, score, conf + ) + + def remove(self, *keys): + """ + Delete the configurations identified by `keys`. + + @type keys: Sequence[ConfigKey] + """ + with self.__client.pipeline() as pipe: + for key in keys: + svc, mon, dvc = key.service, key.monitor, key.device + self.__config.delete(pipe, svc, mon, dvc) + self.__age.delete(pipe, svc, mon, dvc) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.delete(pipe, svc, mon, dvc) + pipe.execute() + devices = [] + for dvc in set(k.device for k in keys): + configs = tuple(self.__config.scan(self.__client, device=dvc)) + if not configs: + devices.append(dvc) + if devices: + self.__uids.delete(self.__client, *devices) + + def set_expired(self, *keys): + """ + Marks the indicated configuration as expired. + + Attempts to mark pending configurations as expired are ignored. + + @type keys: Sequence[ConfigKey] + """ + if len(keys) == 0: + return + + watch_keys = self._get_watch_keys(*keys) + scores = ( + ( + key, + self.__age.score( + self.__client, key.service, key.monitor, key.device + ), + ) + for key in keys + ) + targets = tuple( + (key.service, key.monitor, key.device) + for key, score in scores + if score != 0.0 + ) + + def _impl(pipe): + pipe.multi() + for svc, mon, dvc in targets: + self.__age.add(pipe, svc, mon, dvc, 0) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.delete(pipe, svc, mon, dvc) + + self.__client.transaction(_impl, *watch_keys) + + def set_pending(self, *pending): + """ + Marks an expired configuration as waiting for a new configuration. + + @type pending: Sequence[(ConfigKey, float)] + """ + if len(pending) == 0: + return + + watch_keys = self._get_watch_keys(*(key for key, _ in pending)) + targets = self._get_targets(lambda x: x == 0.0, *pending) + + def _impl(pipe): + pipe.multi() + for svc, mon, dvc, ts in targets: + score = _to_score(ts) + self.__age.add(pipe, svc, mon, dvc, -1) + self.__building.delete(pipe, svc, mon, dvc) + self.__pending.add(pipe, svc, mon, dvc, score) + + self.__client.transaction(_impl, *watch_keys) + + def set_building(self, *building): + """ + Marks a pending configuration as building a new configuration. + + @type pairs: Sequence[(ConfigKey, float)] + """ + if len(building) == 0: + return + + watch_keys = self._get_watch_keys(*(key for key, _ in building)) + targets = self._get_targets(lambda x: x <= 0.0, *building) + + def _impl(pipe): + pipe.multi() + for svc, mon, dvc, ts in targets: + score = _to_score(ts) + self.__age.add(pipe, svc, mon, dvc, -1) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.add(pipe, svc, mon, dvc, score) + + self.__client.transaction(_impl, *watch_keys) + + def _get_targets(self, predicate, *pairs): + scores = ( + ( + key, + started, + self.__age.score( + self.__client, key.service, key.monitor, key.device + ), + ) + for key, started in pairs + ) + return tuple( + (key.service, key.monitor, key.device, started) + for key, started, score in scores + if predicate(score) + ) + + def get_status(self, *keys): + """ + Returns an interable of (ConfigKey, ConfigStatus) tuples. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus]] + """ + scores = ( + ( + key, + self.__age.score( + self.__client, key.service, key.monitor, key.device + ), + ) + for key in keys + ) + return iter(self._iter_status(scores)) + + def _iter_status(self, scores): + for key, score in scores: + if score > 0: + yield (key, ConfigStatus.Current(_to_ts(score))) + elif score == 0: + yield (key, ConfigStatus.Expired()) + else: + pscore = self.__pending.score( + self.__client, key.service, key.monitor, key.device + ) + if pscore is not None: + yield (key, ConfigStatus.Pending(_to_ts(pscore))) + bscore = self.__building.score( + self.__client, key.service, key.monitor, key.device + ) + if bscore is not None: + yield (key, ConfigStatus.Building(_to_ts(bscore))) + + def get_pending(self, service="*", monitor="*"): + """ + Return an iterator producing (ConfigKey, ConfigStatus.Pending) tuples. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Pending]] + """ + return ( + (key, ConfigStatus.Pending(ts)) + for key, ts in self.__range.pending(service, monitor) + ) + + def get_building(self, service="*", monitor="*"): + """ + Return an iterator producing (ConfigKey, ConfigStatus.Building) tuples. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Building]] + """ + return ( + (key, ConfigStatus.Building(ts)) + for key, ts in self.__range.building(service, monitor) + ) + + def get_expired(self, service="*", monitor="*"): + """ + Return an iterator producing (ConfigKey, ConfigStatus.Expired) tuples. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Expired]] + """ + return ( + (key, ConfigStatus.Expired()) + for key, _ in self.__range.age( + service, monitor, minv=0.0, maxv=0.0 + ) + ) + + def get_older(self, maxtimestamp, service="*", monitor="*"): + """ + Returns an iterator producing (ConfigKey, ConfigStatus.Current) + tuples where current timestamp <= `maxtimestamp`. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Current]] + """ + # NOTE: 'older' means timestamps > 0 and <= `maxtimestamp`. + return ( + (key, ConfigStatus.Current(ts)) + for key, ts in self.__range.age( + service, monitor, minv="(0", maxv=_to_score(maxtimestamp) + ) + ) + + def get_newer(self, mintimestamp, service="*", monitor="*"): + """ + Returns an iterator producing (ConfigKey, ConfigStatus.Current) + tuples where current timestamp > `mintimestamp`. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Current]] + """ + # NOTE: 'newer' means timestamps to `maxtimestamp`. + return ( + (key, ConfigStatus.Current(ts)) + for key, ts in self.__range.age( + service, monitor, minv="(%s" % (_to_score(mintimestamp),) + ) + ) + + def _get_watch_keys(self, *keys): + return set( + itertools.chain.from_iterable( + ( + self.__age.make_key(key.service, key.monitor), + self.__pending.make_key(key.service, key.monitor), + self.__building.make_key(key.service, key.monitor), + ) + for key in keys + ) + ) + + +def _range(client, metadata, svc, mon, minv=None, maxv=None): + pairs = metadata.get_pairs(client, svc, mon) + return ( + (ConfigKey(svcId, monId, devId), _to_ts(score)) + for svcId, monId, devId, score in metadata.range( + client, pairs, minscore=minv, maxscore=maxv + ) + ) + + +def _unjelly(data): + return unjelly(ast.literal_eval(data)) + + +def _to_score(ts): + return ts * 1000.0 + + +def _to_ts(score): + return score / 1000.0 + + +def _to_record(svc, mon, dvc, uid, updated, config): + key = ConfigKey(svc, mon, dvc) + updated = _to_ts(updated) + config = _unjelly(config) + return ConfigRecord(key, uid, updated, config) + + +def _from_record(record): + return ( + record.service, + record.monitor, + record.device, + record.uid, + _to_score(record.updated), + jelly(record.config), + ) + + +class _DeviceUIDTable(object): + """ + Manages mapping device names to their ZODB UID. + """ + + def __init__(self, scan_page_size=1000, mget_page_size=10): + """Initialize a _DeviceUIDTable instance.""" + self.__template = "{app}:device:uid:{{device}}".format(app=_app) + self.__scan_count = scan_page_size + self.__mget_count = mget_page_size + + def make_key(self, device): + return self.__template.format(device=device) + + def exists(self, client, device): + """Return True if configuration data exists for the given ID. + + :param device: The ID of the device + :type device: str + :rtype: boolean + """ + return client.exists(self.make_key(device)) + + def scan(self, client, device="*"): + """ + Return an iterable of tuples of device names. + """ + pattern = self.make_key(device) + result = client.scan_iter(match=pattern, count=self.__scan_count) + return (key.rsplit(":", 1)[-1] for key in result) + + def get(self, client, device): + """Return the UID of the given device name. + + :type device: str + :rtype: str + """ + key = self.make_key(device) + return client.get(key) + + def set(self, client, device, uid): + """Insert or replace the UID for the given device. + + :param device: The ID of the configuration + :type device: str + :param uid: The ZODB UID of the device + :type uid: str + :raises: ValueError + """ + key = self.make_key(device) + client.set(key, uid) + + def delete(self, client, *devices): + """Delete one or more keys. + + This method does not fail if the key doesn't exist. + + :type uids: Sequence[str] + """ + keys = tuple(self.make_key(dvc) for dvc in devices) + client.delete(*keys) + + +class _DeviceConfigTable(object): + """ + Manages device configuration data for a specific configuration service. + """ + + def __init__(self, scan_page_size=1000, mget_page_size=10): + """Initialize a _DeviceConfigTable instance.""" + self.__template = ( + "{app}:device:config:{{service}}:{{monitor}}:{{device}}".format( + app=_app + ) + ) + self.__scan_count = scan_page_size + self.__mget_count = mget_page_size + + def make_key(self, service, monitor, device): + return self.__template.format( + service=service, monitor=monitor, device=device + ) + + def exists(self, client, service, monitor, device): + """Return True if configuration data exists for the given ID. + + :param service: Name of the configuration service. + :type service: str + :param monitor: Name of the monitor the device is a member of. + :type monitor: str + :param device: The ID of the device + :type device: str + :rtype: boolean + """ + return client.exists(self.make_key(service, monitor, device)) + + def scan(self, client, service="*", monitor="*", device="*"): + """ + Return an iterable of tuples of (service, monitor, device). + """ + pattern = self.make_key(service, monitor, device) + result = client.scan_iter(match=pattern, count=self.__scan_count) + return (tuple(key.rsplit(":", 3)[1:]) for key in result) + + def get(self, client, service, monitor, device): + """Return the config data for the given config ID. + + If the config ID is not found, the default argument is returned. + + :type service: str + :type monitor: str + :type device: str + :rtype: Union[IJellyable, None] + """ + key = self.make_key(service, monitor, device) + return client.get(key) + + def set(self, client, service, monitor, device, data): + """Insert or replace the config data for the given config ID. + + If existing data for the device exists under a different monitor, + it will be deleted. + + :param service: The name of the configuration service. + :type service: str + :param monitor: The ID of the performance monitor + :type monitor: str + :param device: The ID of the configuration + :type device: str + :param data: The serialized configuration data + :type data: str + :raises: ValueError + """ + key = self.make_key(service, monitor, device) + client.set(key, data) + + def delete(self, client, service, monitor, device): + """Delete a key. + + This method does not fail if the key doesn't exist. + + :type service: str + :type monitor: str + :type device: str + """ + key = self.make_key(service, monitor, device) + client.delete(key) + + +class _ConfigMetadataTable(object): + """ + Manages the mapping of device configurations to monitors. + + Configuration IDs are mapped to service ID/monitor ID pairs. + + A Service ID/monitor ID pair are used as a key to retrieve the + Configuration IDs mapped to the pair. + """ + + def __init__(self, category): + """Initialize a ConfigMetadataStore instance.""" + self.__template = ( + "{app}:device:{category}:{{service}}:{{monitor}}".format( + app=_app, category=category + ) + ) + self.__scan_count = 1000 + + def make_key(self, service, monitor): + return self.__template.format(service=service, monitor=monitor) + + def get_pairs(self, client, service="*", monitor="*"): + pattern = self.make_key(service, monitor) + return ( + key.rsplit(":", 2)[1:] + for key in client.scan_iter(match=pattern, count=self.__scan_count) + ) + + def scan(self, client, pairs): + """ + Return an iterable of tuples of (service, monitor, device, score). + + @type client: redis client + @type pairs: Iterable[Tuple[str, str]] + @rtype Iterator[Tuple[str, str, str, float]] + """ + return ( + (service, monitor, dvc, score) + for service, monitor in pairs + for dvc, score in client.zscan_iter( + self.make_key(service, monitor), count=self.__scan_count + ) + ) + + def range(self, client, pairs, maxscore=None, minscore=None): + """ + Return an iterable of tuples of (service, monitor, device, score). + + @type client: redis client + @type pairs: Iterable[Tuple[str, str]] + @type minscore: Union[float, None] + @type maxscore: Union[float, None] + @rtype Iterator[Tuple[str, str, str, float]] + """ + maxv = maxscore if maxscore is not None else "+inf" + minv = minscore if minscore is not None else "-inf" + return ( + (service, monitor, device, score) + for service, monitor in pairs + for device, score in client.zrangebyscore( + self.make_key(service, monitor), minv, maxv, withscores=True + ) + ) + + def exists(self, client, service, monitor, device): + """Return True if a score for the key and device exists. + + @type client: RedisClient + @type service: str + @type monitor: str + @type device: str + """ + key = self.make_key(service, monitor) + return client.zscore(key, device) is not None + + def add(self, client, service, monitor, device, score): + """ + Add a (device, score) -> (monitor, serviceid) mapping. + This method will replace any existing mapping for device. + + @type client: RedisClient + @type service: str + @type monitor: str + @type device: str + @type score: float + """ + key = self.make_key(service, monitor) + client.zadd(key, score, device) + + def score(self, client, service, monitor, device): + """ + Returns the timestamp associated with the device ID. + Returns None of the device ID is not found. + """ + key = self.make_key(service, monitor) + return client.zscore(key, device) + + def delete(self, client, service, monitor, device): + """ + Removes a device from a (service, monitor) key. + """ + key = self.make_key(service, monitor) + client.zrem(key, device) + + +def _batched(iterable, n): + """ + Batch data into tuples of length `n`. The last batch may be shorter. + + >>> list(batched('ABCDEFG', 3)) + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] + """ + if n < 1: + raise ValueError("n must be greater than zero") + itr = iter(iterable) + while True: + batch = tuple(islice(itr, n)) + if not batch: + break + yield batch + # + # Note: In Python 3.7+, the above loop would be written as + # while (batch := tuple(islice(itr, n))): + # yield batch diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py new file mode 100644 index 0000000000..3297ce56d9 --- /dev/null +++ b/Products/ZenCollector/configcache/cli.py @@ -0,0 +1,284 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import argparse + +from datetime import datetime + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from .app import initialize_environment +from .cache import ConfigQuery, ConfigStatus +from .misc.args import get_subparser + + +class List_(object): + + description = "List configurations" + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, + "list", + List_.description, + parent=_get_common_parser(), + ) + subp.add_argument( + "-u", + dest="show_uid", + default=False, + action="store_true", + help="Display ZODB path for device", + ) + subp.add_argument( + "-f", + dest="states", + action=MultiChoice, + choices=("current", "expired", "pending", "building"), + default=argparse.SUPPRESS, + help="Only list configurations having these states. One or " + "more states may be specified, separated by commas.", + ) + subp.set_defaults(factory=List_) + + def __init__(self, args): + self._monitor = args.monitor + self._service = args.service + self._showuid = args.show_uid + state_names = getattr(args, "states", ()) + if state_names: + states = set() + for name in state_names: + states.add(_name_state_lookup[name]) + self._states = tuple(states) + else: + self._states = () + + def run(self): + initialize_environment() + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + query = ConfigQuery(service=self._service, monitor=self._monitor) + results = store.get_status(*store.search(query)) + if self._states: + results = ( + (key, status) + for key, status in results + if isinstance(status, self._states) + ) + rows = [] + maxd, maxs, maxm = 0, 0, 0 + for key, status in sorted( + results, key=lambda x: (x[0].device, x[0].service) + ): + if self._showuid: + uid = store.get_uid(key.device) + else: + uid = key.device + status_text = _format_status(status) + maxd = max(maxd, len(uid)) + maxs = max(maxs, len(status_text)) + maxm = max(maxm, len(key.monitor)) + rows.append((uid, status_text, key.monitor, key.service)) + if rows: + print( + "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( + "DEVICE", + "STATUS", + "MONITOR", + "SERVICE", + maxd=maxd, + maxs=maxs, + maxm=maxm, + ) + ) + for row in rows: + print( + "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( + row[0], + row[1], + row[2], + row[3], + maxd=maxd, + maxs=maxs, + maxm=maxm, + ) + ) + + +_name_state_lookup = { + "current": ConfigStatus.Current, + "expired": ConfigStatus.Expired, + "pending": ConfigStatus.Pending, + "building": ConfigStatus.Building, +} + + +def _format_status(status): + if isinstance(status, ConfigStatus.Current): + return "last updated {}".format(_format_date(status.updated)) + elif isinstance(status, ConfigStatus.Expired): + return "expired" + elif isinstance(status, ConfigStatus.Pending): + return "build request submitted {}".format( + _format_date(status.submitted) + ) + elif isinstance(status, ConfigStatus.Building): + return "building started {}".format(_format_date(status.started)) + else: + return "????" + + +def _format_date(ts): + when = datetime.fromtimestamp(ts) + return when.strftime("%Y-%m-%d %H:%M:%S") + + +class Show(object): + + description = "Show a configuration (JSON)" + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser(subparsers, "show", "Show a configuration (JSON)") + subp.add_argument( + "service", nargs=1, help="name of the configuration service" + ) + subp.add_argument( + "monitor", nargs=1, help="name of the performance monitor" + ) + subp.add_argument("device", nargs=1, help="name of the device") + subp.set_defaults(factory=Show) + + def __init__(self, args): + pass + + def run(self): + pass + + +class Expire(object): + + description = "" + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, + "expire", + "Mark configurations as expired", + parent=_get_common_parser(), + ) + subp.set_defaults(factory=Expire) + + def __init__(self, args): + pass + + def run(self): + pass + + +# list - list configs; +# ls [-m monitor] [-s service] [-u] [-f state] [device] +# where 'monitor', 'service' and 'device' can be globs. +# Output should look like: +# [device] [state] [monitor] [service] +# if '-u' is given, then +# [device-path] [state] [monitor] [service] +# where 'state' is: +# Current HH:MM:SS - current with time remaining +# Expired - expired configuration +# Pending HH:MM:SS - pending with time remaining +# and 'device-path' is the dmd path (UID) +# +# show - show config in JSON format; +# cat [service] [monitor] [device] +# No wildcard support. +# +# expire - Mark one or more configurations expired; +# expire [-m monitor] [-s service] [device] + + +class MultiChoice(argparse.Action): + """Allow multiple values for a choice option.""" + + def __init__(self, option_strings, dest, **kwargs): + kwargs["type"] = self._split_listed_choices + super(MultiChoice, self).__init__(option_strings, dest, **kwargs) + + @property + def choices(self): + return self._choices_checker + + @choices.setter + def choices(self, values): + self._choices_checker = _ChoicesChecker(values) + + def _split_listed_choices(self, value): + if "," in value: + return tuple(value.split(",")) + return value + + def __call__(self, parser, namespace, values=None, option_string=None): + if isinstance(values, basestring): + values = (values,) + setattr(namespace, self.dest, values) + + +class _ChoicesChecker(object): + def __init__(self, values): + self._choices = values + + def __contains__(self, value): + if isinstance(value, (list, tuple)): + return all(v in self._choices for v in value) + else: + return value in self._choices + + def __iter__(self): + return iter(self._choices) + + +_common_parser = None + + +def _get_common_parser(): + global _common_parser + if _common_parser is None: + _common_parser = argparse.ArgumentParser(add_help=False) + _common_parser.add_argument( + "-m", + "--monitor", + type=str, + default="*", + help="Name of the performance monitor. Supports simple '*' " + "wildcard comparisons. A lone '*' selects all monitors.", + ) + _common_parser.add_argument( + "-s", + "--service", + type=str, + default="*", + help="Name of the configuration service. Supports simple '*' " + "wildcard comparisons. A lone '*' selects all services.", + ) + _common_parser.add_argument( + "device", + nargs="*", + default=argparse.SUPPRESS, + help="Name of the device. Multiple devices may be specified. " + "Supports simple '*' wildcard comparisons. Not specifying a " + "device will select all devices.", + ) + return _common_parser diff --git a/Products/ZenCollector/configcache/configcache.py b/Products/ZenCollector/configcache/configcache.py new file mode 100644 index 0000000000..53a061dffb --- /dev/null +++ b/Products/ZenCollector/configcache/configcache.py @@ -0,0 +1,32 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +from .cli import List_, Show, Expire +from .invalidator import Invalidator +from .manager import Manager +from .misc.args import get_arg_parser +from .version import Version + + +def main(argv=None): + parser = get_arg_parser("configcache commands") + + subparsers = parser.add_subparsers(title="Commands") + + Version.add_arguments(parser, subparsers) + Manager.add_arguments(parser, subparsers) + Invalidator.add_arguments(parser, subparsers) + List_.add_arguments(parser, subparsers) + Show.add_arguments(parser, subparsers) + Expire.add_arguments(parser, subparsers) + + args = parser.parse_args() + args.factory(args).run() diff --git a/Products/ZenCollector/configcache/configure.zcml b/Products/ZenCollector/configcache/configure.zcml new file mode 100644 index 0000000000..1f249e5ad0 --- /dev/null +++ b/Products/ZenCollector/configcache/configure.zcml @@ -0,0 +1,17 @@ + + + + + + + + + diff --git a/Products/ZenCollector/configcache/debug.py b/Products/ZenCollector/configcache/debug.py new file mode 100644 index 0000000000..b6b354f72f --- /dev/null +++ b/Products/ZenCollector/configcache/debug.py @@ -0,0 +1,45 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import os +import signal +import sys + +from .app import pidfile + + +class Debug(object): + @classmethod + def from_args(cls, args): + return cls(args.pidfile) + + def __init__(self, pidfile): + self.pidfile = pidfile + + def run(self): + pf = pidfile({"pidfile": self.pidfile}) + try: + pid = pf.read() + try: + os.kill(pid, signal.SIGUSR1) + print( + "Signaled {} to toggle debug mode".format( + self.pidfile.split(".")[0] + .split("/")[-1] + .replace("-", " ") + ) + ) + except OSError as ex: + print("{} ({})".format(ex, pid), file=sys.stderr) + sys.exit(1) + except IOError as ex: + print("{}".format(ex), file=sys.stderr) + sys.exit(1) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py new file mode 100644 index 0000000000..74417c3e24 --- /dev/null +++ b/Products/ZenCollector/configcache/invalidator.py @@ -0,0 +1,226 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import + +import logging + +from Products.AdvancedQuery import And, Eq +from zenoss.modelindex import constants +from zope.component import createObject + +import Products.ZenCollector.configcache as CONFIGCACHE_MODULE + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl +from Products.Zuul.catalog.interfaces import IModelCatalogTool + +from .app import Application +from .cache import ConfigQuery +from .debug import Debug as DebugCommand +from .misc.args import get_subparser +from .modelchange import InvalidationCause +from .utils import ( + BuildConfigTaskDispatcher, + Constants, + DevicePropertyMap, + getConfigServices, + RelStorageInvalidationPoller, +) + +_default_interval = 30.0 + + +class Invalidator(object): + + description = ( + "Analyzes changes in ZODB to determine whether to update " + "device configurations" + ) + + configs = ( + ("modelchange.zcml", CONFIGCACHE_MODULE), + ) + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, "invalidator", Invalidator.description + ) + subsubparsers = subp.add_subparsers(title="Invalidator Commands") + + subp_run = get_subparser( + subsubparsers, "run", "Run the invalidator service" + ) + Application.add_all_arguments(subp_run) + subp_run.add_argument( + "--poll-interval", + default=_default_interval, + type=float, + help="Invalidation polling interval (in seconds)", + ) + subp_run.set_defaults( + factory=Application.from_args, + parser=subp_run, + task=Invalidator, + ) + + subp_debug = get_subparser( + subsubparsers, + "debug", + "Signal the invalidator service to toggle debug logging", + ) + Application.add_pidfile_arguments(subp_debug) + subp_debug.set_defaults(factory=DebugCommand.from_args) + + Application.add_genconf_command(subsubparsers, (subp_run, subp_debug)) + + def __init__(self, config, context): + self.ctx = context + + configClasses = getConfigServices() + self.dispatcher = BuildConfigTaskDispatcher(configClasses) + + client = getRedisClient(url=getRedisUrl()) + self.store = createObject("configcache-store", client) + + self.interval = config["poll-interval"] + self.log = logging.getLogger("zen.configcache.invalidator") + + def run(self): + self._synchronize() + + poller = RelStorageInvalidationPoller( + self.ctx.db.storage, self.ctx.session, self.ctx.dmd + ) + self.log.info( + "polling for device changes every %s seconds", self.interval + ) + while not self.ctx.controller.shutdown: + for invalidation in poller.poll(): + try: + self._process(invalidation) + except AttributeError: + self.log.info( + "invalidation device=%s reason=%s", + invalidation.device, + invalidation.reason + ) + self.log.exception("failed while processing invalidation") + self.ctx.controller.wait(self.interval) + + def _synchronize(self): + tool = IModelCatalogTool(self.ctx.dmd) + # TODO: if device changed monitors, the config should be same (?) + # so just rekey the config? + count = _removeDeleted(self.log, tool, self.store) + if count == 0: + self.log.info("no dangling configurations found") + timelimitmap = DevicePropertyMap.from_organizer( + self.ctx.dmd.Devices, Constants.build_timeout_id + ) + new_devices = _addNew( + self.log, tool, timelimitmap, self.store, self.dispatcher + ) + if len(new_devices) == 0: + self.log.info("no missing configurations") + + def _process(self, invalidation): + device = invalidation.device + reason = invalidation.reason + monitor = device.getPerformanceServerName() + keys = list( + self.store.search(ConfigQuery(monitor=monitor, device=device.id)) + ) + if reason is InvalidationCause.Updated: + self.store.set_expired(*keys) + for key in keys: + self.log.info( + "expired configuration of changed device " + "device=%s monitor=%s service=%s device-oid=%r", + key.device, + key.monitor, + key.service, + invalidation.oid, + ) + elif reason is InvalidationCause.Deleted: + self.store.remove(*keys) + for key in keys: + self.log.info( + "removed configuration of deleted device " + "device=%s monitor=%s service=%s device-oid=%r", + key.device, + key.monitor, + key.service, + invalidation.oid, + ) + else: + self.log.warn( + "ignored unexpected reason " + "reason=%s device=%s monitor=%s device-oid=%r", + reason, + device, + monitor, + invalidation.oid, + ) + + +_solr_fields = ("id", "collector", "uid") + + +def _deviceExistsInCatalog(tool, monitorId, deviceId): + query = And(Eq("id", deviceId), Eq("collector", monitorId)) + brain = next( + iter(tool.search_model_catalog(query, fields=_solr_fields)), None + ) + return brain is not None + + +def _removeDeleted(log, tool, store): + # Remove deleted devices from the config and metadata store. + devices_not_found = tuple( + key + for key in store.search() + if not _deviceExistsInCatalog(tool, key.monitor, key.device) + ) + store.remove(*devices_not_found) + for key in devices_not_found: + log.info( + "removed configuration for deleted device " + "device=%s monitor=%s service=%s", + key.device, + key.monitor, + key.device, + ) + return len(devices_not_found) + + +def _addNew(log, tool, timelimitmap, store, dispatcher): + # Add new devices to the config and metadata store. + # Query the catalog for all devices + catalog_results = tool.cursor_search( + types=("Products.ZenModel.Device.Device",), + limit=constants.DEFAULT_SEARCH_LIMIT, + fields=_solr_fields, + ).results + new_devices = [] + for brain in catalog_results: + keys = tuple( + store.search(ConfigQuery(monitor=brain.collector, device=brain.id)) + ) + if not keys: + timeout = timelimitmap.get(brain.uid) + dispatcher.dispatch_all(brain.collector, brain.id, timeout) + log.info( + "submitted build jobs for device without any configurations " + "uid=%s monitor=%s", + brain.uid, + brain.collector, + ) + new_devices.append(brain.id) + return new_devices diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py new file mode 100644 index 0000000000..ac02738dc2 --- /dev/null +++ b/Products/ZenCollector/configcache/manager.py @@ -0,0 +1,193 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function + +import logging + +from datetime import datetime +from itertools import chain +from time import time + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from .app import Application +from .cache import ConfigStatus +from .debug import Debug as DebugCommand +from .misc.args import get_subparser +from .utils import ( + BuildConfigTaskDispatcher, + Constants, + DevicePropertyMap, + getConfigServices, +) + +_default_interval = 30.0 # seconds + + +class Manager(object): + + description = ( + "Determines whether device configs are old and regenerates them" + ) + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser(subparsers, "manager", Manager.description) + subsubparsers = subp.add_subparsers(title="Manager Commands") + + subp_run = get_subparser( + subsubparsers, "run", "Run the manager service" + ) + Application.add_all_arguments(subp_run) + subp_run.add_argument( + "--check-interval", + default=_default_interval, + type=float, + help="Config checking interval (in seconds)", + ) + subp_run.set_defaults( + factory=Application.from_args, + parser=subp_run, + task=Manager, + ) + + subp_debug = get_subparser( + subsubparsers, + "debug", + "Signal the manager service to toggle debug logging", + ) + Application.add_pidfile_arguments(subp_debug) + subp_debug.set_defaults(factory=DebugCommand.from_args) + + Application.add_genconf_command(subsubparsers, (subp_run, subp_debug)) + + def __init__(self, config, context): + self.ctx = context + configClasses = getConfigServices() + self.dispatcher = BuildConfigTaskDispatcher(configClasses) + client = getRedisClient(url=getRedisUrl()) + self.store = createObject("configcache-store", client) + self.interval = config["check-interval"] + self.log = logging.getLogger("zen.configcache.manager") + + def run(self): + self.log.info( + "checking for expired configurations and configuration build " + "timeouts every %s seconds", + self.interval, + ) + while not self.ctx.controller.shutdown: + try: + self.ctx.session.sync() + self._retry_pending_builds() + self._rebuild_older_configs() + except Exception as ex: + self.log.exception("unexpected error %s", ex) + self.ctx.controller.wait(self.interval) + + def _retry_pending_builds(self): + pendinglimitmap = DevicePropertyMap.from_organizer( + self.ctx.dmd.Devices, Constants.pending_timeout_id + ) + now = time() + count = 0 + for key, status in self.store.get_pending(): + uid = self.store.get_uid(key.device) + duration = pendinglimitmap.get(uid) + if status.submitted < (now - duration): + self.store.set_expired(key) + self.log.debug( + "pending configuration build has timed out " + "submitted=%s service=%s monitor=%s device=%s", + datetime.fromtimestamp(status.submitted).strftime( + "%Y-%M-%d %H:%m:%S" + ), + Constants.build_timeout_id, + duration, + key.service, + key.monitor, + key.device, + ) + count += 1 + if count == 0: + self.log.debug("no pending configuration builds have timed out") + + def _rebuild_older_configs(self): + buildlimitmap = DevicePropertyMap.from_organizer( + self.ctx.dmd.Devices, Constants.build_timeout_id + ) + agelimitmap = DevicePropertyMap.from_organizer( + self.ctx.dmd.Devices, Constants.time_to_live_id + ) + min_limit = agelimitmap.smallest_value() + self.log.debug( + "minimum age limit is %s", _formatted_interval(min_limit) + ) + now = time() + min_age = now - min_limit + results = chain.from_iterable( + (self.store.get_expired(), self.store.get_older(min_age)) + ) + count = 0 + for key, status in results: + uid = self.store.get_uid(key.device) + ttl_limit = agelimitmap.get(uid) + expiration_threshold = now - ttl_limit + if ( + isinstance(status, ConfigStatus.Expired) + or status.updated <= expiration_threshold + ): + timeout = buildlimitmap.get(uid) + self.store.set_pending((key, time())) + self.dispatcher.dispatch( + key.service, key.monitor, key.device, timeout + ) + if isinstance(status, ConfigStatus.Expired): + self.log.debug( + "submitted job to rebuild expired config " + "service=%s monitor=%s device=%s", + key.service, + key.monitor, + key.device, + ) + else: + self.log.debug( + "submitted job to rebuild old config " + "updated=%s %s=%s service=%s monitor=%s device=%s", + datetime.fromtimestamp(status.updated).strftime( + "%Y-%M-%d %H:%m:%S" + ), + Constants.time_to_live_id, + ttl_limit, + key.service, + key.monitor, + key.device, + ) + count += 1 + if count == 0: + self.log.debug("found no expired or old configurations to rebuild") + + +def _formatted_interval(total_seconds): + minutes, seconds = divmod(total_seconds, 60) + hours, minutes = divmod(minutes, 60) + days, hours = divmod(hours, 24) + text = "" + if seconds: + text = "{:02} seconds".format(seconds) + if minutes: + text = "{:02} minutes {}".format(minutes, text).strip() + if hours: + text = "{:02} hours {}".format(hours, text).strip() + if days: + text = "{} days {}".format(days, text).strip() + return text diff --git a/Products/ZenCollector/configcache/meta.zcml b/Products/ZenCollector/configcache/meta.zcml new file mode 100644 index 0000000000..681ee9d0b5 --- /dev/null +++ b/Products/ZenCollector/configcache/meta.zcml @@ -0,0 +1,14 @@ + + + + + diff --git a/Products/ZenCollector/configcache/misc/__init__.py b/Products/ZenCollector/configcache/misc/__init__.py new file mode 100644 index 0000000000..5fa6665ad9 --- /dev/null +++ b/Products/ZenCollector/configcache/misc/__init__.py @@ -0,0 +1,37 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import sys as _sys + +from functools import wraps as _wraps + + +def coroutine(func): + """Decorator for initializing a generator as a coroutine.""" + + @_wraps(func) + def start(*args, **kw): + coro = func(*args, **kw) + coro.next() + return coro + + return start + + +def into_tuple(args): + if isinstance(args, basestring): + return (args,) + elif not hasattr(args, "__iter__"): + return (args,) + return args + + +def app_name(): + fn = _sys.argv[0].rsplit("/", 1)[-1] + return fn.rsplit(".", 1)[0] if fn.endswith(".py") else fn diff --git a/Products/ZenCollector/configcache/misc/args.py b/Products/ZenCollector/configcache/misc/args.py new file mode 100644 index 0000000000..b16251c9ea --- /dev/null +++ b/Products/ZenCollector/configcache/misc/args.py @@ -0,0 +1,53 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import argparse as _argparse + +from Products.ZenUtils.terminal_size import ( + get_terminal_size as _get_terminal_size, +) + + +def get_arg_parser(description, epilog=None): + parser = _argparse.ArgumentParser( + description=description, + epilog=epilog, + formatter_class=ZenHelpFormatter, + ) + _fix_optional_args_title(parser) + return parser + + +def get_subparser(subparsers, title, description=None, parent=None): + subparser = subparsers.add_parser( + title, + description=description + ".", + help=description, + parents=[parent] if parent else [], + formatter_class=ZenHelpFormatter, + ) + _fix_optional_args_title(subparser, title.capitalize()) + return subparser + + +def _fix_optional_args_title(parser, title="General"): + for grp in parser._action_groups: + if grp.title == "optional arguments": + grp.title = "{} Options".format(title) + + +class ZenHelpFormatter(_argparse.ArgumentDefaultsHelpFormatter): + """ + Derive to set the COLUMNS environment variable when displaying help. + """ + + def __init__(self, *args, **kwargs): + size = _get_terminal_size() + kwargs["width"] = size.columns - 2 + super(ZenHelpFormatter, self).__init__(*args, **kwargs) diff --git a/Products/ZenCollector/configcache/modelchange.zcml b/Products/ZenCollector/configcache/modelchange.zcml new file mode 100644 index 0000000000..09c79fe93a --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange.zcml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Products/ZenCollector/configcache/modelchange/__init__.py b/Products/ZenCollector/configcache/modelchange/__init__.py new file mode 100644 index 0000000000..023c178973 --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/__init__.py @@ -0,0 +1,16 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from .invalidation import Invalidation, InvalidationCause +from .processor import InvalidationProcessor + + +__all__ = ("Invalidation", "InvalidationCause", "InvalidationProcessor") diff --git a/Products/ZenCollector/configcache/modelchange/filters.py b/Products/ZenCollector/configcache/modelchange/filters.py new file mode 100644 index 0000000000..0a7aec89aa --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/filters.py @@ -0,0 +1,222 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2011, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import logging +import re + +from cStringIO import StringIO +from hashlib import md5 + +from zope.interface import implementer + +from Products.ZenHub.interfaces import ( + IInvalidationFilter, + FILTER_EXCLUDE, + FILTER_CONTINUE, +) +from Products.ZenModel.DeviceClass import DeviceClass +from Products.ZenModel.GraphDefinition import GraphDefinition +from Products.ZenModel.GraphPoint import GraphPoint +from Products.ZenModel.IpAddress import IpAddress +from Products.ZenModel.IpNetwork import IpNetwork +from Products.ZenModel.OSProcessClass import OSProcessClass +from Products.ZenModel.OSProcessOrganizer import OSProcessOrganizer +from Products.ZenModel.ProductClass import ProductClass +from Products.ZenModel.Software import Software +from Products.ZenWidgets.Portlet import Portlet +from Products.Zuul.catalog.interfaces import IModelCatalogTool + +from ..utils import Constants + +log = logging.getLogger("zen.{}".format(__name__.split(".")[-1].lower())) + + +@implementer(IInvalidationFilter) +class IgnorableClassesFilter(object): + """Ignore invalidations on certain classes.""" + + CLASSES_TO_IGNORE = ( + IpAddress, + IpNetwork, + GraphDefinition, + GraphPoint, + Portlet, + ProductClass, + Software, + ) + + def initialize(self, context): + pass + + def include(self, obj): + if isinstance(obj, self.CLASSES_TO_IGNORE): + log.debug("IgnorableClassesFilter is ignoring %s ", obj) + return FILTER_EXCLUDE + return FILTER_CONTINUE + + +_iszorcustprop = re.compile("[zc][A-Z]").match + +_excluded_properties = ( + Constants.build_timeout_id, + Constants.pending_timeout_id, + Constants.time_to_live_id, +) + + +def _include_property(propId): + if propId in _excluded_properties: + return None + return _iszorcustprop(propId) + + +def _getZorCProperties(organizer): + for zId in sorted(organizer.zenPropertyIds(pfilt=_include_property)): + try: + if organizer.zenPropIsPassword(zId): + propertyString = organizer.getProperty(zId, "") + else: + propertyString = organizer.zenPropertyString(zId) + yield zId, propertyString + except AttributeError: + # ZEN-3666: If an attribute error is raised on a zProperty + # assume it was produced by a zenpack + # install whose daemons haven't been restarted and continue + # excluding the offending property. + log.debug("Excluding '%s' property", zId) + + +@implementer(IInvalidationFilter) +class BaseOrganizerFilter(object): + """ + Base invalidation filter for organizers. + + The default implementation will reject organizers that do not have + updated calculated checksum values. The checksum is calculated using + accumulation of each 'z' and 'c' property associated with organizer. + """ + + weight = 10 + + def __init__(self, types): + self._types = types + + def getRoot(self, context): + return context.dmd.primaryAq() + + def initialize(self, context): + root = self.getRoot(context) + brains = IModelCatalogTool(root).search(self._types) + results = {} + for brain in brains: + try: + obj = brain.getObject() + results[brain.getPath()] = self.organizerChecksum(obj) + except KeyError: + log.warn("Unable to retrieve object: %s", brain.getPath()) + self.checksum_map = results + + def organizerChecksum(self, organizer): + m = md5() + self.generateChecksum(organizer, m) + return m.hexdigest() + + def generateChecksum(self, organizer, md5_checksum): + # Checksum all zProperties and custom properties + for zId, propertyString in _getZorCProperties(organizer): + md5_checksum.update("%s|%s" % (zId, propertyString)) + + def include(self, obj): + # Move on if it's not one of our types + if not isinstance(obj, self._types): + return FILTER_CONTINUE + + # Checksum the device class + current_checksum = self.organizerChecksum(obj) + organizer_path = "/".join(obj.getPrimaryPath()) + + # Get what we have right now and compare + existing_checksum = self.checksum_map.get(organizer_path) + if current_checksum != existing_checksum: + log.debug("%r has a new checksum! Including.", obj) + self.checksum_map[organizer_path] = current_checksum + return FILTER_CONTINUE + log.debug("%r checksum unchanged. Skipping.", obj) + return FILTER_EXCLUDE + + +class DeviceClassInvalidationFilter(BaseOrganizerFilter): + """ + Invalidation filter for DeviceClass organizers. + + Uses both 'z' and 'c' properties as well as locally bound RRD templates + to create the checksum. + """ + + def __init__(self): + super(DeviceClassInvalidationFilter, self).__init__((DeviceClass,)) + + def getRoot(self, context): + return context.dmd.Devices.primaryAq() + + def generateChecksum(self, organizer, md5_checksum): + """ + Generate a checksum representing the state of the device class as it + pertains to configuration. This takes into account templates and + zProperties, nothing more. + """ + s = StringIO() + # Checksum includes all bound templates + for tpl in organizer.rrdTemplates(): + s.seek(0) + s.truncate() + # TODO: exportXml is a bit of a hack. Sorted, etc. would be better. + tpl.exportXml(s) + md5_checksum.update(s.getvalue()) + # Include z/c properties from base class + super(DeviceClassInvalidationFilter, self).generateChecksum( + organizer, md5_checksum + ) + + +class OSProcessOrganizerFilter(BaseOrganizerFilter): + """Invalidation filter for OSProcessOrganizer objects.""" + + def __init__(self): + super(OSProcessOrganizerFilter, self).__init__((OSProcessOrganizer,)) + + def getRoot(self, context): + return context.dmd.Processes.primaryAq() + + +class OSProcessClassFilter(BaseOrganizerFilter): + """ + Invalidation filter for OSProcessClass objects. + + This filter uses 'z' and 'c' properties as well as local _properties + defined on the organizer to create a checksum. + """ + + def __init__(self): + super(OSProcessClassFilter, self).__init__((OSProcessClass,)) + + def getRoot(self, context): + return context.dmd.Processes.primaryAq() + + def generateChecksum(self, organizer, md5_checksum): + # Include properties of OSProcessClass + for prop in organizer._properties: + prop_id = prop["id"] + md5_checksum.update( + "%s|%s" % (prop_id, getattr(organizer, prop_id, "")) + ) + # Include z/c properties from base class + super(OSProcessClassFilter, self).generateChecksum( + organizer, md5_checksum + ) diff --git a/Products/ZenCollector/configcache/modelchange/invalidation.py b/Products/ZenCollector/configcache/modelchange/invalidation.py new file mode 100644 index 0000000000..57863027db --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/invalidation.py @@ -0,0 +1,52 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging + +from enum import IntEnum + +log = logging.getLogger("zen.{}".format(__name__.split(".")[-1].lower())) + + +class InvalidationCause(IntEnum): + """An enumeration of reasons for the invalidation.""" + + Removed = 1 + Updated = 2 + + +class Invalidation(object): + """Contains the OID and the device referenced by the OID.""" + + __slots__ = ("oid", "device", "reason") + + def __init__(self, oid, device, reason): + """ + Initialize an Invalidation instance. + + :param oid: The object ID of the invalidated device. + :type oid: zodbpickle.binary + :param device: The invalidated device + :type device: PrimaryPathObjectManager | DeviceComponent + :param reason: The reason why the device was invalidated. + :type reason: InvalidationCause + """ + self.oid = oid + self.device = device + self.reason = reason + + def __repr__(self): + return "<%r: oid=%r device=%r reason=%r>" % ( + self.__class__, + self.oid, + self.device, + self.reason, + ) diff --git a/Products/ZenCollector/configcache/modelchange/oids.py b/Products/ZenCollector/configcache/modelchange/oids.py new file mode 100644 index 0000000000..cc96433ac3 --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/oids.py @@ -0,0 +1,109 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging + +from zenoss.modelindex.model_index import CursorSearchParams +from zExceptions import NotFound +from zope.interface import implementer + +from Products.ZenHub.interfaces import IInvalidationOid +from Products.Zuul.catalog.interfaces import IModelCatalogTool + +log = logging.getLogger("zen.configcache.modelchange") + + +class BaseTransform(object): + def __init__(self, entity): + self._entity = entity + + +@implementer(IInvalidationOid) +class DefaultOidTransform(BaseTransform): + """Default transformation returns the OID that was given.""" + + def transformOid(self, oid): + return oid + + +@implementer(IInvalidationOid) +class DeviceOidTransform(BaseTransform): + """ + If the object has a relationship with a device, return the device's OID. + """ + + def transformOid(self, oid): + # get device oid + result = oid + device = getattr(self._entity, "device", lambda: None)() + if device: + result = device._p_oid + log.debug( + "oid for %s transformed to device oid for %s", + self._entity, + device, + ) + return result + + +@implementer(IInvalidationOid) +class DataPointToDevice(BaseTransform): + """Return the device OIDs associated with an RRDDataPoint.""" + + def transformOid(self, oid): + ds = self._entity.datasource().primaryAq() + template = ds.rrdTemplate().primaryAq() + dc = template.deviceClass().primaryAq() + return _getDevicesFromDeviceClass(dc) + + +@implementer(IInvalidationOid) +class DataSourceToDevice(BaseTransform): + """Return the device OIDs associated with an RRDDataSource.""" + + def transformOid(self, oid): + template = self._entity.rrdTemplate().primaryAq() + dc = template.deviceClass().primaryAq() + return _getDevicesFromDeviceClass(dc) + + +@implementer(IInvalidationOid) +class TemplateToDevice(BaseTransform): + """Return the device OIDs associated with an RRDTemplate.""" + + def transformOid(self, oid): + dc = self._entity.deviceClass().primaryAq() + return _getDevicesFromDeviceClass(dc) + + +@implementer(IInvalidationOid) +class DeviceClassToDevice(BaseTransform): + """Return the device OIDs in the DeviceClass hierarchy.""" + + def transformOid(self, oid): + return _getDevicesFromDeviceClass(self._entity) + + +def _getDevicesFromDeviceClass(dc): + tool = IModelCatalogTool(dc.dmd.Devices) + query, _ = tool._build_query( + types=("Products.ZenModel.Device.Device",), + paths=("{}*".format("/".join(dc.getPhysicalPath())),), + ) + params = CursorSearchParams(query) + result = tool.model_catalog_client.cursor_search(params, dc.dmd) + for brain in result.results: + try: + ob = brain.getObject() + if ob: + yield ob._p_oid + except (NotFound, KeyError, AttributeError): + pass diff --git a/Products/ZenCollector/configcache/modelchange/pipeline.py b/Products/ZenCollector/configcache/modelchange/pipeline.py new file mode 100644 index 0000000000..a1f4d6387f --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/pipeline.py @@ -0,0 +1,126 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging + +from ..misc import coroutine, into_tuple + +log = logging.getLogger("zen.configcache.modelchange.pipeline") + + +class Pipe(object): + """ + Abstract base class for the pipes in a pipeline. + + A pipeline is a sequence of generators (coroutines) where each generator + returns data that is forward to the next generator. The `node` method + creates the generator. + + Pipelines are push-style, meaning that the generators do not run until + data is sent into the pipeline. Calling `send` on the generator returned + by `node` pushes data into the pipeline. The `send` function blocks until + the pipeline has finished. + + A Pipe may have one or more outputs or no outputs. Each output includes + an ID to identify which Pipe the output is forwarded to. If there's only + one output, no ID is required and the default ID is used. + + Use the `connect` method to connect one Pipe to another. + + Returning None from `run` (an Action object) stops the pipeline. When + stopped in this way, the pipeline is ready for the next input. + + :param targets: References to the next nodes in the pipeline. + :type targets: Dict[Any, GeneratorType] + :param run: References the callable that's invoked when data is applied. + :type run: Action + """ + + def __init__(self, action): + """Initialize a Pipe instance. + + :param action: Called to process the data passed to this node. + :type action: callable + """ + self.targets = {} + self.run = action + + @coroutine + def node(self): + """Returns the node that forms the pipeline.""" + while True: + args = yield + self.apply(args) + + def apply(self, args): + """Applies the arguments to the action.""" + args = into_tuple(args) + results = self.run(*args) + if results is None: + return + results = into_tuple(results) + if len(results) == 1: + tid, output = self.run.DEFAULT, results[0] + else: + tid, output = results[0], results[1] + if tid not in self.targets: + log.warn("no such target ID: %s", tid) + return + self.targets[tid].send(output) + + def connect(self, target, tid=None): + """ + Connects a Pipe to a specific output. + + If this node will have only one output, a default ID can be used. + + :param target: The pipeline node to receive the output. + :type target: Pipe + :param tid: The ID of the output. + :type tid: int + """ + tid = tid if tid is not None else self.run.DEFAULT + self.targets[tid] = target.node() + + +class IterablePipe(Pipe): + """ + A variation of the Pipe that iterates over the input passing + each item to `run` rather than passing all the data at once. + + If a `None` is returned by `run`, rather than stopping, an IterablePipe + continues on to the next item in the iterable. + """ + + def apply(self, args): + iterable = into_tuple(args) + for item in iterable: + super(IterablePipe, self).apply(item) + + +class Action(object): + """Base class for action objects passed to Pipes.""" + + DEFAULT = object() + """Default target ID.""" + + def __call__(self, *data): + """ + Processes the given data and returns data that is forwarded to the + next node in the pipeline. + + If the returned data is intended for a specific target, return a + two-element tuple where the target ID is the first element and the + output data is the second element. + + :rtype: Any | (Any, Any) + """ + raise NotImplementedError("'__call__' method not implemented") diff --git a/Products/ZenCollector/configcache/modelchange/processor.py b/Products/ZenCollector/configcache/modelchange/processor.py new file mode 100644 index 0000000000..dff7ec82ad --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/processor.py @@ -0,0 +1,227 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging + +from itertools import chain + +from ZODB.POSException import POSKeyError +from zope.component import subscribers + +from Products.ZenHub.interfaces import ( + FILTER_INCLUDE, + FILTER_EXCLUDE, + IInvalidationOid, +) +from Products.ZenModel.DeviceComponent import DeviceComponent +from Products.ZenRelations.PrimaryPathObjectManager import ( + PrimaryPathObjectManager, +) + +from ..misc import into_tuple + +from .invalidation import Invalidation, InvalidationCause +from .pipeline import Pipe, IterablePipe, Action + +log = logging.getLogger("zen.configcache.modelchange") + + +class InvalidationProcessor(object): + """Takes an invalidation and produces ZODB objects. + + An invalidation is represented as an ZODB object ID (oid). + An oid is accepted by the `get` method and a sequence of ZODB objects + are returned. Generally, only one object is in the sequence, but it + is possible for more than one object to be returned. + """ + + def __init__(self, app, filters): + """Initialize an InvalidationProcessor instance. + + :param app: The dmd object + :type app: + :param filters: A list of filters to apply to the invalidation + :type filters: + """ + oid2obj_1 = Pipe(OidToObject(app)) + oid2obj_2 = IterablePipe(OidToObject(app)) + apply_filter = Pipe(ApplyFilters(filters)) + apply_transforms = Pipe(ApplyTransforms()) + + self.__results = CollectInvalidations() + collect = Pipe(self.__results) + + oid2obj_1.connect(apply_filter) + oid2obj_1.connect(collect, tid=OidToObject.SINK) + apply_filter.connect(apply_transforms) + apply_transforms.connect(oid2obj_2) + apply_transforms.connect(collect, tid=ApplyTransforms.SINK) + oid2obj_2.connect(collect) + oid2obj_2.connect(collect, tid=OidToObject.SINK) + + self.__pipeline = oid2obj_1.node() + + def apply(self, oid): + """Send data into the pipeline.""" + self.__pipeline.send(oid) + return self.__results.pop() + + +class OidToObject(Action): + """Validates the OID to ensure it references a device.""" + + SINK = "sink" + + def __init__(self, app): + """ + Initialize an OidToObject instance. + + :param app: ZODB application root object. + :type app: OFS.Application.Application + """ + super(OidToObject, self).__init__() + self._app = app + + def __call__(self, oid): + try: + # Retrieve the object using its OID. + obj = self._app._p_jar[oid] + except POSKeyError: + # Skip if this OID doesn't exist. + return + # Exclude the object if it doesn't have the right base class. + if not isinstance(obj, (PrimaryPathObjectManager, DeviceComponent)): + return + try: + # Wrap the bare object into an Acquisition wrapper. + obj = obj.__of__(self._app.zport.dmd).primaryAq() + except (AttributeError, KeyError): + # An exception at this implies a deleted device. + return ( + self.SINK, + Invalidation(oid, obj, InvalidationCause.Removed), + ) + else: + return ( + self.DEFAULT, + Invalidation(oid, obj, InvalidationCause.Updated), + ) + + +class ApplyFilters(Action): + """ + Filter the invalidation against IInvalidationFilter objects. + + Invalidations explicitely excluded by a filter are dropped from the + pipeline. + """ + + def __init__(self, filters): + """ + Initialize a FilterObject instance. + + :param filters: The invalidation filters. + :type filters: Sequence[IInvalidationFilter] + """ + super(ApplyFilters, self).__init__() + self._filters = filters + + def __call__(self, invalidation): + for fltr in self._filters: + result = fltr.include(invalidation.device) + if result in (FILTER_INCLUDE, FILTER_EXCLUDE): + if result is FILTER_EXCLUDE: + log.debug( + "invalidation excluded by filter filter=%r device=%s", + fltr, + invalidation.device + ) + break + else: + result = FILTER_INCLUDE + if result is not FILTER_EXCLUDE: + return invalidation + + +class ApplyTransforms(Action): + """ + The invalidation pipeline concerns itself with certain types of + objects. The `ApplyTransforms` node determines whether the OID refers + to a nested object within a desired object type and if the OID is + a nested object, the OID of the parent object is returned to be used + in its place. + """ + + SINK = "sink" + + def __call__(self, invalidation): + # First, get any subscription adapters registered as transforms + adapters = subscribers((invalidation.device,), IInvalidationOid) + # Next check for an old-style (regular adapter) transform + try: + adapters = chain( + adapters, (IInvalidationOid(invalidation.device),) + ) + except TypeError: + # No old-style adapter is registered + pass + transformed = set() + for adapter_ in adapters: + result = adapter_.transformOid(invalidation.oid) + if isinstance(result, str): + transformed.add(result) + elif hasattr(result, "__iter__"): + # If the transform didn't give back a string, it should have + # given back an iterable + transformed.update(result) + else: + log.warn( + "IInvalidationOid adaptor returned a bad result " + "adaptor=%r result=%r device=%s oid=%s", + adapter_, + result, + invalidation.device, + invalidation.oid, + ) + # Remove any Nones a transform may have returned. + transformed.discard(None) + # Remove the original OID from the set of transformed OIDs; + # we don't want the original OID if any OIDs were returned. + transformed.discard(invalidation.oid) + + if transformed: + return (self.DEFAULT, transformed) + return (self.SINK, (invalidation,)) + + +class CollectInvalidations(Action): + """Collects the results of the pipeline.""" + + def __init__(self): + self._output = set() + + def __call__(self, result): + results = into_tuple(result) + for result in results: + log.debug( + "collected an invalidation reason=%s device=%s oid=%r", + result.reason, + result.device, + result.oid, + ) + self._output.update(results) + + def pop(self): + """Return the collected data, removing it from the set.""" + try: + return self._output.copy() + finally: + self._output.clear() diff --git a/Products/ZenCollector/configcache/modelchange/tests/mock_interface.py b/Products/ZenCollector/configcache/modelchange/tests/mock_interface.py new file mode 100644 index 0000000000..c02d7c18f5 --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/tests/mock_interface.py @@ -0,0 +1,29 @@ +import types +from mock import Mock +from zope.interface import classImplements + + +def create_interface_mock(interface_class): + """given a Zope Interface class + return a Mock sub class + that implements the given Zope interface class. + + Mock objects created from this InterfaceMock will + have Attributes and Methods required in the Interface + will not have Attributes or Methods that are not specified + """ + + # the init method, automatically spec the interface methods + def init(self, *args, **kwargs): + Mock.__init__(self, spec=interface_class.names(), *args, **kwargs) + + # subclass named 'Mock' + name = interface_class.__name__ + "Mock" + + # create the class object and provide the init method + klass = types.TypeType(name, (Mock,), {"__init__": init}) + + # the new class should implement the interface + classImplements(klass, interface_class) + + return klass diff --git a/Products/ZenCollector/configcache/modelchange/tests/test_filters.py b/Products/ZenCollector/configcache/modelchange/tests/test_filters.py new file mode 100644 index 0000000000..59d343ebaf --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/tests/test_filters.py @@ -0,0 +1,303 @@ +from mock import Mock, patch, create_autospec +from Products.ZCatalog.interfaces import ICatalogBrain +from unittest import TestCase +from zope.interface.verify import verifyObject + +from ..filters import ( + _getZorCProperties, + _iszorcustprop, + BaseOrganizerFilter, + DeviceClass, + DeviceClassInvalidationFilter, + FILTER_CONTINUE, + FILTER_EXCLUDE, + IgnorableClassesFilter, + IInvalidationFilter, + md5, + OSProcessClass, + OSProcessClassFilter, + OSProcessOrganizer, + OSProcessOrganizerFilter, +) +from .mock_interface import create_interface_mock + +PATH = {"invalidationfilter": "Products.ZenHub.invalidationfilter"} + + +class IgnorableClassesFilterTest(TestCase): + def setUp(t): + t.icf = IgnorableClassesFilter() + + def test_init(t): + IInvalidationFilter.providedBy(t.icf) + # current version fails because weight attribute is not defined + # icf.weight = 1 + # verifyObject(IInvalidationFilter, icf) + t.assertTrue(hasattr(t.icf, "CLASSES_TO_IGNORE")) + + def test_initialize(t): + context = Mock(name="context") + t.icf.initialize(context) + # No return or side-effects + + def test_include(t): + obj = Mock(name="object") + out = t.icf.include(obj) + t.assertEqual(out, FILTER_CONTINUE) + + def test_include_excludes_classes_to_ignore(t): + t.icf.CLASSES_TO_IGNORE = str + out = t.icf.include("ignore me!") + t.assertEqual(out, FILTER_EXCLUDE) + + +class BaseOrganizerFilterTest(TestCase): + def setUp(t): + t.types = Mock(name="types") + t.bof = BaseOrganizerFilter(t.types) + + # @patch with autospec fails (https://bugs.python.org/issue23078) + # manually spec ZenPropertyManager + t.organizer = Mock( + name="Products.ZenRelations.ZenPropertyManager", + spec_set=[ + "zenPropertyIds", + "getProperty", + "zenPropIsPassword", + "zenPropertyString", + ], + ) + + def test_init(t): + IInvalidationFilter.providedBy(t.bof) + verifyObject(IInvalidationFilter, t.bof) + t.assertEqual(t.bof.weight, 10) + t.assertEqual(t.bof._types, t.types) + + def test_iszorcustprop(t): + result = _iszorcustprop("no match") + t.assertEqual(result, None) + result = _iszorcustprop("cProperty") + t.assertTrue(result) + result = _iszorcustprop("zProperty") + t.assertTrue(result) + + def test_getRoot(t): + context = Mock(name="context") + root = t.bof.getRoot(context) + t.assertEqual(root, context.dmd.primaryAq()) + + @patch( + "{invalidationfilter}.IModelCatalogTool".format(**PATH), + autospec=True, + spec_set=True, + ) + def test_initialize(t, IModelCatalogTool): + # Create a Mock object that provides the ICatalogBrain interface + ICatalogBrainMock = create_interface_mock(ICatalogBrain) + brain = ICatalogBrainMock() + + IModelCatalogTool.return_value.search.return_value = [brain] + checksum = create_autospec(t.bof.organizerChecksum) + t.bof.organizerChecksum = checksum + context = Mock(name="context") + + t.bof.initialize(context) + + t.assertEqual( + t.bof.checksum_map, + {brain.getPath.return_value: checksum.return_value}, + ) + + def test_getZorCProperties(t): + zprop = Mock(name="zenPropertyId", spec_set=[]) + t.organizer.zenPropertyIds.return_value = [zprop, zprop] + + # getZorCProperties returns a generator + results = _getZorCProperties(t.organizer) + + t.organizer.zenPropIsPassword.return_value = False + zId, propertyString = next(results) + t.assertEqual(zId, zprop) + t.assertEqual( + propertyString, t.organizer.zenPropertyString.return_value + ) + t.organizer.zenPropertyString.assert_called_with(zprop) + + t.organizer.zenPropIsPassword.return_value = True + zId, propertyString = next(results) + t.assertEqual(zId, zprop) + t.assertEqual( + propertyString, t.organizer.getProperty.return_value + ) + t.organizer.getProperty.assert_called_with(zprop, "") + + with t.assertRaises(StopIteration): + next(results) + + @patch( + "{invalidationfilter}._getZorCProperties".format(**PATH), + autospec=True, + spec_set=True, + ) + def test_generateChecksum(t, _getZorCProps): + zprop = Mock(name="zenPropertyId", spec_set=[]) + data = (zprop, "property_string") + _getZorCProps.return_value = [data] + actual = md5() + + expect = md5() + expect.update("%s|%s" % data) + + t.bof.generateChecksum(t.organizer, actual) + + _getZorCProps.assert_called_with(t.organizer) + t.assertEqual(actual.hexdigest(), expect.hexdigest()) + + @patch( + "{invalidationfilter}._getZorCProperties".format(**PATH), + autospec=True, + spec_set=True, + ) + def test_organizerChecksum(t, _getZorCProps): + zprop = Mock(name="zenPropertyId", spec_set=[]) + data = (zprop, "property_string") + _getZorCProps.return_value = [data] + + out = t.bof.organizerChecksum(t.organizer) + + expect = md5() + expect.update("%s|%s" % data) + t.assertEqual(out, expect.hexdigest()) + + def test_include_ignores_non_matching_types(t): + t.bof._types = (str,) + ret = t.bof.include(False) + t.assertEqual(ret, FILTER_CONTINUE) + + def test_include_if_checksum_changed(t): + organizerChecksum = create_autospec(t.bof.organizerChecksum) + t.bof.organizerChecksum = organizerChecksum + t.bof._types = (Mock,) + obj = Mock(name="object", spec_set=["getPrimaryPath"]) + obj.getPrimaryPath.return_value = ["dmd", "brain"] + organizer_path = "/".join(obj.getPrimaryPath()) + t.bof.checksum_map = {organizer_path: "existing_checksum"} + organizerChecksum.return_value = "current_checksum" + + ret = t.bof.include(obj) + + t.assertEqual(ret, FILTER_CONTINUE) + + def test_include_if_checksum_unchanged(t): + organizerChecksum = create_autospec(t.bof.organizerChecksum) + t.bof.organizerChecksum = organizerChecksum + existing_checksum = "checksum" + current_checksum = "checksum" + organizerChecksum.return_value = current_checksum + t.bof._types = (Mock,) + obj = Mock(name="object", spec_set=["getPrimaryPath"]) + obj.getPrimaryPath.return_value = ["dmd", "brain"] + organizer_path = "/".join(obj.getPrimaryPath()) + t.bof.checksum_map = {organizer_path: existing_checksum} + + ret = t.bof.include(obj) + + t.assertEqual(ret, FILTER_EXCLUDE) + + +class DeviceClassInvalidationFilterTest(TestCase): + def setUp(t): + t.dcif = DeviceClassInvalidationFilter() + + def test_init(t): + IInvalidationFilter.providedBy(t.dcif) + verifyObject(IInvalidationFilter, t.dcif) + t.assertEqual(t.dcif._types, (DeviceClass,)) + + def test_getRoot(t): + context = Mock(name="context") + root = t.dcif.getRoot(context) + t.assertEqual(root, context.dmd.Devices.primaryAq()) + + @patch( + "{invalidationfilter}.BaseOrganizerFilter.generateChecksum".format( + **PATH + ), + autospec=True, + spec_set=True, + ) + def test_generateChecksum(t, super_generateChecksum): + md5_checksum = md5() + organizer = Mock( + name="Products.ZenRelations.ZenPropertyManager", + spec_set=["rrdTemplates"], + ) + rrdTemplate = Mock(name="rrdTemplate") + rrdTemplate.exportXml.return_value = "some exemel" + organizer.rrdTemplates.return_value = [rrdTemplate] + + t.dcif.generateChecksum(organizer, md5_checksum) + + # We cannot validate the output of the current version, refactor needed + rrdTemplate.exportXml.was_called_once() + super_generateChecksum.assert_called_with( + t.dcif, organizer, md5_checksum + ) + + +class OSProcessOrganizerFilterTest(TestCase): + def test_init(t): + ospof = OSProcessOrganizerFilter() + + IInvalidationFilter.providedBy(ospof) + verifyObject(IInvalidationFilter, ospof) + t.assertEqual(ospof._types, (OSProcessOrganizer,)) + + def test_getRoot(t): + ospof = OSProcessOrganizerFilter() + context = Mock(name="context") + root = ospof.getRoot(context) + t.assertEqual(root, context.dmd.Processes.primaryAq()) + + +class OSProcessClassFilterTest(TestCase): + def setUp(t): + t.ospcf = OSProcessClassFilter() + + def test_init(t): + IInvalidationFilter.providedBy(t.ospcf) + verifyObject(IInvalidationFilter, t.ospcf) + + t.assertEqual(t.ospcf._types, (OSProcessClass,)) + + def test_getRoot(t): + context = Mock(name="context") + root = t.ospcf.getRoot(context) + t.assertEqual(root, context.dmd.Processes.primaryAq()) + + @patch( + "{invalidationfilter}.BaseOrganizerFilter.generateChecksum".format( + **PATH + ), + autospec=True, + spec_set=True, + ) + def test_generateChecksum(t, super_generateChecksum): + organizer = Mock( + name="Products.ZenRelations.ZenPropertyManager", + spec_set=["property_id", "_properties"], + ) + prop = {"id": "property_id"} + organizer._properties = [prop] + organizer.property_id = "value" + md5_checksum = md5() + + t.ospcf.generateChecksum(organizer, md5_checksum) + + expect = md5() + expect.update("%s|%s" % (prop["id"], getattr(organizer, prop["id"]))) + t.assertEqual(md5_checksum.hexdigest(), expect.hexdigest()) + super_generateChecksum.assert_called_with( + t.ospcf, organizer, md5_checksum + ) diff --git a/Products/ZenCollector/configcache/modelchange/tests/test_oids.py b/Products/ZenCollector/configcache/modelchange/tests/test_oids.py new file mode 100644 index 0000000000..bb3a9f0151 --- /dev/null +++ b/Products/ZenCollector/configcache/modelchange/tests/test_oids.py @@ -0,0 +1,70 @@ +from unittest import TestCase +from mock import Mock + +# Breaks unittest independence due to +# ImportError: No module named CMFCore.DirectoryView +from Products.ZenHub.invalidationoid import ( + DefaultOidTransform, + DeviceOidTransform, + IInvalidationOid, + PrimaryPathObjectManager, +) + +from zope.interface.verify import verifyObject +from zope.component import adaptedBy + + +class DefaultOidTransformTest(TestCase): + def setUp(self): + self.obj = Mock(spec_set=PrimaryPathObjectManager) + self.default_oid_transform = DefaultOidTransform(self.obj) + + def test_implements_IInvalidationOid(self): + # Provides the interface + IInvalidationOid.providedBy(self.default_oid_transform) + # Implements the interface it according to spec + verifyObject(IInvalidationOid, self.default_oid_transform) + + def test_adapts_PrimaryPathObjectManager(self): + self.assertEqual( + list(adaptedBy(DefaultOidTransform)), [PrimaryPathObjectManager] + ) + + def test_init(self): + self.assertEqual(self.default_oid_transform._obj, self.obj) + + def test_transformOid(self): + ret = self.default_oid_transform.transformOid("unmodified oid") + self.assertEqual(ret, "unmodified oid") + + +class DeviceOidTransformTest(TestCase): + def setUp(self): + self.obj = Mock(spec_set=PrimaryPathObjectManager) + self.device_oid_transform = DeviceOidTransform(self.obj) + + def test_implements_IInvalidationOid(self): + # Provides the interface + IInvalidationOid.providedBy(self.device_oid_transform) + # Implements the interface it according to spec + verifyObject(IInvalidationOid, self.device_oid_transform) + + def test_init(self): + self.assertEqual(self.device_oid_transform._obj, self.obj) + + def test_transformOid(self): + """returns unmodified oid, if _obj has no device attribute""" + self.assertFalse(hasattr(self.obj, "device")) + ret = self.device_oid_transform.transformOid("unmodified oid") + self.assertEqual(ret, "unmodified oid") + + def test_transformOid_returns_device_oid(self): + """returns obj.device()._p_oid if obj.device exists""" + obj = Mock(name="PrimaryPathObjectManager", spec_set=["device"]) + device = Mock(name="device", spec_set=["_p_oid"]) + obj.device.return_value = device + + device_oid_transform = DeviceOidTransform(obj) + ret = device_oid_transform.transformOid("ignored oid") + + self.assertEqual(ret, obj.device.return_value._p_oid) diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py new file mode 100644 index 0000000000..433f1d7a40 --- /dev/null +++ b/Products/ZenCollector/configcache/task.py @@ -0,0 +1,106 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from datetime import datetime +from time import time + +from zope.component import createObject +from zope.dottedname.resolve import resolve + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from Products.Jobber.task import requires, DMD, Abortable +from Products.Jobber.zenjobs import app + +from .cache import ConfigKey, ConfigQuery, ConfigRecord + + +@app.task( + bind=True, + base=requires(DMD, Abortable), + name="configcache.build_device_config", + summary="Create Device Configuration Task", + description_template="Create the configuration for device {2}.", + ignore_result=True, +) +def build_device_config(self, monitorname, deviceid, configclassname): + """ + Create a configuration for the given device. + + @param monitorname: The name of the monitor/collector the device + is a member of. + @type monitorname: str + @param deviceid: The ID of the device + @type deviceid: str + @param configclassname: The fully qualified name of the class that + will create the device configuration. + @type configclassname: str + """ + svcconfigclass = resolve(configclassname) + svcname = configclassname.rsplit(".", 1)[0] + store = _getStore() + # Change the configuration's status from 'pending' to 'building' so + # that configcache-manager doesn't prematurely timeout the build. + store.set_building((ConfigKey(svcname, monitorname, deviceid), time())) + self.log.info( + "building device configuration device=%s monitor=%s service=%s", + deviceid, + monitorname, + svcname, + ) + + service = svcconfigclass(self.dmd, monitorname) + configs = service.remote_getDeviceConfigs((deviceid,)) + if not configs: + self.log.info( + "no configuration built device=%s monitor=%s service=%s", + deviceid, + monitorname, + svcname, + ) + key = next( + store.search( + ConfigQuery( + service=svcname, monitor=monitorname, device=deviceid + ) + ), + None, + ) + if key is not None: + # No result means device was deleted or moved to another monitor. + store.remove(key) + self.log.info( + "removed previously built configuration " + "device=%s monitor=%s service=%s", + key.device, + key.monitor, + key.service, + ) + else: + config = configs[0] + uid = self.dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() + record = ConfigRecord.make( + svcname, monitorname, deviceid, uid, time(), config + ) + store.add(record) + self.log.info( + "added/replaced config " + "updated=%s device=%s monitor=%s service=%s", + datetime.fromtimestamp(record.updated).isoformat(), + deviceid, + monitorname, + svcname, + ) + + +def _getStore(): + client = getRedisClient(url=getRedisUrl()) + return createObject("configcache-store", client) diff --git a/Products/ZenCollector/configcache/tests/__init__.py b/Products/ZenCollector/configcache/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Products/ZenCollector/configcache/tests/test_bestmatchmap.py b/Products/ZenCollector/configcache/tests/test_bestmatchmap.py new file mode 100644 index 0000000000..53f3f5f80e --- /dev/null +++ b/Products/ZenCollector/configcache/tests/test_bestmatchmap.py @@ -0,0 +1,70 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2019, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +from unittest import TestCase + +from ..utils.propertymap import DevicePropertyMap + + +class EmptyBestMatchMapTest(TestCase): + """Test an empty BestMatchMap object.""" + + def setUp(t): + t.bmm = DevicePropertyMap({}) + + def tearDown(t): + del t.bmm + + def test_get(t): + t.assertIsNone(t.bmm.get("/zport/dmd/Devices")) + + def test_smallest_value(t): + t.assertIsNone(t.bmm.smallest_value()) + + +class BestMatchMapTest(TestCase): + """Test a BestMatchMap object.""" + + mapping = { + "/zport/dmd/Devices": 10, + "/zport/dmd/Devices/Server/Linux": 11, + "/zport/dmd/Devices/Server/SSH/Linux/devices/my-device": 12, + "/zport/dmd/Devices/vSphere": 13, + "/zport/dmd/Devices/Network": 14, + } + + def setUp(t): + t.bmm = DevicePropertyMap(t.mapping) + + def tearDown(t): + del t.bmm + + def test_get_root(t): + value = t.bmm.get("/zport/dmd/Devices/Server-stuff/devices/dev2") + t.assertEqual(10, value) + + def test_get_exact_match(t): + value = t.bmm.get( + "/zport/dmd/Devices/Server/SSH/Linux/devices/my-device" + ) + t.assertEqual(12, value) + + def test_get_best_match(t): + value = t.bmm.get("/zport/dmd/Devices/Server/Linux/devices/dev3") + t.assertEqual(11, value) + + def test_get_too_short_request(t): + value = t.bmm.get("/Devices") + t.assertIsNone(value) + + def test_smallest_value(t): + value = t.bmm.smallest_value() + t.assertEqual(10, value) diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py new file mode 100644 index 0000000000..21b35a6995 --- /dev/null +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -0,0 +1,667 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import collections + +from unittest import TestCase + +from Products.ZenCollector.services.config import DeviceProxy +from Products.Jobber.tests.utils import subTest, RedisLayer + +from ..cache.storage import ConfigStore +from ..cache import ( + ConfigKey, + ConfigQuery, + ConfigRecord, + ConfigStatus, +) + + +_fields = collections.namedtuple( + "_fields", "service monitor device uid updated" +) + + +class EmptyConfigStoreTest(TestCase): + """Test an empty ConfigStore object.""" + + layer = RedisLayer + + def setUp(t): + t.store = ConfigStore(t.layer.redis) + + def tearDown(t): + del t.store + + def test_search(t): + t.assertIsInstance(t.store.search(), collections.Iterable) + t.assertTupleEqual(tuple(t.store.search()), ()) + + def test_get_with_default_default(t): + key = ConfigKey("a", "b", "c") + t.assertIsNone(t.store.get(key)) + + def test_get_with_nondefault_default(t): + key = ConfigKey("a", "b", "c") + dflt = object() + t.assertEqual(t.store.get(key, dflt), dflt) + + def test_remove(t): + t.assertIsNone(t.store.remove()) + + def test_get_status_no_keys(t): + result = t.store.get_status() + t.assertIsInstance(result, collections.Iterable) + t.assertTupleEqual(tuple(result), ()) + + def test_get_status_unknown_key(t): + key = ConfigKey("a", "b", "c") + result = t.store.get_status(key) + t.assertIsInstance(result, collections.Iterable) + t.assertTupleEqual(tuple(result), ()) + + def test_get_pending(t): + result = t.store.get_pending() + t.assertIsInstance(result, collections.Iterable) + t.assertTupleEqual(tuple(result), ()) + + def test_get_older(t): + result = t.store.get_older(1.0) + t.assertIsInstance(result, collections.Iterable) + t.assertTupleEqual(tuple(result), ()) + + def test_get_newer(t): + result = t.store.get_newer(1.0) + t.assertIsInstance(result, collections.Iterable) + t.assertTupleEqual(tuple(result), ()) + + def test_search_badarg(t): + with t.assertRaises(TypeError): + t.store.search("blargh") + + +class _BaseTest(TestCase): + # Base class to share setup code + + layer = RedisLayer + + fields = ( + _fields("a", "b", "c1", "/c1", 1234500.0), + _fields("a", "b", "c2", "/c1", 1234550.0), + ) + + def setUp(t): + DeviceProxy.__eq__ = _compare_configs + t.store = ConfigStore(t.layer.redis) + t.config1 = _make_config("test1", "_test1", "abc-test-01") + t.config2 = _make_config("test2", "_test2", "abc-test-02") + t.record1 = ConfigRecord.make( + t.fields[0].service, + t.fields[0].monitor, + t.fields[0].device, + t.fields[0].uid, + t.fields[0].updated, + t.config1, + ) + t.record2 = ConfigRecord.make( + t.fields[1].service, + t.fields[1].monitor, + t.fields[1].device, + t.fields[1].uid, + t.fields[1].updated, + t.config2, + ) + + def tearDown(t): + del t.store + del t.config1 + del t.config2 + del t.record1 + del t.record2 + del DeviceProxy.__eq__ + + +class ConfigStoreAddTest(_BaseTest): + """Test the `add` method of ConfigStore.""" + + def test_add_new_config(t): + t.store.add(t.record1) + t.store.add(t.record2) + expected1 = ConfigKey( + t.fields[0].service, + t.fields[0].monitor, + t.fields[0].device, + ) + expected2 = ConfigKey( + t.fields[1].service, + t.fields[1].monitor, + t.fields[1].device, + ) + result = tuple(t.store.search()) + t.assertEqual(2, len(result)) + t.assertIn(expected1, result) + t.assertIn(expected2, result) + + result = t.store.get(t.record1.key) + t.assertIsInstance(result, ConfigRecord) + t.assertEqual(t.record1, result) + + result = t.store.get(t.record2.key) + t.assertIsInstance(result, ConfigRecord) + t.assertEqual(t.record2, result) + + +class ConfigStoreSearchTest(_BaseTest): + """Test the `search` method of ConfigStore.""" + + def test_negative_search(t): + t.store.add(t.record1) + cases = ( + {"service": "x"}, + {"service": "x", "monitor": "y"}, + {"service": "x", "monitor": "y", "device": "z"}, + {"monitor": "y"}, + {"monitor": "y", "device": "z"}, + {"device": "z"}, + ) + for case in cases: + with subTest(key=case): + result = tuple(t.store.search(ConfigQuery(**case))) + t.assertTupleEqual((), result) + + def test_positive_search_single(t): + t.store.add(t.record1) + f0 = t.fields[0] + cases = ( + {"service": f0.service}, + {"service": f0.service, "monitor": f0.monitor}, + { + "service": f0.service, + "monitor": f0.monitor, + "device": f0.device, + }, + {"monitor": f0.monitor}, + {"monitor": f0.monitor, "device": f0.device}, + {"device": f0.device}, + ) + for case in cases: + with subTest(key=case): + result = tuple(t.store.search(ConfigQuery(**case))) + t.assertTupleEqual((t.record1.key,), result) + + def test_positive_search_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + f0 = t.fields[0] + cases = ( + ({"service": f0.service}, 2), + ({"service": f0.service, "monitor": f0.monitor}, 2), + ( + { + "service": f0.service, + "monitor": f0.monitor, + "device": f0.device, + }, + 1, + ), + ({"monitor": f0.monitor}, 2), + ({"monitor": f0.monitor, "device": f0.device}, 1), + ({"device": f0.device}, 1), + ) + for args, count in cases: + with subTest(key=args): + result = tuple(t.store.search(ConfigQuery(**args))) + t.assertEqual(count, len(result)) + + +class ConfigStoreGetStatusTest(_BaseTest): + """Test the `get_status` method of ConfigStore.""" + + def test_get_status(t): + t.store.add(t.record1) + t.store.add(t.record2) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.fields[0].updated, status.updated) + + result = tuple(t.store.get_status(t.record2.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record2.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.fields[1].updated, status.updated) + + +class ConfigStoreGetOlderTest(_BaseTest): + """Test the `get_older` method of ConfigStore.""" + + def test_get_older_less_single(t): + t.store.add(t.record1) + result = tuple(t.store.get_older(t.record1.updated - 1)) + t.assertEqual(0, len(result)) + + def test_get_older_less_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + + result = tuple(t.store.get_older(t.record1.updated - 1)) + t.assertEqual(0, len(result)) + + result = tuple(t.store.get_older(t.record2.updated - 1)) + key, status = result[0] + t.assertEqual(1, len(result)) + t.assertEqual(t.record1.key, key) + t.assertEqual(t.record1.updated, status.updated) + + def test_get_older_equal_single(t): + t.store.add(t.record1) + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + def test_get_older_equal_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + result = sorted( + t.store.get_older(t.record2.updated), key=lambda x: x[1].updated + ) + t.assertEqual(2, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + key, status = result[1] + t.assertEqual(t.record2.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record2.updated, status.updated) + + def test_get_older_greater_single(t): + t.store.add(t.record1) + result = tuple(t.store.get_older(t.record1.updated + 1)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + def test_get_older_greater_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + + result = tuple(t.store.get_older(t.record1.updated + 1)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + result = sorted( + t.store.get_older(t.record2.updated + 1), + key=lambda x: x[1].updated, + ) + t.assertEqual(2, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + key, status = result[1] + t.assertEqual(t.record2.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record2.updated, status.updated) + + +class ConfigStoreGetNewerTest(_BaseTest): + """Test the `get_newer` method of ConfigStore.""" + + def test_get_newer_less_single(t): + t.store.add(t.record1) + result = tuple(t.store.get_newer(t.record1.updated - 1)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + def test_get_newer_less_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + + result = sorted( + t.store.get_newer(t.record1.updated - 1), + key=lambda x: x[1].updated, + ) + t.assertEqual(2, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + key, status = result[1] + t.assertEqual(t.record2.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record2.updated, status.updated) + + def test_get_newer_equal_single(t): + t.store.add(t.record1) + result = tuple(t.store.get_newer(t.record1.updated)) + t.assertEqual(0, len(result)) + + def test_get_newer_equal_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + + result = tuple(t.store.get_newer(t.record1.updated)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record2.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record2.updated, status.updated) + + def test_get_newer_greater_single(t): + t.store.add(t.record1) + result = tuple(t.store.get_newer(t.record1.updated + 1)) + t.assertEqual(0, len(result)) + + def test_get_newer_greater_multiple(t): + t.store.add(t.record1) + t.store.add(t.record2) + result = tuple(t.store.get_newer(t.record1.updated + 1)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record2.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record2.updated, status.updated) + + +class ConfigLifeCycleTest(_BaseTest): + """ + Test status transitions. + """ + + def test_expired(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Expired) + + def test_expired_is_not_older(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(0, len(result)) + + def test_expire_pending(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) + t.store.set_expired(t.record1.key) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Expired) + + def test_pending_without_expiring(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + t.store.set_pending((t.record1.key, submitted)) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + def test_pending(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Pending) + t.assertEqual(submitted, status.submitted) + + def test_pending_is_not_older(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + submitted = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) + + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(0, len(result)) + + def test_building(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started)) + t.store.set_building((t.record1.key, started)) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + def test_building_without_pending(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_building((t.record1.key, started)) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + def test_building_is_not_older(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + started = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_building((t.record1.key, started)) + + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(0, len(result)) + + def test_add_overwrites_expired(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + t.store.add(t.record1) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + + def test_add_overwrites_pending(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) + t.store.add(t.record1) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + + def test_add_overwrites_building(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_building((t.record1.key, submitted)) + t.store.add(t.record1) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + + +class DeviceMonitorChangeTest(_BaseTest): + """ + Test when a device changes its monitor. + """ + + def test_add_monitor_change(t): + t.store.add(t.record1) + newmonitor = "b2" + updated = t.record1.updated + 1000 + newrecord = ConfigRecord.make( + t.record1.service, + newmonitor, + t.record1.device, + t.record1.uid, + updated, + t.record1.config, + ) + t.store.add(newrecord) + + result = t.store.get(t.record1.key) + t.assertIsNone(result) + + result = t.store.get(newrecord.key) + t.assertEqual(newrecord, result) + + +class DeviceUIDTest(TestCase): + + layer = RedisLayer + + def setUp(t): + t.device_name = "qadevice" + t.device_uid = "/zport/dmd/Devices/Server/Linux/devices/qadevice" + t.store = ConfigStore(t.layer.redis) + t.config1 = _make_config("qadevice", "qadevice", "abc-test-01") + t.config2 = _make_config("qadevice", "qadevice", "abc-test-01") + t.record1 = ConfigRecord.make( + "snmp", + "localhost", + t.device_name, + t.device_uid, + 123456.23, + t.config1, + ) + t.record2 = ConfigRecord.make( + "ping", + "localhost", + t.device_name, + t.device_uid, + 123654.23, + t.config2, + ) + + def tearDown(t): + del t.store + del t.config1 + del t.config2 + del t.record1 + del t.record2 + + def test_uid(t): + t.store.add(t.record1) + t.store.add(t.record2) + + t.assertEqual(t.device_uid, t.store.get_uid(t.device_name)) + + records = tuple( + t.store.get(key) + for key in t.store.search(ConfigQuery(device=t.device_name)) + ) + + t.assertEqual(2, len(records)) + t.assertNotEqual(records[0], records[1]) + t.assertEqual(records[0].uid, records[1].uid) + t.assertEqual(t.device_uid, records[0].uid) + + def test_uid_after_one_removal(t): + t.store.add(t.record1) + t.store.add(t.record2) + t.store.remove(t.record1.key) + + records = tuple( + t.store.get(key) + for key in t.store.search(ConfigQuery(device=t.device_name)) + ) + + t.assertEqual(1, len(records)) + t.assertEqual(t.device_uid, records[0].uid) + + def test_uid_after_removing_all(t): + t.store.add(t.record1) + t.store.add(t.record2) + t.store.remove(t.record1.key, t.record2.key) + + records = tuple( + t.store.get(key) + for key in t.store.search(ConfigQuery(device=t.device_name)) + ) + + t.assertEqual(0, len(records)) + t.assertIsNone(t.store.get_uid(t.device_name)) + + +def _make_config(_id, configId, guid): + config = DeviceProxy() + config.id = _id + config._config_id = configId + config._device_guid = guid + return config + + +# _compare_configs used to monkeypatch DeviceProxy +# to make equivalent instances equal. + + +def _compare_configs(self, cfg): + return all( + ( + self.id == cfg.id, + self._config_id == cfg._config_id, + self._device_guid == cfg._device_guid, + ) + ) diff --git a/Products/ZenCollector/configcache/utils/__init__.py b/Products/ZenCollector/configcache/utils/__init__.py new file mode 100644 index 0000000000..dfac1040ef --- /dev/null +++ b/Products/ZenCollector/configcache/utils/__init__.py @@ -0,0 +1,31 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from .propertymap import DevicePropertyMap +from .dispatcher import BuildConfigTaskDispatcher +from .pollers import RelStorageInvalidationPoller +from .services import getConfigServices + + +class Constants(object): + + build_timeout_id = "zDeviceConfigBuildTimeout" + pending_timeout_id = "zDeviceConfigPendingTimeout" + time_to_live_id = "zDeviceConfigTTL" + + +__all__ = ( + "BuildConfigTaskDispatcher", + "Constants", + "DevicePropertyMap", + "RelStorageInvalidationPoller", + "getConfigServices", +) diff --git a/Products/ZenCollector/configcache/utils/dispatcher.py b/Products/ZenCollector/configcache/utils/dispatcher.py new file mode 100644 index 0000000000..1fafe55312 --- /dev/null +++ b/Products/ZenCollector/configcache/utils/dispatcher.py @@ -0,0 +1,66 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from ..task import build_device_config + + +class BuildConfigTaskDispatcher(object): + """Encapsulates the act of dispatching the build_device_config task.""" + + def __init__(self, configClasses): + """ + Initialize a BuildConfigTaskDispatcher instance. + + The `configClasses` parameter should be the classes used to create + the device configurations. + + @type configClasses: Sequence[Class] + """ + self._sigs = { + cls.__module__: build_device_config.s( + ".".join((cls.__module__, cls.__name__)) + ) + for cls in configClasses + } + + def dispatch_all(self, monitorid, deviceid, timeout): + """ + Submit a task to build a device configuration from each + configuration service. + """ + soft_limit, hard_limit = _get_limits(timeout) + for sig in self._sigs.values(): + sig.apply_async( + (monitorid, deviceid), + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) + + def dispatch(self, servicename, monitorid, deviceid, timeout): + """ + Submit a task to build device configurations for the specified device. + + @type servicename: str + @type monitorid: str + @type deviceId: str + """ + sig = self._sigs[servicename] + if sig: + soft_limit, hard_limit = _get_limits(timeout) + sig.apply_async( + (monitorid, deviceid), + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) + + +def _get_limits(timeout): + return timeout, (timeout + (timeout * 0.1)) diff --git a/Products/ZenCollector/configcache/utils/pollers.py b/Products/ZenCollector/configcache/utils/pollers.py new file mode 100644 index 0000000000..fad585b642 --- /dev/null +++ b/Products/ZenCollector/configcache/utils/pollers.py @@ -0,0 +1,88 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging + +from itertools import chain + +from zope.component import getUtilitiesFor + +from Products.ZenHub.interfaces import IInvalidationFilter + +from ..modelchange import InvalidationProcessor + +log = logging.getLogger("zen.configcache") + + +class RelStorageInvalidationPoller(object): + """ + Wraps a :class:`relstorage.storage.RelStorage` object to provide an + API to return the latest database invalidations. + """ + + def __init__(self, storage, session, dmd): + """ + Initialize a RelStorageInvalidationPoller instance. + + :param storage: relstorage storage object + :type storage: :class:`relstorage.storage.RelStorage` + """ + self.__storage = storage + self.__session = session + app = dmd.getPhysicalRoot() + filters = initialize_invalidation_filters(dmd) + self.__processor = InvalidationProcessor(app, filters) + + def poll(self): + """ + Return an iterable of ZODB objects that have changed since the last + time `poll` was called. + + :rtype: Iterable[ZODB object] + """ + self.__session.sync() + oids = self.__storage.poll_invalidations() + if not oids: + return () + return set( + chain.from_iterable(self.__processor.apply(oid) for oid in oids) + ) + + +def initialize_invalidation_filters(ctx): + """ + Return initialized IInvalidationFilter objects in a list. + + :param ctx: Used to initialize the IInvalidationFilter objects. + :type ctx: DataRoot + :return: Initialized IInvalidationFilter objects + :rtype: List[IInvalidationFilter] + """ + try: + filters = (f for n, f in getUtilitiesFor(IInvalidationFilter)) + invalidation_filters = [] + for fltr in sorted(filters, key=lambda f: getattr(f, "weight", 100)): + fltr.initialize(ctx) + invalidation_filters.append(fltr) + log.info( + "registered %s invalidation filters.", len(invalidation_filters) + ) + if log.isEnabledFor(logging.DEBUG): + log.debug( + "invalidation filters: %s", + ", ".join( + "{0.__module__}.{0.__class__.__name__}".format(flt) + for flt in invalidation_filters + ) + ) + return invalidation_filters + except Exception: + log.exception("error in initialize_invalidation_filters") diff --git a/Products/ZenCollector/configcache/utils/propertymap.py b/Products/ZenCollector/configcache/utils/propertymap.py new file mode 100644 index 0000000000..3ebfc4edee --- /dev/null +++ b/Products/ZenCollector/configcache/utils/propertymap.py @@ -0,0 +1,72 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +class DevicePropertyMap(object): + """ + This class accepts a mapping of ZODB paths to a value. + + Users can interrogate the class instance by providing a path and + expecting the value from the mapping that best matches the given path. + + A 'best match' means the path with the longest match starting from the + left end of the path. + """ + + @classmethod + def from_organizer(cls, obj, propname, relName="devices"): + return cls(getPropertyValues(obj, propname, relName=relName)) + + def __init__(self, values): + self.__values = tuple((p.split("/")[1:], v) for p, v in values.items()) + + def smallest_value(self): + try: + return min(self.__values, key=lambda item: item[1])[1] + except ValueError: + return None + + def get(self, request_uid): + # Split the request into its parts + req_parts = request_uid.split("/")[1:] + # Find all the path parts that match the request + matches = ( + (len(parts), value) + for parts, value in self.__values + if req_parts[0 : len(parts)] == parts + ) + try: + # Return the value associated with the path parts having + # the longest match with the request. + return max(matches, key=lambda item: item[0])[1] + except ValueError: + # No path parts matched the request. + return None + + +def getPropertyValues(obj, propname, relName="devices"): + """ + Returns a mapping of UID -> property-value for the given z-property. + """ + values = {obj.getPrimaryId(): obj.getZ(propname)} + values.update( + (inst.getPrimaryId(), inst.getZ(propname)) + for inst in obj.getSubInstances(relName) + if inst.isLocal(propname) + ) + values.update( + (inst.getPrimaryId(), inst.getZ(propname)) + for inst in obj.getOverriddenObjects(propname) + ) + if not values or any(v is None for v in values.values()): + raise RuntimeError( + "one or more values are None or z-property is missing " + "z-property=%s" % (propname,) + ) + return values diff --git a/Products/ZenCollector/configcache/utils/services.py b/Products/ZenCollector/configcache/utils/services.py new file mode 100644 index 0000000000..df8bbe8e1a --- /dev/null +++ b/Products/ZenCollector/configcache/utils/services.py @@ -0,0 +1,107 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import importlib +import inspect +import itertools +import pathlib2 as pathlib + +import Products +import ZenPacks + +from Products.ZenCollector.services.config import CollectorConfigService + +_excluded_config_classes = ("NullConfigService", "NullConfig") + + +def mod_from_path(path): + """ + Returns the module path of the given path to a Python code file. + + The module path is the path to a file with a ".py" extension. + The package path is rooted at "Products" or "ZenPacks". + + >>> mod_from_path("/opt/zenoss/Products/ZenHub/services/ProcessConfig.py") + Products.ZenHub.services.ProcessConfig + + :param path: The module path + :type path: pathlib.Path + :returns: The package path + :rtype: pathlib.Path + """ + if "Products" in path.parts: + offset = path.parts.index("Products") + elif "ZenPacks" in path.parts: + offset = path.parts.index("ZenPacks") + return ".".join(itertools.chain(path.parts[offset:-1], [path.stem])) + + +def getConfigServicesFromModule(name): + """ + Returns a tuple containing all the config service classes in the module. + An empty tuple is returned if no config service classes are found. + + :param name: The full name of the module. + :type name: pathlib.Path + :returns: Tuple of Configuration service classes + :rtype: tuple[CollectorConfigService] + """ + try: + mod = importlib.import_module(name) + classes = ( + cls + for nm, cls in inspect.getmembers(mod, inspect.isclass) + if cls.__module__ == mod.__name__ + ) + # CollectorConfigService is excluded because it is the base + # class for all other configuration services and not used + # directly by any collection daemon. + return tuple( + cls + for cls in classes + if cls is not CollectorConfigService + and issubclass(cls, CollectorConfigService) + ) + except ImportError: + return () + + +def getConfigServices(): + """ + Returns a tuple containing all the installed config service classes. + An empty tuple is returned if no config service classes are found. + + Configuration service classes are expected to be found in modules + that found in a package named "services". The "services" package can + be found in multiple package paths. + + :returns: Tuple of configuration service classes + :rtype: tuple[CollectorConfigService] + """ + search_paths = itertools.chain(Products.__path__, ZenPacks.__path__) + service_paths = ( + svcpath + for path in search_paths + for svcpath in pathlib.Path(path).rglob("**/services") + ) + module_names = ( + mod_from_path(codepath) + for path in service_paths + for codepath in path.rglob("*.py") + if codepath.stem != "__init__" and "tests" not in codepath.parts + ) + return tuple( + cls + for cls in itertools.chain.from_iterable( + getConfigServicesFromModule(name) for name in module_names + ) + if cls.__name__ not in _excluded_config_classes + ) diff --git a/Products/ZenCollector/configcache/version.py b/Products/ZenCollector/configcache/version.py new file mode 100644 index 0000000000..1b488684a3 --- /dev/null +++ b/Products/ZenCollector/configcache/version.py @@ -0,0 +1,33 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from .misc import app_name +from .misc.args import get_subparser + + +class Version(object): + + description = "Display the version and exit" + + @staticmethod + def add_arguments(parser, subparsers): + subp_version = get_subparser( + subparsers, "version", Version.description + ) + subp_version.set_defaults(factory=Version) + + def __init__(self, args): + pass + + def run(self): + from Products.ZenModel.ZenossInfo import ZenossInfo + + zinfo = ZenossInfo("") + version = zinfo.getZenossVersion().short() + print("{} {}".format(app_name(), version)) diff --git a/Products/ZenCollector/configure.zcml b/Products/ZenCollector/configure.zcml index 7118ff77ed..5e4a6fb977 100644 --- a/Products/ZenCollector/configure.zcml +++ b/Products/ZenCollector/configure.zcml @@ -1,11 +1,24 @@ + + - + + + - + diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index d66cebc2d4..86c3ee9455 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -7,11 +7,11 @@ # ############################################################################## -import signal -import time -import logging +import itertools import json import re +import signal +import time from optparse import SUPPRESS_HELP @@ -19,17 +19,23 @@ from metrology.instruments import Gauge from twisted.internet import defer, reactor, task from twisted.python.failure import Failure -from zope.component import getUtilitiesFor, provideUtility, queryUtility +from zope.component import ( + getUtilitiesFor, + provideUtility, + queryUtility, + getUtility, +) from zope.interface import implementer -from Products.ZenHub.PBDaemon import PBDaemon, FakeRemote +import Products.ZenCollector as ZENCOLLECTOR_MODULE + from Products.ZenRRD.RRDDaemon import RRDDaemon from Products.ZenUtils import metrics from Products.ZenUtils.deprecated import deprecated from Products.ZenUtils.observable import ObservableProxy -from Products.ZenUtils.picklezipper import Zipper -from Products.ZenUtils.Utils import importClass, unused +from Products.ZenUtils.Utils import importClass, load_config +from .config import DeviceConfigLoader from .interfaces import ( ICollector, ICollectorPreferences, @@ -38,107 +44,14 @@ IDataService, IEventService, IFrameworkFactory, - IStatistic, IStatisticsService, ITaskSplitter, ) -from .utils.maintenance import MaintenanceCycle - -log = logging.getLogger("zen.daemon") - - -@implementer(IConfigurationListener) -class DummyListener(object): - """ - No-op implementation of a listener that can be registered with instances - of ConfigListenerNotifier class. - """ - - def deleted(self, configurationId): - log.debug("DummyListener: configuration %s deleted", configurationId) - - def added(self, configuration): - log.debug("DummyListener: configuration %s added", configuration) - - def updated(self, newConfiguration): - log.debug("DummyListener: configuration %s updated", newConfiguration) - - -@implementer(IConfigurationListener) -class ConfigListenerNotifier(object): - """ - Registers other IConfigurationListener objects and notifies them when - this object is notified of configuration removals, adds, and updates. - """ - - _listeners = [] - - def addListener(self, listener): - self._listeners.append(listener) - - def deleted(self, configurationId): - """ - Notify listener when a configuration is deleted. - - :param configurationId: The ID of the deleted configuration. - :type configurationId: str - """ - for listener in self._listeners: - listener.deleted(configurationId) - - def added(self, configuration): - """ - Notify the listeners when a configuration is added. - - :param configuration: The added configuration object. - :type configuration: DeviceProxy - """ - for listener in self._listeners: - listener.added(configuration) - - def updated(self, newConfiguration): - """ - Notify the listeners when a configuration has changed. - - :param newConfiguration: The updated configuration object. - :type newConfiguration: DeviceProxy - """ - for listener in self._listeners: - listener.updated(newConfiguration) - - -@implementer(IConfigurationListener) -class DeviceGuidListener(object): - """ - Manages configuration IDs on the given 'daemon' object, making the - necessary changes when notified of configuration additions, removals, - and updates. - """ - - def __init__(self, daemon): - """ - Initialize a DeviceGuidListener instance. - - :param daemon: The daemon object. - :type daemon: CollectorDaemon - """ - self._daemon = daemon - - def deleted(self, configurationId): - self._daemon._deviceGuids.pop(configurationId, None) +from .listeners import ConfigListenerNotifier - def added(self, configuration): - deviceGuid = getattr(configuration, "deviceGuid", None) - if deviceGuid: - self._daemon._deviceGuids[configuration.id] = deviceGuid +# from .statistics import StatisticsService +from .utils.maintenance import MaintenanceCycle, ZenHubHeartbeatSender - def updated(self, newConfiguration): - deviceGuid = getattr(newConfiguration, "deviceGuid", None) - if deviceGuid: - self._daemon._deviceGuids[newConfiguration.id] = deviceGuid - - -DUMMY_LISTENER = DummyListener() CONFIG_LOADER_NAME = "configLoader" @@ -146,27 +59,26 @@ def updated(self, newConfiguration): class CollectorDaemon(RRDDaemon): """The daemon class for the entire ZenCollector framework.""" - _frameworkFactoryName = "" - """ - Identifies the IFrameworkFactory implementation to use. + _frameworkFactoryName = "default" # type: str + """Identifies the IFrameworkFactory implementation to use.""" - :type: str - """ + # CollectorDaemon has an additional service: ConfigCache + initialServices = RRDDaemon.initialServices + ["ConfigCache"] @property - def preferences(self): - """ - The preferences object of this daemon. - - :rtype: ICollectorPreferences - """ + def preferences(self): # type: () -> ICollectorPreferences + """The preferences object of this daemon.""" return self._prefs + @property + def frameworkFactoryName(self): + return self._frameworkFactoryName + def __init__( self, preferences, taskSplitter, - configurationListener=DUMMY_LISTENER, + configurationListener=None, initializationCallback=None, stoppingCallback=None, ): @@ -195,10 +107,11 @@ def __init__( raise TypeError("configuration must provide ICollectorPreferences") if not ITaskSplitter.providedBy(taskSplitter): raise TypeError("taskSplitter must provide ITaskSplitter") - if not IConfigurationListener.providedBy(configurationListener): - raise TypeError( - "configurationListener must provide IConfigurationListener" - ) + if configurationListener is not None: + if not IConfigurationListener.providedBy(configurationListener): + raise TypeError( + "configurationListener must provide IConfigurationListener" + ) self._prefs = ObservableProxy(preferences) self._prefs.attachAttributeObserver( @@ -206,8 +119,8 @@ def __init__( ) self._taskSplitter = taskSplitter self._configListener = ConfigListenerNotifier() - self._configListener.addListener(configurationListener) - self._configListener.addListener(DeviceGuidListener(self)) + if configurationListener is not None: + self._configListener.addListener(configurationListener) self._initializationCallback = initializationCallback self._stoppingCallback = stoppingCallback @@ -225,71 +138,35 @@ def __init__( ICollectorPreferences, self.preferences.collectorName, ) + # There's only one preferences object, so also register an + # anonymous ICollectorPreferences utility. + provideUtility( + self.preferences, + ICollectorPreferences, + ) super(CollectorDaemon, self).__init__( name=self.preferences.collectorName ) - self._statService = StatisticsService() - provideUtility(self._statService, IStatisticsService) - - if self.options.cycle: - # setup daemon statistics (deprecated names) - self._statService.addStatistic("devices", "GAUGE") - self._statService.addStatistic("dataPoints", "DERIVE") - self._statService.addStatistic("runningTasks", "GAUGE") - self._statService.addStatistic("taskCount", "GAUGE") - self._statService.addStatistic("queuedTasks", "GAUGE") - self._statService.addStatistic("missedRuns", "GAUGE") - - # namespace these a bit so they can be used in ZP monitoring. - # prefer these stat names and metrology in future refs - self._dataPointsMetric = Metrology.meter( - "collectordaemon.dataPoints" - ) - daemon = self - - class DeviceGauge(Gauge): - @property - def value(self): - return len(daemon._devices) - - Metrology.gauge("collectordaemon.devices", DeviceGauge()) - - # Scheduler statistics - class RunningTasks(Gauge): - @property - def value(self): - return daemon._scheduler._executor.running - Metrology.gauge("collectordaemon.runningTasks", RunningTasks()) + load_config("collector.zcml", ZENCOLLECTOR_MODULE) - class TaskCount(Gauge): - @property - def value(self): - return daemon._scheduler.taskCount - - Metrology.gauge("collectordaemon.taskCount", TaskCount()) - - class QueuedTasks(Gauge): - @property - def value(self): - return daemon._scheduler._executor.queued - - Metrology.gauge("collectordaemon.queuedTasks", QueuedTasks()) - - class MissedRuns(Gauge): - @property - def value(self): - return daemon._scheduler.missedRuns + configFilter = parseWorkerOptions(self.options.__dict__, self.log) + if configFilter: + self.preferences.configFilter = configFilter - Metrology.gauge("collectordaemon.missedRuns", MissedRuns()) + dcui = self.options.device_config_update_interval + if dcui: + # Convert minutes to seconds + self._device_config_update_interval = dcui * 60 + else: + # This covers the case where the device_config_update_interval + # value is None, zero, or some other False-like value. + self._device_config_update_interval = 300 self._deviceGuids = {} - self._devices = set() self._unresponsiveDevices = set() self._rrd = None - self._metric_writer = None - self._derivative_tracker = None self.reconfigureTimeout = None # Keep track of pending tasks if we're doing a single run, and not a @@ -298,21 +175,19 @@ def value(self): self._completedTasks = 0 self._pendingTasks = [] - frameworkFactory = queryUtility( - IFrameworkFactory, self._frameworkFactoryName - ) - self._configProxy = frameworkFactory.getConfigurationProxy() - self._scheduler = frameworkFactory.getScheduler() + self._configProxy = None + self._ConfigurationLoaderTask = None + framework = _getFramework(self.frameworkFactoryName) + self._scheduler = framework.getScheduler() self._scheduler.maxTasks = self.options.maxTasks - self._ConfigurationLoaderTask = ( - frameworkFactory.getConfigurationLoaderTask() - ) + + self._statService = getUtility(IStatisticsService) + if self.options.cycle: + _configure_stats_service(self._statService, self) # Set the initialServices attribute so that the PBDaemon class # will load all of the remote services we need. - self.initialServices = PBDaemon.initialServices + [ - self.preferences.configurationService - ] + self.initialServices.append(self.preferences.configurationService) # Trap SIGUSR2 so that we can display detailed statistics signal.signal(signal.SIGUSR2, self._signalHandler) @@ -332,8 +207,11 @@ def value(self): # Flag that indicates the daemon is loading the cached configs self.loadingCachedConfigs = False + self._deviceloader = None + self._deviceloadertask = None + self._deviceloadertaskd = None + def buildOptions(self): - """Overrides base class to add additional configuration options.""" super(CollectorDaemon, self).buildOptions() maxTasks = getattr(self.preferences, "maxTasks", None) @@ -351,8 +229,9 @@ def buildOptions(self): dest="logTaskStats", type="int", default=0, - help="How often to logs statistics of current tasks, " - "value in seconds; very verbose", + help="How often to logs statistics of current tasks, value in " + "seconds; very verbose. Value of zero disables logging of " + "task statistics.", ) addWorkerOptions(self.parser) self.parser.add_option( @@ -369,17 +248,18 @@ def buildOptions(self): default=None, help="trace metrics whose key value matches this regex", ) - - frameworkFactory = queryUtility( - IFrameworkFactory, self._frameworkFactoryName + self.parser.add_option( + "--device-config-update-interval", + type="int", + default=5, + help="The interval, in minutes, that device configs are " + "checked for updates (default %default).", ) - if hasattr(frameworkFactory, "getFrameworkBuildOptions"): - # During upgrades we'll be missing this option - self._frameworkBuildOptions = ( - frameworkFactory.getFrameworkBuildOptions() - ) - if self._frameworkBuildOptions: - self._frameworkBuildOptions(self.parser) + + framework = _getFramework(self.frameworkFactoryName) + buildOpts = framework.getFrameworkBuildOptions() + if buildOpts: + buildOpts(self.parser) # give the collector configuration a chance to add options, too self.preferences.buildOptions(self.parser) @@ -389,26 +269,32 @@ def parseOptions(self): super(CollectorDaemon, self).parseOptions() self.preferences.options = self.options - configFilter = parseWorkerOptions(self.options.__dict__) - if configFilter: - self.preferences.configFilter = configFilter + def watchdogCycleTime(self): + """ + Return our cycle time (in minutes) + :return: cycle time + :rtype: integer + """ + return self.preferences.cycleInterval * 2 + + @defer.inlineCallbacks def connected(self): """Invoked after a connection to ZenHub is established.""" - return self._startup() - - def connectTimeout(self): - """Invoked after timing out while connecting to ZenHub.""" - super(CollectorDaemon, self).connectTimeout() - return self._startup() - - def _startup(self): - d = defer.maybeDeferred(self._getInitializationCallback()) - d.addCallback(self._initEncryptionKey) - d.addCallback(self._startConfigCycle) - d.addCallback(self._startMaintenance) - d.addErrback(self._errorStop) - return d + try: + yield defer.maybeDeferred(self._getInitializationCallback()) + framework = _getFramework(self.frameworkFactoryName) + self.log.info("Using framework -> %r", framework) + self._configProxy = framework.getConfigurationProxy() + yield self._initEncryptionKey() + yield self._startConfigCycle() + yield self._startMaintenance() + yield self._startTaskStatsLogging() + yield self._startDeviceConfigLoader() + except Exception as ex: + self.log.critical("unrecoverable error: %s", ex) + self.log.exception("failed during startup") + self.stop() def _getInitializationCallback(self): if self._initializationCallback is not None: @@ -416,28 +302,97 @@ def _getInitializationCallback(self): return lambda: None @defer.inlineCallbacks - def _initEncryptionKey(self, prv_cb_result=None): + def _initEncryptionKey(self): # Encrypt dummy msg in order to initialize the encryption key. # The 'yield' does not return until the key is initialized. data = yield self._configProxy.encrypt("Hello") if data: # Encrypt returns None if an exception is raised self.encryptionKeyInitialized = True - self.log.info("Daemon's encryption key initialized") + self.log.debug("initialized encryption key") - def watchdogCycleTime(self): - """ - Return our cycle time (in minutes) + def _startConfigCycle(self, startDelay=0): + framework = _getFramework(self.frameworkFactoryName) + configLoader = framework.getConfigurationLoaderTask()( + CONFIG_LOADER_NAME, taskConfig=self.preferences + ) + configLoader.startDelay = startDelay + # Don't add the config loader task if the scheduler already has + # an instance of it. + if configLoader not in self._scheduler: + # Run initial maintenance cycle as soon as possible + # TODO: should we not run maintenance if running in + # non-cycle mode? + self._scheduler.addTask(configLoader) + self.log.info("scheduled task task=%s", configLoader.name) + else: + self.log.info("task already scheduled task=%s", configLoader.name) - :return: cycle time - :rtype: integer - """ - return self.preferences.cycleInterval * 2 + def _startMaintenance(self): + if not self.options.cycle: + return + interval = self.preferences.cycleInterval + + if self.worker_id == 0: + heartbeatSender = ZenHubHeartbeatSender( + self.options.monitor, + self.name, + self.options.heartbeatTimeout, + self._eventqueue, + ) + else: + heartbeatSender = None + self._maintenanceCycle = MaintenanceCycle( + interval, heartbeatSender, self._maintenanceCallback + ) + self._maintenanceCycle.start() + self.log.debug("started maintenance cycle interval=%s", interval) + def _startTaskStatsLogging(self): + if not (self.options.cycle and self.options.logTaskStats): + return + self._taskstatslogger = task.LoopingCall( + self._displayStatistics, verbose=True + ) + self._taskstatsloggerd = self._taskstatslogger.start( + self.options.logTaskStats, now=False + ) + self.log.debug( + "started logging task statistics interval=%d", + self.options.logTaskStats, + ) + reactor.addSystemEventTrigger( + "before", "shutdown", self._taskstatslogger.stop, "before" + ) + + def _startDeviceConfigLoader(self): + self.log.info( + "running the device config loader every %d seconds", + self._device_config_update_interval, + ) + self._deviceloader = DeviceConfigLoader( + self.options, + self._configProxy, + self._deviceConfgCallback, + ) + self._deviceloadertask = task.LoopingCall(self._deviceloader) + self._deviceloadertaskd = self._deviceloadertask.start( + self._device_config_update_interval + ) + reactor.addSystemEventTrigger( + "before", "shutdown", self._deviceloadertask.stop, "before" + ) + + @defer.inlineCallbacks + def getRemoteConfigCacheProxy(self): + """Return the remote configuration cache proxy.""" + proxy = yield self.getService("ConfigCache") + defer.returnValue(proxy) + + @defer.inlineCallbacks def getRemoteConfigServiceProxy(self): """Return the remote configuration service proxy object.""" - return self.services.get( - self.preferences.configurationService, FakeRemote() - ) + proxy = yield self.getService(self.preferences.configurationService) + defer.returnValue(proxy) def generateEvent(self, event, **kw): eventCopy = super(CollectorDaemon, self).generateEvent(event, **kw) @@ -518,7 +473,7 @@ def writeMetric( min = 0 dkey = "%s:%s" % (contextUUID, metric) - value = self._derivative_tracker.derivative( + value = self.derivativeTracker().derivative( dkey, (float(value), timestamp), min, max ) @@ -527,14 +482,14 @@ def writeMetric( # write the metric to Redis try: yield defer.maybeDeferred( - self._metric_writer.write_metric, + self.metricWriter().write_metric, metric_name, value, timestamp, tags, ) except Exception as e: - self.log.debug("Error sending metric %s", e) + self.log.debug("error sending metric %s", e) yield defer.maybeDeferred( self._threshold_notifier.notify, contextUUID, @@ -626,74 +581,9 @@ def stop(self, ignored=""): try: self._stoppingCallback() except Exception: - self.log.exception("Exception while stopping daemon") + self.log.exception("exception while stopping daemon") super(CollectorDaemon, self).stop(ignored) - def remote_deleteDevice(self, devId): - """Remote method invoked by ZenHub when a device is deleted.""" - # guard against parsing updates during a disconnect - if devId is None: - return - self._deleteDevice(devId) - - def remote_deleteDevices(self, deviceIds): - """Remote method invoked by ZenHub when many devices are deleted.""" - # guard against parsing updates during a disconnect - if deviceIds is None: - return - for devId in Zipper.load(deviceIds): - self._deleteDevice(devId) - - def remote_updateDeviceConfig(self, config): - """Remote method invoked by ZenHub when a device config is updated.""" - # guard against parsing updates during a disconnect - if config is None: - return - self.log.debug("Device %s updated", config.configId) - if self._updateConfig(config): - self._configProxy.updateConfigProxy(self.preferences, config) - else: - self.log.debug("Device %s config filtered", config.configId) - - def remote_updateDeviceConfigs(self, configs): - """ - Remote method invoked by ZenHub for multiple device config updates. - """ - if configs is None: - return - configs = Zipper.load(configs) - self.log.debug( - "remote_updateDeviceConfigs: workerid %s processing " - "%s device configs", - self.options.workerid, - len(configs), - ) - for config in configs: - self.remote_updateDeviceConfig(config) - - def remote_notifyConfigChanged(self): - """ - Remote method invoked by ZenHub when the all the device configs - should be replaced. - """ - if self.reconfigureTimeout and self.reconfigureTimeout.active(): - # We will run along with the already scheduled task - self.log.debug("notifyConfigChanged - using existing call") - return - - self.log.debug("notifyConfigChanged - scheduling call in 30 seconds") - self.reconfigureTimeout = reactor.callLater(30, self._rebuildConfig) - - def _rebuildConfig(self): - """ - Delete and re-add the configuration tasks to completely re-build - the configuration. - """ - if self.reconfigureTimeout and not self.reconfigureTimeout.active(): - self.reconfigureTimeout = None - self._scheduler.removeTasksForConfig(CONFIG_LOADER_NAME) - self._startConfigCycle() - def _rescheduleConfig( self, observable, attrName, oldValue, newValue, **kwargs ): @@ -701,14 +591,14 @@ def _rescheduleConfig( Delete and re-add the configuration tasks to start on new interval. """ if oldValue != newValue: - self.log.debug( - "Changing config task interval from %s to %s minutes", + self.log.info( + "changing config task interval from %s to %s minutes", oldValue, newValue, ) self._scheduler.removeTasksForConfig(CONFIG_LOADER_NAME) # values are in minutes, scheduler takes seconds - self._startConfigCycle(startDelay=newValue * 60) + self._startConfigCycle(newValue * 60) def _taskCompleteCallback(self, taskName): # if we're not running a normal daemon cycle then we need to shutdown @@ -726,31 +616,52 @@ def _taskCompleteCallback(self, taskName): self._displayStatistics() self.stop() - def _updateConfig(self, cfg): + def _deviceConfgCallback(self, new, updated, removed): """ - Update device configuration. + Update the device configs for the devices this collector manages. - Return true if config is updated, false if config is skipped. + :param deviceConfigs: a list of device configurations + :type deviceConfigs: list of name,value tuples """ + for deviceId in removed: + self._deleteDevice(deviceId) + for cfg in itertools.chain(new, updated): + self._updateConfig(cfg) + + self.log.debug( + "processed %d new, %d updated, %d removed device configs", + len(new), + len(updated), + len(removed), + ) + + def _deleteDevice(self, deviceId): + self.log.debug("deleted device device-id=%s", deviceId) + self._configListener.deleted(deviceId) + self._scheduler.removeTasksForConfig(deviceId) + + def _updateConfig(self, cfg): + """Update device configuration.""" # guard against parsing updates during a disconnect if cfg is None: - return False - configFilter = getattr(self.preferences, "configFilter", None) or ( - lambda x: True - ) + return + + configFilter = getattr(self.preferences, "configFilter", _always_ok) if not ( (not self.options.device and configFilter(cfg)) or self.options.device in (cfg.id, cfg.configId) ): - self.log.info("Device %s config filtered", cfg.configId) - return False + self.log.info( + "filtered out device config config-id=%s", cfg.configId + ) + return configId = cfg.configId - self.log.debug("Processing configuration for %s", configId) + self.log.info("processing device config config-id=%s", configId) nextExpectedRuns = {} - if configId in self._devices: + if configId in self._deviceloader.deviceIds: tasksToRemove = self._scheduler.getTasksForConfig(configId) nextExpectedRuns = { taskToRemove.name: self._scheduler.getNextExpectedRun( @@ -761,11 +672,10 @@ def _updateConfig(self, cfg): self._scheduler.removeTasks(task.name for task in tasksToRemove) self._configListener.updated(cfg) else: - self._devices.add(configId) self._configListener.added(cfg) newTasks = self._taskSplitter.splitConfiguration([cfg]) - self.log.debug("Tasks for config %s: %s", configId, newTasks) + self.log.debug("tasks for config %s: %s", configId, newTasks) nowTime = time.time() for (taskName, task_) in newTasks.iteritems(): @@ -785,7 +695,7 @@ def _updateConfig(self, cfg): try: self._scheduler.addTask(task_, self._taskCompleteCallback, now) except ValueError: - self.log.exception("Error adding device config") + self.log.exception("failed to schedule task task=%r", task_) continue # TODO: another hack? @@ -797,88 +707,13 @@ def _updateConfig(self, cfg): # all pending tasks have completed if not self.options.cycle: self._pendingTasks.append(taskName) + # put tasks on pause after configuration update to prevent # unnecessary collections ZEN-25463 if configId in self._unresponsiveDevices: - self.log.debug("Pausing tasks for device %s", configId) + self.log.debug("pausing tasks for device %s", configId) self._scheduler.pauseTasksForConfig(configId) - return True - - @defer.inlineCallbacks - def _updateDeviceConfigs(self, updatedConfigs, purgeOmitted): - """ - Update the device configs for the devices this collector manages. - - :param deviceConfigs: a list of device configurations - :type deviceConfigs: list of name,value tuples - """ - self.log.debug( - "updateDeviceConfigs: updatedConfigs=%s", - map(str, updatedConfigs), - ) - - for cfg in updatedConfigs: - self._updateConfig(cfg) - # yield time to reactor so other things can happen - yield task.deferLater(reactor, 0, lambda: None) - - if purgeOmitted: - self._purgeOmittedDevices(cfg.configId for cfg in updatedConfigs) - - def _purgeOmittedDevices(self, updatedDevices): - """ - Delete all current devices that are omitted from the list of devices - being updated. - - :param updatedDevices: a collection of device ids - :type updatedDevices: a sequence of strings - """ - # remove tasks for the deleted devices - deletedDevices = set(self._devices) - set(updatedDevices) - self.log.debug( - "purgeOmittedDevices: deletedConfigs=%s", ",".join(deletedDevices) - ) - for configId in deletedDevices: - self._deleteDevice(configId) - - def _deleteDevice(self, deviceId): - self.log.debug("Device %s deleted", deviceId) - - self._devices.discard(deviceId) - self._configListener.deleted(deviceId) - self._configProxy.deleteConfigProxy(self.preferences, deviceId) - self._scheduler.removeTasksForConfig(deviceId) - - def _errorStop(self, result): - """ - Twisted callback to receive fatal messages. - - :param result: the Twisted failure - :type result: failure object - """ - if isinstance(result, Failure): - msg = result.getErrorMessage() - else: - msg = str(result) - self.log.critical("Unrecoverable Error: %s", msg) - self.stop() - - def _startConfigCycle(self, result=None, startDelay=0): - configLoader = self._ConfigurationLoaderTask( - CONFIG_LOADER_NAME, taskConfig=self.preferences - ) - configLoader.startDelay = startDelay - # Don't add the config loader task if the scheduler already has - # an instance of it. - if configLoader not in self._scheduler: - # Run initial maintenance cycle as soon as possible - # TODO: should we not run maintenance if running in non-cycle mode? - self._scheduler.addTask(configLoader) - else: - self.log.info("%s already added to scheduler", configLoader.name) - return defer.succeed("Configuration loader task started") - def setPropertyItems(self, items): """Override so that preferences are updated.""" super(CollectorDaemon, self).setPropertyItems(items) @@ -889,40 +724,22 @@ def _setCollectorPreferences(self, preferenceItems): if not hasattr(self.preferences, name): setattr(self.preferences, name, value) elif getattr(self.preferences, name) != value: - self.log.debug("Updated %s preference to %s", name, value) + self.log.debug("updated %s preference to %s", name, value) setattr(self.preferences, name, value) def _loadThresholdClasses(self, thresholdClasses): - self.log.debug("Loading classes %s", thresholdClasses) for c in thresholdClasses: try: importClass(c) + self.log.info("imported threshold class class=%r", c) except ImportError: - log.exception("Unable to import class %s", c) + self.log.exception("unable to import class %s", c) def _configureThresholds(self, thresholds): self.getThresholds().updateList(thresholds) - def _startMaintenance(self, ignored=None): - unused(ignored) - if not self.options.cycle: - self._maintenanceCycle() - return - if self.options.logTaskStats > 0: - log.debug("Starting Task Stat logging") - loop = task.LoopingCall(self._displayStatistics, verbose=True) - loop.start(self.options.logTaskStats, now=False) - - interval = self.preferences.cycleInterval - self.log.debug("Initializing maintenance Cycle") - heartbeatSender = self if self.worker_id == 0 else None - maintenanceCycle = MaintenanceCycle( - interval, heartbeatSender, self._maintenanceCycle - ) - maintenanceCycle.start() - @defer.inlineCallbacks - def _maintenanceCycle(self, ignored=None): + def _maintenanceCallback(self, ignored=None): """ Perform daemon maintenance processing on a periodic schedule. @@ -930,7 +747,7 @@ def _maintenanceCycle(self, ignored=None): but afterward will self-schedule each run. """ try: - self.log.debug("Performing periodic maintenance") + self.log.debug("performing periodic maintenance") if not self.options.cycle: ret = "No maintenance required" elif getattr(self.preferences, "pauseUnreachableDevices", True): @@ -940,7 +757,7 @@ def _maintenanceCycle(self, ignored=None): ret = None defer.returnValue(ret) except Exception: - self.log.exception("failure in _maintenanceCycle") + self.log.exception("failure while running maintenance callback") raise @defer.inlineCallbacks @@ -958,12 +775,12 @@ def _pauseUnreachableDevices(self): newUnresponsiveDevices ) for devId in clearedDevices: - self.log.debug("Resuming tasks for device %s", devId) + self.log.debug("resuming tasks for device %s", devId) self._scheduler.resumeTasksForConfig(devId) self._unresponsiveDevices = newUnresponsiveDevices for devId in self._unresponsiveDevices: - self.log.debug("Pausing tasks for device %s", devId) + self.log.debug("pausing tasks for device %s", devId) self._scheduler.pauseTasksForConfig(devId) defer.returnValue(issues) @@ -991,7 +808,7 @@ def postStatisticsImpl(self): # update and post statistics if we've been configured to do so if self.rrdStats: stat = self._statService.getStatistic("devices") - stat.value = len(self._devices) + stat.value = len(self._deviceloader.deviceIds) # stat = self._statService.getStatistic("cyclePoints") # stat.value = self._rrd.endCycle() @@ -1021,14 +838,15 @@ def postStatisticsImpl(self): def _displayStatistics(self, verbose=False): if self.metricWriter(): - self.log.info( - "%d devices processed (%d datapoints)", - len(self._devices), + self.log.debug( + "%d devices processed (%d samples)", + len(self._deviceloader.deviceIds), self.metricWriter().dataPoints, ) else: - self.log.info( - "%d devices processed (0 datapoints)", len(self._devices) + self.log.debug( + "%d devices processed (0 samples)", + len(self._deviceloader.deviceIds), ) self._scheduler.displayStatistics(verbose) @@ -1047,52 +865,8 @@ def worker_id(self): return getattr(self.options, "workerid", 0) -@implementer(IStatistic) -class Statistic(object): - def __init__(self, name, type, **kwargs): - self.value = 0 - self.name = name - self.type = type - self.kwargs = kwargs - - -@implementer(IStatisticsService) -class StatisticsService(object): - def __init__(self): - self._stats = {} - - def addStatistic(self, name, type, **kwargs): - if name in self._stats: - raise NameError("Statistic %s already exists" % name) - - if type not in ("DERIVE", "COUNTER", "GAUGE"): - raise TypeError("Statistic type %s not supported" % type) - - stat = Statistic(name, type, **kwargs) - self._stats[name] = stat - - def getStatistic(self, name): - return self._stats[name] - - def postStatistics(self, rrdStats): - for stat in self._stats.values(): - # figure out which function to use to post this statistical data - try: - func = { - "COUNTER": rrdStats.counter, - "GAUGE": rrdStats.gauge, - "DERIVE": rrdStats.derive, - }[stat.type] - except KeyError: - raise TypeError("Statistic type %s not supported" % stat.type) - - # These should always come back empty now because DaemonStats - # posts the events for us - func(stat.name, stat.value, **stat.kwargs) - - # counter is an ever-increasing value, but otherwise... - if stat.type != "COUNTER": - stat.value = 0 +def _always_ok(*args): + return True def addWorkerOptions(parser): @@ -1109,13 +883,67 @@ def addWorkerOptions(parser): parser.add_option("--workers", type="int", default=1, help=SUPPRESS_HELP) -def parseWorkerOptions(options): +def _getFramework(name): + return queryUtility(IFrameworkFactory, name) + + +def parseWorkerOptions(options, log): dispatchFilterName = options.get("configDispatch", "") if options else "" filterFactories = dict(getUtilitiesFor(IConfigurationDispatchingFilter)) filterFactory = filterFactories.get( dispatchFilterName, None ) or filterFactories.get("", None) if filterFactory: - filter = filterFactory.getFilter(options) - log.debug("Filter configured: %s:%s", filterFactory, filter) - return filter + filt = filterFactory.getFilter(options) + log.debug("configured filter: %s:%s", filterFactory, filt) + return filt + + +def _configure_stats_service(service, daemon): + # setup daemon statistics (deprecated names) + service.addStatistic("devices", "GAUGE") + service.addStatistic("dataPoints", "DERIVE") + service.addStatistic("runningTasks", "GAUGE") + service.addStatistic("taskCount", "GAUGE") + service.addStatistic("queuedTasks", "GAUGE") + service.addStatistic("missedRuns", "GAUGE") + + # namespace these a bit so they can be used in ZP monitoring. + # prefer these stat names and metrology in future refs + daemon._dataPointsMetric = Metrology.meter("collectordaemon.dataPoints") + + class DeviceGauge(Gauge): + @property + def value(self): + return len(daemon._deviceloader.deviceIds) + + Metrology.gauge("collectordaemon.devices", DeviceGauge()) + + # Scheduler statistics + class RunningTasks(Gauge): + @property + def value(self): + return daemon._scheduler._executor.running + + Metrology.gauge("collectordaemon.runningTasks", RunningTasks()) + + class TaskCount(Gauge): + @property + def value(self): + return daemon._scheduler.taskCount + + Metrology.gauge("collectordaemon.taskCount", TaskCount()) + + class QueuedTasks(Gauge): + @property + def value(self): + return daemon._scheduler._executor.queued + + Metrology.gauge("collectordaemon.queuedTasks", QueuedTasks()) + + class MissedRuns(Gauge): + @property + def value(self): + return daemon._scheduler.missedRuns + + Metrology.gauge("collectordaemon.missedRuns", MissedRuns()) diff --git a/Products/ZenCollector/frameworkfactory.py b/Products/ZenCollector/frameworkfactory.py new file mode 100644 index 0000000000..d97a079107 --- /dev/null +++ b/Products/ZenCollector/frameworkfactory.py @@ -0,0 +1,39 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from zope.component import queryUtility +from zope.interface import implementer + +from .config import ConfigurationLoaderTask, ConfigurationProxy +from .interfaces import IFrameworkFactory, ICollectorPreferences +from .scheduler import Scheduler + + +@implementer(IFrameworkFactory) +class CoreCollectorFrameworkFactory(object): + def __init__(self): + self.__configProxy = None + self.__scheduler = None + + def getConfigurationProxy(self): + if self.__configProxy is None: + prefs = queryUtility(ICollectorPreferences) + self.__configProxy = ConfigurationProxy(prefs) + return self.__configProxy + + def getScheduler(self): + if self.__scheduler is None: + self.__scheduler = Scheduler() + return self.__scheduler + + def getConfigurationLoaderTask(self): + return ConfigurationLoaderTask + + def getFrameworkBuildOptions(self): + return None diff --git a/Products/ZenCollector/interfaces.py b/Products/ZenCollector/interfaces.py index e48f06d816..d85e61c89b 100644 --- a/Products/ZenCollector/interfaces.py +++ b/Products/ZenCollector/interfaces.py @@ -113,43 +113,35 @@ class IConfigurationProxy(zope.interface.Interface): the configuration for a collector. """ - def getPropertyItems(prefs): + def getPropertyItems(): """ Retrieve the collector's property items. - @param prefs: the collector preferences object - @type prefs: an object providing ICollectorPreferences @return: properties for this collector @rtype: either a dict or a Deferred """ - def getThresholdClasses(prefs): + def getThresholdClasses(): """ Retrieve the collector's required threshold classes. - @param prefs: the collector preferences object - @type prefs: an object providing ICollectorPreferences @return: the names of all the collector threshold classes to loaded @rtype: an iterable set of strings containing Python class names """ - def getThresholds(prefs): + def getThresholds(): """ Retrieve the collector's threshold definitions. - @param prefs: the collector preferences object - @type prefs: an object providing ICollectorPreferences @return: the threshold definitions @rtype: an iterable set of threshold definitions """ - def getConfigProxies(prefs, ids=[]): + def getConfigProxies(configIds=[]): """ Called by the framework whenever the configuration for this collector should be retrieved. - @param prefs: the collector preferences object - @type prefs: an object providing ICollectorPreferences @param configIds: specific config Ids to be configured @type configIds: an iterable @return: a twisted Deferred, optional in case the configure operation @@ -157,24 +149,6 @@ def getConfigProxies(prefs, ids=[]): @rtype: twisted.internet.defer.Deferred """ - def deleteConfigProxy(prefs, configId): - """ - Called by the framework whenever a configuration should be removed. - @param prefs: the collector preferences object - @type prefs: an object providing ICollectorPreferences - @param configId: the identifier to remove - @type: string - """ - - def updateConfigProxy(prefs, config): - """ - Called by the framework whenever the configuration has been updated by - an external event. - @param prefs: the collector preferences object - @type prefs: an object providing ICollectorPreferences - @param config: the updated configuration - """ - class IScheduler(zope.interface.Interface): """ @@ -441,33 +415,93 @@ class IDataService(zope.interface.Interface): """ def writeMetric( - path, + contextKey, metric, value, - timestamp, metricType, - metricId, - min, - max, - hasThresholds, - threshEventData, - allowStaleDatapoint, + contextId, + timestamp="N", + min="U", + max="U", + threshEventData={}, + deviceId=None, + contextUUID=None, + deviceUUID=None, ): """ Write the value provided for the specified metric to Redis - @param path: metric path - @param metric: name of the incoming metric - @param value: value to be writen to Redis - @param metricType: COUNTER, DERIVE, GAUGE, etc. - @param timestamp: when the value was received - @param metricId: unique identifier for the metric - @param min: metric minimum - @param max: metric maximum - @param hasThresholds: boolean indicating presence of thresholds - for this metricId. - @param allowStaleDatapoint: boolean indicating whether stale - values are OK. + @param contextKey: The device or component the metric applies to. + This is typically in the form a path. + @type contextKey: str + @param metric: The name of the metric, we expect it to be of the form + datasource_datapoint. + @type metric: str + @param value: the value of the metric. + @type value: float + @param metricType: type of the metric (e.g. 'COUNTER', 'GAUGE', + 'DERIVE' etc) + @type metricType: str + @param contextId: used for the threshold events, the ID of the device. + @type contextId: str + @param timestamp: defaults to time.time() if not specified, + the time the metric occurred. + @type timestamp: float + @param min: used in the derive the min value for the metric. + @type min: float + @param max: used in the derive the max value for the metric. + @type max: float + @param threshEventData: extra data put into threshold events. + @type threshEventData: dict + @param deviceId: the id of the device for this metric. + @type deviceId: str + @param contextUUID: The device/component UUID value + @type contextUUID: str + @param deviceUUID: The device UUID value + @type deviceUUID: str + """ + + def writeMetricWithMetadata( + metric, + value, + metricType, + timestamp="N", + min="U", + max="U", + threshEventData={}, + metadata=None, + ): + """ + Basically wraps the `writeMetric` method. The `metadata` parameter + must contain the following fields: + + contextKey : str + contextId : str + deviceId : str + contextUUID : str + deviceUUID : str + + These fields have the same meaning as in the `writeMetric` method. + + @param metric: The name of the metric, we expect it to be of the form + datasource_datapoint. + @type metric: str + @param value: the value of the metric. + @type value: float + @param metricType: type of the metric (e.g. 'COUNTER', 'GAUGE', + 'DERIVE' etc) + @type metricType: str + @param timestamp: defaults to time.time() if not specified, + the time the metric occurred. + @type timestamp: float + @param min: used in the derive the min value for the metric. + @type min: float + @param max: used in the derive the max value for the metric. + @type max: float + @param threshEventData: extra data put into threshold events. + @type threshEventData: dict + @param metadata: Contains contextual data about the metric. + @type metadata: dict """ def writeRRD( @@ -541,15 +575,15 @@ def getScheduler(): Retrieve the framework's implementation of the IScheduler interface. """ - def getConfigurationLoaderTask(): + def getConfigurationLoaderTask(*args, **kw): """ - Retrieve the class definition used by the framework to load - configuration information from zenhub. + Return an instance of the configuration loader task constructed + from the provided arguments. """ - def getFrameworkBuildOptions(): + def getBuildOptions(parser): """ - Retrieve the framework's buildOptions method. + Apply the framework's build options to the given parser object. """ diff --git a/Products/ZenCollector/listeners.py b/Products/ZenCollector/listeners.py new file mode 100644 index 0000000000..5e4d9429ac --- /dev/null +++ b/Products/ZenCollector/listeners.py @@ -0,0 +1,98 @@ +import logging + +from zope.interface import implementer + +from Products.ZenCollector.interfaces import IConfigurationListener + +log = logging.getLogger("zen.daemon.listeners") + + +@implementer(IConfigurationListener) +class DummyListener(object): + """ + No-op implementation of a listener that can be registered with instances + of ConfigListenerNotifier class. + """ + + def deleted(self, configurationId): + log.debug("DummyListener: configuration %s deleted", configurationId) + + def added(self, configuration): + log.debug("DummyListener: configuration %s added", configuration) + + def updated(self, newConfiguration): + log.debug("DummyListener: configuration %s updated", newConfiguration) + + +@implementer(IConfigurationListener) +class ConfigListenerNotifier(object): + """ + Registers other IConfigurationListener objects and notifies them when + this object is notified of configuration removals, adds, and updates. + """ + + _listeners = [] + + def addListener(self, listener): + self._listeners.append(listener) + + def deleted(self, configurationId): + """ + Notify listener when a configuration is deleted. + + :param configurationId: The ID of the deleted configuration. + :type configurationId: str + """ + for listener in self._listeners: + listener.deleted(configurationId) + + def added(self, configuration): + """ + Notify the listeners when a configuration is added. + + :param configuration: The added configuration object. + :type configuration: DeviceProxy + """ + for listener in self._listeners: + listener.added(configuration) + + def updated(self, newConfiguration): + """ + Notify the listeners when a configuration has changed. + + :param newConfiguration: The updated configuration object. + :type newConfiguration: DeviceProxy + """ + for listener in self._listeners: + listener.updated(newConfiguration) + + +@implementer(IConfigurationListener) +class DeviceGuidListener(object): + """ + Manages configuration IDs on the given 'daemon' object, making the + necessary changes when notified of configuration additions, removals, + and updates. + """ + + def __init__(self, daemon): + """ + Initialize a DeviceGuidListener instance. + + :param daemon: The daemon object. + :type daemon: CollectorDaemon + """ + self._daemon = daemon + + def deleted(self, configurationId): + self._daemon._deviceGuids.pop(configurationId, None) + + def added(self, configuration): + deviceGuid = getattr(configuration, "deviceGuid", None) + if deviceGuid: + self._daemon._deviceGuids[configuration.id] = deviceGuid + + def updated(self, newConfiguration): + deviceGuid = getattr(newConfiguration, "deviceGuid", None) + if deviceGuid: + self._daemon._deviceGuids[newConfiguration.id] = deviceGuid diff --git a/Products/ZenCollector/services/config.py b/Products/ZenCollector/services/config.py index bf6a7f7184..d37a3298dd 100644 --- a/Products/ZenCollector/services/config.py +++ b/Products/ZenCollector/services/config.py @@ -10,33 +10,21 @@ import base64 import hashlib import logging -import traceback -from Acquisition import aq_parent from cryptography.fernet import Fernet -from twisted.internet import defer from twisted.spread import pb from ZODB.transact import transact -from zope import component -from Products.ZenEvents.ZenEventClasses import Critical from Products.ZenHub.HubService import HubService -from Products.ZenHub.interfaces import IBatchNotifier from Products.ZenHub.PBDaemon import translateError -from Products.ZenHub.services.Procrastinator import Procrastinate from Products.ZenHub.services.ThresholdMixin import ThresholdMixin -from Products.ZenHub.zodb import onUpdate, onDelete from Products.ZenModel.Device import Device -from Products.ZenModel.DeviceClass import DeviceClass -from Products.ZenModel.PerformanceConf import PerformanceConf -from Products.ZenModel.privateobject import is_private -from Products.ZenModel.RRDTemplate import RRDTemplate -from Products.ZenModel.ZenPack import ZenPack -from Products.ZenUtils.AutoGCObjectReader import gc_cache_every -from Products.ZenUtils.picklezipper import Zipper +from Products.ZenUtils.guid.interfaces import IGlobalIdentifier from Products.Zuul.utils import safe_hasattr as hasattr -from ..interfaces import IConfigurationDispatchingFilter +from .error import trapException +from .optionsfilter import getOptionsFilter +from .push import UpdateCollectorMixin class DeviceProxy(pb.Copyable, pb.RemoteCopy): @@ -75,193 +63,83 @@ def __repr__(self): ) -class CollectorConfigService(HubService, ThresholdMixin): +class CollectorConfigService(HubService, UpdateCollectorMixin, ThresholdMixin): """Base class for ZenHub configuration service classes.""" def __init__(self, dmd, instance, deviceProxyAttributes=()): """ Initializes a CollectorConfigService instance. - @param dmd: the Zenoss DMD reference - @param instance: the collector instance name - @param deviceProxyAttributes: a tuple of names for device attributes + :param dmd: the Zenoss DMD reference + :param instance: the collector instance name + :param deviceProxyAttributes: a tuple of names for device attributes that should be copied to every device proxy created - @type deviceProxyAttributes: tuple + :type deviceProxyAttributes: tuple """ HubService.__init__(self, dmd, instance) + UpdateCollectorMixin.__init__(self) self._deviceProxyAttributes = BASE_ATTRIBUTES + deviceProxyAttributes # Get the collector information (eg the 'localhost' collector) - self._prefs = self.dmd.Monitors.Performance._getOb(self.instance) - self.config = self._prefs # Needed for ThresholdMixin - self.configFilter = None + self.conf = self.dmd.Monitors.getPerformanceMonitor(self.instance) - # When about to notify daemons about device changes, wait for a little - # bit to batch up operations. - self._procrastinator = Procrastinate(self._pushConfig) - self._reconfigProcrastinator = Procrastinate(self._pushReconfigure) - - self._notifier = component.getUtility(IBatchNotifier) - - def _wrapFunction(self, functor, *args, **kwargs): - """ - Call the functor using the arguments, - and trap any unhandled exceptions. - - @parameter functor: function to call - @type functor: method - @parameter args: positional arguments - @type args: array of arguments - @parameter kwargs: keyword arguments - @type kwargs: dictionary - @return: result of functor(*args, **kwargs) or None if failure - @rtype: result of functor - """ - try: - return functor(*args, **kwargs) - except Exception as ex: - msg = "Unhandled exception in zenhub service %s: %s" % ( - self.__class__, - ex, - ) - self.log.exception(msg) - self.sendEvent( - dict( - severity=Critical, - component=str(self.__class__), - traceback=traceback.format_exc(), - summary=msg, - device=self.instance, - methodCall="%s(%s, %s)" % (functor.__name__, args, kwargs), - ) - ) - - @onUpdate(PerformanceConf) - def perfConfUpdated(self, conf, event): - with gc_cache_every(1000, db=self.dmd._p_jar._db): - if conf.id == self.instance: - for listener in self.listeners: - listener.callRemote( - "setPropertyItems", conf.propertyItems() - ) - - @onUpdate(ZenPack) - def zenPackUpdated(self, zenpack, event): - with gc_cache_every(1000, db=self.dmd._p_jar._db): - for listener in self.listeners: - try: - listener.callRemote( - "updateThresholdClasses", - self.remote_getThresholdClasses(), - ) - except Exception: - self.log.warning( - "Error notifying a listener of new classes" - ) - - @onUpdate(Device) - def deviceUpdated(self, device, event): - with gc_cache_every(1000, db=self.dmd._p_jar._db): - self._notifyAll(device) - - @onUpdate(None) # Matches all - def notifyAffectedDevices(self, entity, event): - # FIXME: This is horrible - with gc_cache_every(1000, db=self.dmd._p_jar._db): - if isinstance(entity, self._getNotifiableClasses()): - self._reconfigureIfNotify(entity) - else: - if isinstance(entity, Device): - return - # Something else... mark the devices as out-of-date - template = None - while entity: - # Don't bother with privately managed objects; the ZenPack - # will handle them on its own - if is_private(entity): - return - # Walk up until you hit an organizer or a device - if isinstance(entity, RRDTemplate): - template = entity - if isinstance(entity, DeviceClass): - uid = (self.name(), self.instance) - devfilter = None - if template: - devfilter = _HasTemplate(template, self.log) - self._notifier.notify_subdevices( - entity, uid, self._notifyAll, devfilter - ) - break - if isinstance(entity, Device): - self._notifyAll(entity) - break - entity = aq_parent(entity) - - @onDelete(Device) - def deviceDeleted(self, device, event): - with gc_cache_every(1000, db=self.dmd._p_jar._db): - devid = device.id - collector = device.getPerformanceServer().getId() - # The invalidation is only sent to the collector where the - # deleted device was. - if collector == self.instance: - self.log.debug( - "Invalidation: Performing remote call to delete " - "device %s from collector %s", - devid, - self.instance, - ) - for listener in self.listeners: - listener.callRemote("deleteDevice", devid) - else: - self.log.debug( - "Invalidation: Skipping remote call to delete " - "device %s from collector %s", - devid, - self.instance, - ) + @property + def configFilter(self): + return None @translateError def remote_getConfigProperties(self): - return self._prefs.propertyItems() + try: + items = self.conf.propertyItems() + finally: + pass + return items @translateError def remote_getDeviceNames(self, options=None): - devices = self._getDevices( - deviceFilter=self._getOptionsFilter(options) - ) - return [x.id for x in self._filterDevices(devices)] - - def _getDevices(self, deviceNames=None, deviceFilter=None): - - if not deviceNames: - devices = filter(deviceFilter, self._prefs.devices()) - else: - devices = [] - for name in deviceNames: - device = self.dmd.Devices.findDeviceByIdExact(name) - if not device: - continue - else: - if deviceFilter(device): - devices.append(device) - return devices + return [ + device.id + for device in self._selectDevices(self.conf.devices(), options) + ] @translateError def remote_getDeviceConfigs(self, deviceNames=None, options=None): - deviceFilter = self._getOptionsFilter(options) - devices = self._getDevices(deviceNames, deviceFilter) - devices = self._filterDevices(devices) - - deviceConfigs = [] - for device in devices: - proxies = self._wrapFunction(self._createDeviceProxies, device) + if deviceNames: + devices = _getDevicesByName(self.dmd.Devices, deviceNames) + else: + devices = self.conf.devices() + selected_devices = self._selectDevices(devices, options) + configs = [] + for device in selected_devices: + proxies = trapException(self, self._createDeviceProxies, device) if proxies: - deviceConfigs.extend(proxies) + configs.extend(proxies) + + trapException(self, self._postCreateDeviceProxy, configs) + return configs - self._wrapFunction(self._postCreateDeviceProxy, deviceConfigs) - return deviceConfigs + def _selectDevices(self, devices, options): + # _selectDevices is a generator function returning Device objects. + # `devices` is an iterator returning Device objects. + # `options` is a dict-like object. + predicate = getOptionsFilter(options) + for device in devices: + try: + if all( + ( + predicate(device), + self._perfIdFilter(device), + self._filterDevice(device), + ) + ): + yield device + except Exception as ex: + if self.log.isEnabledFor(logging.DEBUG): + method = self.log.exception + else: + method = self.log.warn + method("error filtering device %r: %s", device, ex) @transact def _create_encryption_key(self): @@ -303,19 +181,18 @@ def _createDeviceProxy(self, device, proxy=None): instance, and then add any additional data to the proxy as their needs require. - @param device: the regular device object to create a proxy from - @return: a new device proxy object, or None if no proxy can be created - @rtype: DeviceProxy + :param device: the regular device object to create a proxy from + :type device: Products.ZenModel.Device + :return: a new device proxy object, or None if no proxy can be created + :rtype: DeviceProxy """ - proxy = proxy if (proxy is not None) else DeviceProxy() + proxy = DeviceProxy() if proxy is None else proxy # copy over all the attributes requested for attrName in self._deviceProxyAttributes: setattr(proxy, attrName, getattr(device, attrName, None)) if isinstance(device, Device): - from Products.ZenUtils.guid.interfaces import IGlobalIdentifier - guid = IGlobalIdentifier(device).getGUID() if guid: setattr(proxy, "_device_guid", guid) @@ -338,58 +215,9 @@ def _filterDevice(self, device): not self.configFilter or self.configFilter(device) ) except AttributeError as e: - self.log.warn( - "got an attribute exception on device.monitorDevice()" - ) - self.log.debug(e) + self.log.warn("No such attribute device=%r error=%s", device, e) return False - def _getOptionsFilter(self, options): - def _alwaysTrue(x): - return True - - deviceFilter = _alwaysTrue - if options: - dispatchFilterName = ( - options.get("configDispatch", "") if options else "" - ) - filterFactories = dict( - component.getUtilitiesFor(IConfigurationDispatchingFilter) - ) - filterFactory = filterFactories.get( - dispatchFilterName, None - ) or filterFactories.get("", None) - if filterFactory: - deviceFilter = filterFactory.getFilter(options) or deviceFilter - return deviceFilter - - def _filterDevices(self, devices): - """ - Filters out devices from the provided list that should not be - converted into DeviceProxy instances and sent back to the collector - client. - - @param device: the device object to filter - @return: a list of devices that are to be included - @rtype: list - """ - filteredDevices = [] - for dev in (d for d in devices if d is not None): - try: - device = dev.primaryAq() - if self._perfIdFilter(device) and self._filterDevice(device): - filteredDevices.append(device) - self.log.debug("Device %s included by filter", device.id) - else: - # don't use .id just in case something crazy returned. - self.log.debug("Device %r excluded by filter", device) - except Exception: - if self.log.isEnabledFor(logging.DEBUG): - self.log.exception("Got an exception filtering %r", dev) - else: - self.log.warn("Got an exception filtering %r", dev) - return filteredDevices - def _perfIdFilter(self, obj): """ Return True if obj is not a device (has no perfServer attribute) @@ -401,184 +229,14 @@ def _perfIdFilter(self, obj): or obj.perfServer.getRelatedId() == self.instance ) - def _notifyAll(self, device): - """Notify all instances (daemons) of a change for the device.""" - # procrastinator schedules a call to _pushConfig - self._procrastinator.doLater(device) - def _pushConfig(self, device): - """Push device config and deletes to relevent collectors/instances.""" - deferreds = [] - - if self._perfIdFilter(device) and self._filterDevice(device): - proxies = self._wrapFunction(self._createDeviceProxies, device) - if proxies: - self._wrapFunction(self._postCreateDeviceProxy, proxies) - else: - proxies = None - - prev_collector = ( - device.dmd.Monitors.primaryAq().getPreviousCollectorForDevice( - device.id - ) - ) - for listener in self.listeners: - if not proxies: - if hasattr(device, "getPerformanceServer"): - # The invalidation is only sent to the previous and - # current collectors. - if self.instance in ( - prev_collector, - device.getPerformanceServer().getId(), - ): - self.log.debug( - "Invalidation: Performing remote call for " - "device %s on collector %s", - device.id, - self.instance, - ) - deferreds.append( - listener.callRemote("deleteDevice", device.id) - ) - else: - self.log.debug( - "Invalidation: Skipping remote call for " - "device %s on collector %s", - device.id, - self.instance, - ) - else: - deferreds.append( - listener.callRemote("deleteDevice", device.id) - ) - self.log.debug( - "Invalidation: Performing remote call for " - "device %s on collector %s", - device.id, - self.instance, - ) - else: - options = self.listenerOptions.get(listener, None) - deviceFilter = self._getOptionsFilter(options) - for proxy in proxies: - if deviceFilter(proxy): - deferreds.append( - self._sendDeviceProxy(listener, proxy) - ) - - return defer.DeferredList(deferreds) - - def _sendDeviceProxy(self, listener, proxy): - return listener.callRemote("updateDeviceConfig", proxy) - - def sendDeviceConfigs(self, configs): - deferreds = [] - - def errback(failure): - self.log.critical( - "Unable to update configs for service instance %s: %s", - self.name(), - failure, - ) - - for listener in self.listeners: - options = self.listenerOptions.get(listener, None) - deviceFilter = self._getOptionsFilter(options) - filteredConfigs = filter(deviceFilter, configs) - args = Zipper.dump(filteredConfigs) - d = listener.callRemote("updateDeviceConfigs", args).addErrback( - errback - ) - deferreds.append(d) - return deferreds - - # FIXME: Don't use _getNotifiableClasses, use @onUpdate(myclasses) - def _getNotifiableClasses(self): - """ - Return a tuple of classes. - - When any object of a type in the sequence is modified the collector - connected to the service will be notified to update its configuration. - - @rtype: tuple - """ - return () - - def _pushReconfigure(self, value): - """Notify the collector to reread the entire configuration.""" - # value is unused but needed for the procrastinator framework - for listener in self.listeners: - listener.callRemote("notifyConfigChanged") - self._reconfigProcrastinator.clear() - - def _reconfigureIfNotify(self, object): - ncc = self._notifyConfigChange(object) - self.log.debug( - "services/config.py _reconfigureIfNotify object=%r " - "_notifyConfigChange=%s", - object, - ncc, - ) - if ncc: - self.log.debug("scheduling collector reconfigure") - self._reconfigProcrastinator.doLater(True) - - def _notifyConfigChange(self, object): - """ - Called when an object of a type from _getNotifiableClasses is - encountered - - @return: should a notify config changed be sent - @rtype: boolean - """ - return True - - -class _HasTemplate(object): - """ - Predicate class that checks whether a given device has a template - matching the given template. - """ - - def __init__(self, template, log): - self.template = template - self.log = log - - def __call__(self, device): - if issubclass(self.template.getTargetPythonClass(), Device): - if self.template in device.getRRDTemplates(): - self.log.debug( - "%s bound to template %s", - device.getPrimaryId(), - self.template.getPrimaryId(), - ) - return True - else: - self.log.debug( - "%s not bound to template %s", - device.getPrimaryId(), - self.template.getPrimaryId(), - ) - return False - else: - # check components, Too expensive? - for comp in device.getMonitoredComponents( - type=self.template.getTargetPythonClass().meta_type - ): - if self.template in comp.getRRDTemplates(): - self.log.debug( - "%s bound to template %s", - comp.getPrimaryId(), - self.template.getPrimaryId(), - ) - return True - else: - self.log.debug( - "%s not bound to template %s", - comp.getPrimaryId(), - self.template.getPrimaryId(), - ) - return False +def _getDevicesByName(ctx, names): + # Returns a generator that produces Device objects. + return ( + device + for device in (ctx.findDeviceByIdExact(name) for name in names) + if device is not None + ) class NullConfigService(CollectorConfigService): diff --git a/Products/ZenCollector/services/error.py b/Products/ZenCollector/services/error.py new file mode 100644 index 0000000000..b7590fb577 --- /dev/null +++ b/Products/ZenCollector/services/error.py @@ -0,0 +1,46 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import traceback + +from Products.ZenEvents.ZenEventClasses import Critical + + +def trapException(service, functor, *args, **kwargs): + """ + Call the functor using the arguments and trap unhandled exceptions. + + :parameter functor: function to call. + :type functor: Callable[Any, Any] + :parameter args: positional arguments to functor. + :type args: Sequence[Any] + :parameter kwargs: keyword arguments to functor. + :type kwargs: Map[Any, Any] + :returns: result of calling functor(*args, **kwargs) + or None if functor raises an exception. + :rtype: Any + """ + try: + return functor(*args, **kwargs) + except Exception as ex: + msg = "Unhandled exception in zenhub service %s: %s" % ( + service.__class__, + ex, + ) + service.log.exception(msg) + service.sendEvent( + { + "severity": Critical, + "component": str(service.__class__), + "traceback": traceback.format_exc(), + "summary": msg, + "device": service.instance, + "methodCall": "%s(%s, %s)" % (functor.__name__, args, kwargs), + } + ) diff --git a/Products/ZenCollector/services/optionsfilter.py b/Products/ZenCollector/services/optionsfilter.py new file mode 100644 index 0000000000..2593773c29 --- /dev/null +++ b/Products/ZenCollector/services/optionsfilter.py @@ -0,0 +1,30 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from zope.component import getUtilitiesFor +from ..interfaces import IConfigurationDispatchingFilter + + +def getOptionsFilter(options): + if options: + name = options.get("configDispatch", "") if options else "" + factories = dict(getUtilitiesFor(IConfigurationDispatchingFilter)) + factory = factories.get(name) + if factory is None: + factory = factories.get("") + if factory is not None: + devicefilter = factory.getFilter(options) + if devicefilter: + return devicefilter + + return _alwaysTrue + + +def _alwaysTrue(*args): + return True diff --git a/Products/ZenCollector/services/push.py b/Products/ZenCollector/services/push.py new file mode 100644 index 0000000000..4f05a1aedf --- /dev/null +++ b/Products/ZenCollector/services/push.py @@ -0,0 +1,303 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from Acquisition import aq_parent +from twisted.internet import defer +from zope.component import getUtility + +from Products.ZenHub.interfaces import IBatchNotifier +from Products.ZenHub.services.Procrastinator import Procrastinate +from Products.ZenHub.zodb import onUpdate, onDelete +from Products.ZenModel.Device import Device +from Products.ZenModel.DeviceClass import DeviceClass +from Products.ZenModel.PerformanceConf import PerformanceConf +from Products.ZenModel.privateobject import is_private +from Products.ZenModel.RRDTemplate import RRDTemplate +from Products.ZenModel.ZenPack import ZenPack +from Products.ZenUtils.AutoGCObjectReader import gc_cache_every +from Products.ZenUtils.picklezipper import Zipper + +from .error import trapException +from .optionsfilter import getOptionsFilter + + +class UpdateCollectorMixin: + """Push data back to collection daemons.""" + + def __init__(self): + # When about to notify daemons about device changes, wait for a little + # bit to batch up operations. + self._procrastinator = Procrastinate(self._pushConfig) + self._reconfigProcrastinator = Procrastinate(self._pushReconfigure) + + self._notifier = getUtility(IBatchNotifier) + + @onUpdate(PerformanceConf) + def perfConfUpdated(self, conf, event): + with gc_cache_every(1000, db=self.dmd._p_jar._db): + if conf.id == self.instance: + for listener in self.listeners: + listener.callRemote( + "setPropertyItems", conf.propertyItems() + ) + + @onUpdate(ZenPack) + def zenPackUpdated(self, zenpack, event): + with gc_cache_every(1000, db=self.dmd._p_jar._db): + for listener in self.listeners: + try: + listener.callRemote( + "updateThresholdClasses", + self.remote_getThresholdClasses(), + ) + except Exception: + self.log.warning( + "Error notifying a listener of new classes" + ) + + @onUpdate(Device) + def deviceUpdated(self, device, event): + with gc_cache_every(1000, db=self.dmd._p_jar._db): + self._notifyAll(device) + + @onUpdate(None) # Matches all + def notifyAffectedDevices(self, entity, event): + # FIXME: This is horrible + with gc_cache_every(1000, db=self.dmd._p_jar._db): + if isinstance(entity, self._getNotifiableClasses()): + self._reconfigureIfNotify(entity) + else: + if isinstance(entity, Device): + return + # Something else... mark the devices as out-of-date + template = None + while entity: + # Don't bother with privately managed objects; the ZenPack + # will handle them on its own + if is_private(entity): + return + # Walk up until you hit an organizer or a device + if isinstance(entity, RRDTemplate): + template = entity + if isinstance(entity, DeviceClass): + uid = (self.name(), self.instance) + devfilter = None + if template: + devfilter = _HasTemplate(template, self.log) + self._notifier.notify_subdevices( + entity, uid, self._notifyAll, devfilter + ) + break + if isinstance(entity, Device): + self._notifyAll(entity) + break + entity = aq_parent(entity) + + @onDelete(Device) + def deviceDeleted(self, device, event): + with gc_cache_every(1000, db=self.dmd._p_jar._db): + devid = device.id + collector = device.getPerformanceServer().getId() + # The invalidation is only sent to the collector where the + # deleted device was. + if collector == self.instance: + self.log.debug( + "Invalidation: Performing remote call to delete " + "device %s from collector %s", + devid, + self.instance, + ) + for listener in self.listeners: + listener.callRemote("deleteDevice", devid) + else: + self.log.debug( + "Invalidation: Skipping remote call to delete " + "device %s from collector %s", + devid, + self.instance, + ) + + def _notifyAll(self, device): + """Notify all instances (daemons) of a change for the device.""" + # procrastinator schedules a call to _pushConfig + self._procrastinator.doLater(device) + + def _pushConfig(self, device): + """Push device config and deletes to relevent collectors/instances.""" + deferreds = [] + + if self._perfIdFilter(device) and self._filterDevice(device): + proxies = trapException(self, self._createDeviceProxies, device) + if proxies: + trapException(self, self._postCreateDeviceProxy, proxies) + else: + proxies = None + + prev_collector = ( + device.dmd.Monitors.primaryAq().getPreviousCollectorForDevice( + device.id + ) + ) + for listener in self.listeners: + if not proxies: + if hasattr(device, "getPerformanceServer"): + # The invalidation is only sent to the previous and + # current collectors. + if self.instance in ( + prev_collector, + device.getPerformanceServer().getId(), + ): + self.log.debug( + "Invalidation: Performing remote call for " + "device %s on collector %s", + device.id, + self.instance, + ) + deferreds.append( + listener.callRemote("deleteDevice", device.id) + ) + else: + self.log.debug( + "Invalidation: Skipping remote call for " + "device %s on collector %s", + device.id, + self.instance, + ) + else: + deferreds.append( + listener.callRemote("deleteDevice", device.id) + ) + self.log.debug( + "Invalidation: Performing remote call for " + "device %s on collector %s", + device.id, + self.instance, + ) + else: + options = self.listenerOptions.get(listener, None) + deviceFilter = getOptionsFilter(options) + for proxy in proxies: + if deviceFilter(proxy): + deferreds.append( + self._sendDeviceProxy(listener, proxy) + ) + + return defer.DeferredList(deferreds) + + def _sendDeviceProxy(self, listener, proxy): + return listener.callRemote("updateDeviceConfig", proxy) + + # FIXME: Don't use _getNotifiableClasses, use @onUpdate(myclasses) + def _getNotifiableClasses(self): + """ + Return a tuple of classes. + + When any object of a type in the sequence is modified the collector + connected to the service will be notified to update its configuration. + + @rtype: tuple + """ + return () + + def _pushReconfigure(self, value): + """Notify the collector to reread the entire configuration.""" + # value is unused but needed for the procrastinator framework + for listener in self.listeners: + listener.callRemote("notifyConfigChanged") + self._reconfigProcrastinator.clear() + + def _reconfigureIfNotify(self, object): + ncc = self._notifyConfigChange(object) + self.log.debug( + "services/config.py _reconfigureIfNotify object=%r " + "_notifyConfigChange=%s", + object, + ncc, + ) + if ncc: + self.log.debug("scheduling collector reconfigure") + self._reconfigProcrastinator.doLater(True) + + def _notifyConfigChange(self, object): + """ + Called when an object of a type from _getNotifiableClasses is + encountered + + @return: should a notify config changed be sent + @rtype: boolean + """ + return True + + def sendDeviceConfigs(self, configs): + deferreds = [] + + def errback(failure): + self.log.critical( + "Unable to update configs for service instance %s: %s", + self.name(), + failure, + ) + + for listener in self.listeners: + options = self.listenerOptions.get(listener, None) + deviceFilter = getOptionsFilter(options) + filteredConfigs = filter(deviceFilter, configs) + args = Zipper.dump(filteredConfigs) + d = listener.callRemote("updateDeviceConfigs", args).addErrback( + errback + ) + deferreds.append(d) + return deferreds + + +class _HasTemplate(object): + """ + Predicate class that checks whether a given device has a template + matching the given template. + """ + + def __init__(self, template, log): + self.template = template + self.log = log + + def __call__(self, device): + if issubclass(self.template.getTargetPythonClass(), Device): + if self.template in device.getRRDTemplates(): + self.log.debug( + "%s bound to template %s", + device.getPrimaryId(), + self.template.getPrimaryId(), + ) + return True + else: + self.log.debug( + "%s not bound to template %s", + device.getPrimaryId(), + self.template.getPrimaryId(), + ) + return False + else: + # check components, Too expensive? + for comp in device.getMonitoredComponents( + type=self.template.getTargetPythonClass().meta_type + ): + if self.template in comp.getRRDTemplates(): + self.log.debug( + "%s bound to template %s", + comp.getPrimaryId(), + self.template.getPrimaryId(), + ) + return True + else: + self.log.debug( + "%s not bound to template %s", + comp.getPrimaryId(), + self.template.getPrimaryId(), + ) + return False diff --git a/Products/ZenCollector/statistics.py b/Products/ZenCollector/statistics.py new file mode 100644 index 0000000000..89edbd794e --- /dev/null +++ b/Products/ZenCollector/statistics.py @@ -0,0 +1,72 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from zope.component.factory import Factory +from zope.interface import implementer + +from .interfaces import IStatistic, IStatisticsService + + +@implementer(IStatistic) +class Statistic(object): + def __init__(self, name, type, **kwargs): + self.value = 0 + self.name = name + self.type = type + self.kwargs = kwargs + + +@implementer(IStatisticsService) +class StatisticsService(object): + def __init__(self): + self._stats = {} + + def addStatistic(self, name, type, **kwargs): + if name in self._stats: + raise NameError("Statistic %s already exists" % name) + + if type not in ("DERIVE", "COUNTER", "GAUGE"): + raise TypeError("Statistic type %s not supported" % type) + + stat = Statistic(name, type, **kwargs) + self._stats[name] = stat + + def getStatistic(self, name): + return self._stats[name] + + def postStatistics(self, rrdStats): + for stat in self._stats.values(): + # figure out which function to use to post this statistical data + try: + func = { + "COUNTER": rrdStats.counter, + "GAUGE": rrdStats.gauge, + "DERIVE": rrdStats.derive, + }[stat.type] + except KeyError: + raise TypeError("Statistic type %s not supported" % stat.type) + + # These should always come back empty now because DaemonStats + # posts the events for us + func(stat.name, stat.value, **stat.kwargs) + + # counter is an ever-increasing value, but otherwise... + if stat.type != "COUNTER": + stat.value = 0 + + +class StatisticsServiceFactory(Factory): + """Factory for StatisticsService objects.""" + + def __init__(self): + super(StatisticsServiceFactory, self).__init__( + StatisticsService, + "StatisticsService", + "Creates a StatisticsService instance", + ) diff --git a/Products/ZenCollector/tasks.py b/Products/ZenCollector/tasks.py index 49ae3658b0..fa92c64aa3 100644 --- a/Products/ZenCollector/tasks.py +++ b/Products/ZenCollector/tasks.py @@ -15,21 +15,10 @@ import zope.component import zope.interface -from twisted.internet import defer - -from Products.ZenEvents.ZenEventClasses import Cmd_Fail, Error from Products.ZenUtils.observable import ObservableMixin from Products.ZenUtils.Utils import readable_time -from .interfaces import ( - ICollector, - ICollectorWorker, - IScheduledTask, - IScheduledTaskFactory, - ISubTaskSplitter, - ITaskSplitter, - IWorkerExecutor, -) +from .interfaces import IScheduledTaskFactory, ISubTaskSplitter, ITaskSplitter log = logging.getLogger("zen.collector.tasks") @@ -253,208 +242,6 @@ def sendEvent(self, event, **eventData): self.delegate.sendEvent(evt) -class WorkerOutputProxy(object): - def __init__(self, daemon=None, rrdWriter=None, eventSender=None): - self.daemon = daemon - self.rrdWriter = rrdWriter if not daemon else RRDWriter(daemon) - self.eventSender = eventSender if not daemon else EventSender(daemon) - - @defer.inlineCallbacks - def sendOutput(self, data, events, intervalSeconds): - if self.rrdWriter: - for d in data: - yield self.rrdWriter.writeRRD( - d["path"], - d["value"], - d["rrdType"], - rrdCommand=d["rrdCommand"], - cycleTime=intervalSeconds, - min=d["min"], - max=d["max"], - ) - - if self.eventSender: - for ev in events: - self.sendEvent(ev) - - @defer.inlineCallbacks - def sendEvent(self, event): - yield self.eventSender.sendEvent(event) - - -@zope.interface.implementer(IScheduledTask) -class SingleWorkerTask(ObservableMixin): - def __init__( - self, deviceId, taskName, scheduleIntervalSeconds, taskConfig - ): - """ - Construct a new task instance to fetch data from the configured - worker object. - - @param deviceId: the Zenoss deviceId to watch - @type deviceId: string - @param taskName: the unique identifier for this task - @type taskName: string - @param scheduleIntervalSeconds: the interval at which this task will be - collected - @type scheduleIntervalSeconds: int - @param taskConfig: the configuration for this task - """ - super(SingleWorkerTask, self).__init__() - - self.name = taskName - self.configId = deviceId - self.interval = scheduleIntervalSeconds - self.state = TaskStates.STATE_IDLE - - self._taskConfig = taskConfig - self._devId = deviceId - self._manageIp = self._taskConfig.manageIp - self._worker = None - - self.daemon = zope.component.getUtility(ICollector) - self.outputProxy = WorkerOutputProxy(self.daemon) - self.component = self.daemon.preferences.collectorName - - options = self.daemon.options - taskOptionDict = dict( - (attr, value) - for (attr, value) in options.__dict__.items() - if value is not None - and not attr.startswith("_") - and not callable(value) - ) - self._taskConfig.options = taskOptionDict - - @property - def deviceId(self): - return self._devId - - @property - def worker(self): - """ - Instance of the worker class to use for all tasks - """ - return self._worker - - @worker.setter - def worker(self, value): - self._worker = value - - @defer.inlineCallbacks - def cleanup(self): - """ - Delegate cleanup directly to the worker object - """ - try: - self.state = TaskStates.STATE_CLEANING - if self.worker: - yield self.worker.stop() - finally: - self.state = TaskStates.STATE_COMPLETED - - @defer.inlineCallbacks - def doTask(self): - """ - Delegate collection directly work to the worker object - """ - results = None - try: - self.state = TaskStates.STATE_RUNNING - if self.worker: - # perform data collection in the worker object - results = yield self.worker.collect( - self._devId, self._taskConfig - ) - - except Exception as ex: - log.error( - "worker collection: results (exception) = %r (%s)", results, ex - ) - collectionErrorEvent = { - "device": self.deviceId, - "severity": Error, - "eventClass": Cmd_Fail, - "summary": "Exception collecting:" + str(ex), - "component": self.component, - "agent": self.component, - } - yield self.outputProxy.sendEvent(collectionErrorEvent) - - else: - if results: - # send the data through the output proxy - data, events = results - if "testcounter" in self._taskConfig.options: - testCounter = self._taskConfig.options["testcounter"] - for dp in data: - if dp["counter"] == testCounter: - log.info( - "Collected value for %s: %s (%s)", - dp["counter"], - dp["value"], - dp["path"], - ) - break - else: - log.info( - "No value collected for %s from device %s", - testCounter, - self._devId, - ) - log.debug( - "Valid counters: %s", - [dp["counter"] for dp in data], - ) - - yield self.outputProxy.sendOutput(data, events, self.interval) - - finally: - self.state = TaskStates.STATE_IDLE - - -@zope.interface.implementer(IScheduledTaskFactory) -class SingleWorkerTaskFactory(SimpleTaskFactory): - """ - A task factory that creates a scheduled task using the provided - task class and the minimum attributes needed for a task, plus redirects - the 'doTask' and 'cleanup' methods to a single ICollectorWorker instance. - """ - - def __init__(self, taskClass=SingleWorkerTask, iCollectorWorker=None): - super(SingleWorkerTaskFactory, self).__init__(taskClass) - self.workerClass = iCollectorWorker - - def setWorkerClass(self, iCollectorWorker): - self.workerClass = iCollectorWorker - - def postInitialization(self): - pass - - def build(self): - task = super(SingleWorkerTaskFactory, self).build() - if self.workerClass and ICollectorWorker.implementedBy( - self.workerClass - ): - worker = self.workerClass() - worker.prepareToRun() - task.worker = worker - return task - - -@zope.interface.implementer(IWorkerExecutor) -class NullWorkerExecutor(object): - """ - IWorkerExecutor that does nothing with the provided worker - """ - - def setWorkerClass(self, workerClass): - pass - - def run(self): - pass - - class TaskStates(object): STATE_IDLE = "IDLE" STATE_CONNECTING = "CONNECTING" diff --git a/Products/ZenCollector/tests/testConfig.py b/Products/ZenCollector/tests/testConfig.py index 811364f468..d0bbfab5bb 100644 --- a/Products/ZenCollector/tests/testConfig.py +++ b/Products/ZenCollector/tests/testConfig.py @@ -13,11 +13,11 @@ from cryptography.fernet import Fernet from twisted.internet import defer -from Products.ZenCollector.config import ConfigurationProxy -from Products.ZenCollector.interfaces import ICollector, ICollectorPreferences - from Products.ZenTestCase.BaseTestCase import BaseTestCase +from ..config import ConfigurationProxy +from ..interfaces import ICollector, ICollectorPreferences + @zope.interface.implementer(ICollector) class MyCollector(object): @@ -31,9 +31,6 @@ def remote_getThresholdClasses(self): def remote_getCollectorThresholds(self): return defer.succeed(["yabba dabba do", "ho ho hum"]) - def remote_getDeviceConfigs(self, devices=[]): - return defer.succeed(["hmm", "foo", "bar"]) - def remote_getEncryptionKey(self): return defer.succeed(Fernet.generate_key()) @@ -44,11 +41,20 @@ def callRemote(self, methodName, *args, **kwargs): return self.remote_getThresholdClasses() elif methodName == "getCollectorThresholds": return self.remote_getCollectorThresholds() - elif methodName == "getDeviceConfigs": - return self.remote_getDeviceConfigs(args) elif methodName == "getEncryptionKey": return self.remote_getEncryptionKey() + class MyConfigCacheProxy(object): + def remote_getDeviceConfigs(self, devices=[]): + return defer.succeed(["hmm", "foo", "bar"]) + + def callRemote(self, methodName, *args, **kwargs): + if methodName == "getDeviceConfigs": + return self.remote_getDeviceConfigs(args) + + def getRemoteConfigCacheProxy(self): + return MyCollector.MyConfigCacheProxy() + def getRemoteConfigServiceProxy(self): return MyCollector.MyConfigServiceProxy() @@ -64,8 +70,10 @@ class Dummy(object): class MyPrefs(object): def __init__(self): self.collectorName = "testcollector" + self.configurationService = "MyConfigService" self.options = Dummy() self.options.monitor = "localhost" + self.options.workerid = 0 class TestConfig(BaseTestCase): @@ -78,9 +86,9 @@ def validate(result): self.assertEquals(result["foobar"], "abcxyz") return result - cfgService = ConfigurationProxy() prefs = MyPrefs() - d = cfgService.getPropertyItems(prefs) + cfgService = ConfigurationProxy(prefs) + d = cfgService.getPropertyItems() d.addBoth(validate) return d @@ -89,10 +97,10 @@ def validate(result): self.assertTrue("Products.ZenModel.FooBarThreshold" in result) return result - cfgService = ConfigurationProxy() prefs = MyPrefs() + cfgService = ConfigurationProxy(prefs) - d = cfgService.getThresholdClasses(prefs) + d = cfgService.getThresholdClasses() d.addBoth(validate) return d @@ -102,10 +110,10 @@ def validate(result): self.assertTrue("ho ho hum" in result) return result - cfgService = ConfigurationProxy() prefs = MyPrefs() + cfgService = ConfigurationProxy(prefs) - d = cfgService.getThresholds(prefs) + d = cfgService.getThresholds() d.addBoth(validate) return d @@ -115,15 +123,17 @@ def validate(result): self.assertFalse("abcdef" in result) return result - cfgService = ConfigurationProxy() prefs = MyPrefs() + cfgService = ConfigurationProxy(prefs) + token = 10 + deviceIds = [] - d = cfgService.getConfigProxies(prefs) + d = cfgService.getConfigProxies(token, deviceIds) d.addBoth(validate) - return d def testCrypt(self): - cfgService = ConfigurationProxy() + prefs = MyPrefs() + cfgService = ConfigurationProxy(prefs) s = "this is a string I wish to encrypt" diff --git a/Products/ZenCollector/tests/testFactory.py b/Products/ZenCollector/tests/testFactory.py index 4f9e6816be..77c12737ba 100644 --- a/Products/ZenCollector/tests/testFactory.py +++ b/Products/ZenCollector/tests/testFactory.py @@ -7,11 +7,12 @@ # ############################################################################## -from Products.ZenCollector import CoreCollectorFrameworkFactory -from Products.ZenCollector.config import ConfigurationProxy -from Products.ZenCollector.scheduler import Scheduler from Products.ZenTestCase.BaseTestCase import BaseTestCase +from ..frameworkfactory import CoreCollectorFrameworkFactory +from ..config import ConfigurationProxy +from ..scheduler import Scheduler + class TestFactory(BaseTestCase): def testFactoryInstall(self): diff --git a/Products/ZenCollector/tests/test_daemon.py b/Products/ZenCollector/tests/test_daemon.py index a6d6660e28..af8bfe42a4 100644 --- a/Products/ZenCollector/tests/test_daemon.py +++ b/Products/ZenCollector/tests/test_daemon.py @@ -13,7 +13,7 @@ ) -class TestCollectorDaemon_maintenanceCycle(TestCase): +class TestCollectorDaemon_maintenanceCallback(TestCase): def setUp(t): # Patch out the __init__ method, due to excessive side-effects t.init_patcher = patch.object( @@ -39,8 +39,8 @@ def setUp(t): t.cd.getDevicePingIssues = create_autospec(t.cd.getDevicePingIssues) t.cd._unresponsiveDevices = set() - def test__maintenanceCycle(t): - ret = t.cd._maintenanceCycle() + def test__maintenanceCallback(t): + ret = t.cd._maintenanceCallback() t.cd.log.debug.assert_called_with( "deviceIssues=%r", t.cd.getDevicePingIssues.return_value @@ -51,7 +51,7 @@ def test_ignores_unresponsive_devices(t): t.cd.log = Mock(name="log") t.cd._prefs.pauseUnreachableDevices = False - ret = t.cd._maintenanceCycle() + ret = t.cd._maintenanceCallback() t.assertEqual(ret.result, None) @@ -60,7 +60,7 @@ def test_no_cycle_option(t): t.cd._prefs.pauseUnreachableDevices = False t.cd.options.cycle = False - ret = t.cd._maintenanceCycle() + ret = t.cd._maintenanceCallback() t.assertEqual(ret.result, "No maintenance required") @@ -68,7 +68,7 @@ def test_handle_getDevicePingIssues_exception(t): t.cd.getDevicePingIssues.side_effect = Exception handler = _Capture() - ret = t.cd._maintenanceCycle() + ret = t.cd._maintenanceCallback() ret.addErrback(handler) t.assertIsInstance(handler.err, Failure) @@ -81,7 +81,7 @@ def test_handle__pauseUnreachableDevices_exception(t): t.cd._pauseUnreachableDevices.side_effect = Exception handler = _Capture() - ret = t.cd._maintenanceCycle() + ret = t.cd._maintenanceCallback() ret.addErrback(handler) t.assertIsInstance(handler.err, Failure) diff --git a/Products/ZenCollector/utils/maintenance.py b/Products/ZenCollector/utils/maintenance.py index 6867a2b3db..8430540241 100644 --- a/Products/ZenCollector/utils/maintenance.py +++ b/Products/ZenCollector/utils/maintenance.py @@ -14,6 +14,7 @@ from zenoss.protocols.protobufs.zep_pb2 import DaemonHeartbeat from zope.component import getUtility +from Products.ZenEvents.ZenEventClasses import Heartbeat from Products.ZenMessaging.queuemessaging.interfaces import IQueuePublisher log = logging.getLogger("zen.maintenance") @@ -28,8 +29,8 @@ def maintenanceBuildOptions(parser, defaultCycle=60): dest="maintenancecycle", default=defaultCycle, type="int", - help="Cycle, in seconds, for maintenance tasks." - " Default is %s." % defaultCycle, + help="Cycle, in seconds, for maintenance tasks " + "[default %s]" % defaultCycle, ) @@ -59,6 +60,24 @@ def heartbeat(self): ) +class ZenHubHeartbeatSender(object): + """ + Default heartbeat sender for CollectorDaemon. + """ + + def __init__(self, monitor, daemon, timeout, queue): + self.__event = { + "eventClass": Heartbeat, + "device": monitor, + "component": daemon, + "timeout": timeout + } + self.__queue = queue + + def heartbeat(self): + self.__queue.addHeartbeatEvent(self.__event) + + class MaintenanceCycle(object): def __init__( self, cycleInterval, heartbeatSender=None, maintenanceCallback=None @@ -72,7 +91,7 @@ def start(self): reactor.callWhenRunning(self._doMaintenance) def stop(self): - log.debug("Maintenance stopped") + log.debug("maintenance stopped") self._stop = True def _doMaintenance(self): @@ -82,20 +101,20 @@ def _doMaintenance(self): afterward will self-schedule each run. """ if self._stop: - log.debug("Skipping, maintenance stopped") + log.debug("skipping, maintenance stopped") return - log.info("Performing periodic maintenance") + log.info("performing periodic maintenance") interval = self._cycleInterval def _maintenance(): if self._heartbeatSender is not None: - log.debug("Calling heartbeat sender") + log.debug("calling heartbeat sender") d = defer.maybeDeferred(self._heartbeatSender.heartbeat) d.addCallback(self._additionalMaintenance) return d else: - log.debug("Skipping heartbeat: no sender configured") + log.debug("skipping heartbeat: no sender configured") return defer.maybeDeferred(self._additionalMaintenance) def _reschedule(result): @@ -103,12 +122,12 @@ def _reschedule(result): # The full error message is actually the entire traceback, so # just get the last line with the actual message. log.error( - "Maintenance failed. Message from hub: %s", - result.getErrorMessage(), + "maintenance failed. message from hub: (%s) %s", + result.type, result.getErrorMessage(), ) if interval > 0: - log.debug("Rescheduling maintenance in %ds", interval) + log.debug("rescheduling maintenance in %ds", interval) reactor.callLater(interval, self._doMaintenance) d = _maintenance() diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index 77d27f9e06..c48439153c 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -7,46 +7,23 @@ # ############################################################################## -"""PBDaemon - -Base for daemons that connect to zenhub. - -""" - import collections import os import sys -import time -import traceback -from functools import partial -from hashlib import sha1 from itertools import chain from urlparse import urlparse -import twisted.python.log - -from metrology import Metrology -from metrology.instruments import Gauge -from metrology.registry import registry -from twisted.cred import credentials -from twisted.internet import reactor, defer, task -from twisted.internet.error import ( - AlreadyCalled, - ConnectionLost, - ReactorNotRunning, -) -from twisted.python.failure import Failure +from twisted.cred.credentials import UsernamePassword +from twisted.internet.endpoints import clientFromString +from twisted.internet import defer, reactor, task +from twisted.internet.error import ReactorNotRunning from twisted.spread import pb -from ZODB.POSException import ConflictError -from zope.component import getUtilitiesFor -from zope.interface import implementer from Products.ZenEvents.ZenEventClasses import ( App_Start, App_Stop, Clear, - Heartbeat, Warning, ) from Products.ZenRRD.Thresholds import Thresholds @@ -59,99 +36,19 @@ MetricWriter, ThresholdNotifier, ) -from Products.ZenUtils.PBUtil import ReconnectingPBClientFactory -from Products.ZenUtils.Utils import zenPath, atomicWrite from Products.ZenUtils.ZenDaemon import ZenDaemon -from .interfaces import ( - ICollectorEventFingerprintGenerator, - ICollectorEventTransformer, - TRANSFORM_DROP, - TRANSFORM_STOP, -) +from .errors import HubDown, translateError +from .events import EventClient, EventQueueManager from .metricpublisher import publisher +from .zenhubclient import ZenHubClient + +PB_PORT = 8789 # field size limits for events DEFAULT_LIMIT = 524288 # 512k LIMITS = {"summary": 256, "message": 4096} - -class RemoteException(pb.Error, pb.Copyable, pb.RemoteCopy): - """Exception that can cross the PB barrier""" - - def __init__(self, msg, tb): - super(RemoteException, self).__init__(msg) - self.traceback = tb - - def getStateToCopy(self): - return { - "args": tuple(self.args), - "traceback": self.traceback, - } - - def setCopyableState(self, state): - self.args = state["args"] - self.traceback = state["traceback"] - - def __str__(self): - return "%s:%s" % ( - super(RemoteException, self).__str__(), - ("\n" + self.traceback) if self.traceback else " ", - ) - - -pb.setUnjellyableForClass(RemoteException, RemoteException) - - -# ZODB conflicts -class RemoteConflictError(RemoteException): - pass - - -pb.setUnjellyableForClass(RemoteConflictError, RemoteConflictError) - - -# Invalid monitor specified -class RemoteBadMonitor(RemoteException): - pass - - -pb.setUnjellyableForClass(RemoteBadMonitor, RemoteBadMonitor) - - -def translateError(callable): - """ - Decorator function to wrap remote exceptions into something - understandable by our daemon. - - @parameter callable: function to wrap - @type callable: function - @return: function's return or an exception - @rtype: various - """ - - def inner(*args, **kw): - """ - Interior decorator - """ - try: - return callable(*args, **kw) - except ConflictError as ex: - raise RemoteConflictError( - "Remote exception: %s: %s" % (ex.__class__, ex), - traceback.format_exc(), - ) - except Exception as ex: - raise RemoteException( - "Remote exception: %s: %s" % (ex.__class__, ex), - traceback.format_exc(), - ) - - return inner - - -PB_PORT = 8789 - startEvent = { "eventClass": App_Start, "summary": "started", @@ -164,7 +61,6 @@ def inner(*args, **kw): "severity": Warning, } - DEFAULT_HUB_HOST = "localhost" DEFAULT_HUB_PORT = PB_PORT DEFAULT_HUB_USERNAME = "admin" @@ -172,530 +68,100 @@ def inner(*args, **kw): DEFAULT_HUB_MONITOR = "localhost" -class HubDown(Exception): - pass - - class FakeRemote: def callRemote(self, *args, **kwargs): - ex = HubDown("ZenHub is down") - return defer.fail(ex) - - -@implementer(ICollectorEventFingerprintGenerator) -class DefaultFingerprintGenerator(object): - """Generates a fingerprint using a checksum of properties of the event.""" - - weight = 100 - - _IGNORE_FIELDS = ("rcvtime", "firstTime", "lastTime") - - def generate(self, event): - fields = [] - for k, v in sorted(event.iteritems()): - if k not in DefaultFingerprintGenerator._IGNORE_FIELDS: - if isinstance(v, unicode): - v = v.encode("utf-8") - else: - v = str(v) - fields.extend((k, v)) - return sha1("|".join(fields)).hexdigest() - - -def _load_utilities(utility_class): - """ - Loads ZCA utilities of the specified class. + return defer.fail(HubDown()) - @param utility_class: The type of utility to load. - @return: A list of utilities, sorted by their 'weight' attribute. - """ - utilities = (f for n, f in getUtilitiesFor(utility_class)) - return sorted(utilities, key=lambda f: getattr(f, "weight", 100)) +class PBDaemon(ZenDaemon, pb.Referenceable): + """Base class for services that connect to ZenHub.""" -class BaseEventQueue(object): - def __init__(self, maxlen): - self.maxlen = maxlen - - def append(self, event): - """ - Appends the event to the queue. - - @param event: The event. - @return: If the queue is full, this will return the oldest event - which was discarded when this event was added. - """ - raise NotImplementedError() + mname = name = "pbdaemon" - def popleft(self): - """ - Removes and returns the oldest event from the queue. If the queue - is empty, raises IndexError. + initialServices = ["EventService"] - @return: The oldest event from the queue. - @raise IndexError: If the queue is empty. - """ - raise NotImplementedError() + _customexitcode = 0 - def extendleft(self, events): - """ - Appends the events to the beginning of the queue (they will be the - first ones removed with calls to popleft). The list of events are - expected to be in order, with the earliest queued events listed - first. - - @param events: The events to add to the beginning of the queue. - @type events: list - @return A list of discarded events that didn't fit on the queue. - @rtype list - """ - raise NotImplementedError() + def __init__( + self, + noopts=0, + keeproot=False, + name=None, + publisher=None, + internal_publisher=None, + ): + # if we were provided our collector name via the constructor + # instead of via code, be sure to store it correctly. + if name is not None: + self.name = self.mname = name - def __len__(self): - """ - Returns the length of the queue. + super(PBDaemon, self).__init__(noopts, keeproot) - @return: The length of the queue. - """ - raise NotImplementedError() + # Configure/initialize the ZenHub client + self.__zhclient = _getZenHubClient(self, self.options) + self.__zhclient.notifyOnConnect(self._load_initial_services) + self.__zenhub_ready = None - def __iter__(self): - """ - Returns an iterator over the elements in the queue (oldest events - are returned first). - """ - raise NotImplementedError() - - -class DequeEventQueue(BaseEventQueue): - """ - Event queue implementation backed by a deque. This queue does not - perform de-duplication of events. - """ - - def __init__(self, maxlen): - super(DequeEventQueue, self).__init__(maxlen) - self.queue = collections.deque() - - def append(self, event): - # Make sure every processed event specifies the time it was queued. - if "rcvtime" not in event: - event["rcvtime"] = time.time() - - discarded = None - if len(self.queue) == self.maxlen: - discarded = self.popleft() - self.queue.append(event) - return discarded - - def popleft(self): - return self.queue.popleft() - - def extendleft(self, events): - if not events: - return events - available = self.maxlen - len(self.queue) - if not available: - return events - to_discard = 0 - if available < len(events): - to_discard = len(events) - available - self.queue.extendleft(reversed(events[to_discard:])) - return events[:to_discard] - - def __len__(self): - return len(self.queue) - - def __iter__(self): - return iter(self.queue) - - -class DeDupingEventQueue(BaseEventQueue): - """ - Event queue implementation backed by a OrderedDict. This queue performs - de-duplication of events (when an event with the same fingerprint is - seen, the 'count' field of the event is incremented by one instead of - sending an additional event). - """ - - def __init__(self, maxlen): - super(DeDupingEventQueue, self).__init__(maxlen) - self.default_fingerprinter = DefaultFingerprintGenerator() - self.fingerprinters = _load_utilities( - ICollectorEventFingerprintGenerator + self._thresholds = Thresholds() + self._threshold_notifier = ThresholdNotifier( + self.sendEvent, self._thresholds ) - self.queue = collections.OrderedDict() - def _event_fingerprint(self, event): - for fingerprinter in self.fingerprinters: - event_fingerprint = fingerprinter.generate(event) - if event_fingerprint is not None: - break - else: - event_fingerprint = self.default_fingerprinter.generate(event) - - return event_fingerprint - - def _first_time(self, event1, event2): - def first(evt): - return evt.get("firstTime", evt["rcvtime"]) - - return min(first(event1), first(event2)) - - def append(self, event): - # Make sure every processed event specifies the time it was queued. - if "rcvtime" not in event: - event["rcvtime"] = time.time() + self.rrdStats = DaemonStats() + self.lastStats = 0 + self.counters = collections.Counter() - fingerprint = self._event_fingerprint(event) - if fingerprint in self.queue: - # Remove the currently queued item - we will insert again which - # will move to the end. - current_event = self.queue.pop(fingerprint) - event["count"] = current_event.get("count", 1) + 1 - event["firstTime"] = self._first_time(current_event, event) - self.queue[fingerprint] = event - return + self.startEvent = startEvent.copy() + self.stopEvent = stopEvent.copy() + details = dict(component=self.name, device=self.options.monitor) + for evt in self.startEvent, self.stopEvent: + evt.update(details) - discarded = None - if len(self.queue) == self.maxlen: - discarded = self.popleft() + self._eventqueue = EventQueueManager(self.options, self.log) + self._metrologyReporter = None - self.queue[fingerprint] = event - return discarded + self.__publisher = publisher + self.__internal_publisher = internal_publisher + self.__metric_writer = None + self.__derivative_tracker = None - def popleft(self): - try: - return self.queue.popitem(last=False)[1] - except KeyError: - # Re-raise KeyError as IndexError for common interface across - # queues. - raise IndexError() - - def extendleft(self, events): - # Attempt to de-duplicate with events currently in queue - events_to_add = [] - for event in events: - fingerprint = self._event_fingerprint(event) - if fingerprint in self.queue: - current_event = self.queue[fingerprint] - current_event["count"] = current_event.get("count", 1) + 1 - current_event["firstTime"] = self._first_time( - current_event, event - ) - else: - events_to_add.append(event) - - if not events_to_add: - return events_to_add - available = self.maxlen - len(self.queue) - if not available: - return events_to_add - to_discard = 0 - if available < len(events_to_add): - to_discard = len(events_to_add) - available - old_queue, self.queue = self.queue, collections.OrderedDict() - for event in events_to_add[to_discard:]: - self.queue[self._event_fingerprint(event)] = event - for fingerprint, event in old_queue.iteritems(): - self.queue[fingerprint] = event - return events_to_add[:to_discard] - - def __len__(self): - return len(self.queue) - - def __iter__(self): - return self.queue.itervalues() - - -class EventQueueManager(object): - - CLEAR_FINGERPRINT_FIELDS = ( - "device", - "component", - "eventKey", - "eventClass", - ) - - def __init__(self, options, log): - self.options = options - self.transformers = _load_utilities(ICollectorEventTransformer) - self.log = log - self.discarded_events = 0 - # TODO: Do we want to limit the size of the clear event dictionary? - self.clear_events_count = {} - self._initQueues() - self._eventsSent = Metrology.meter("collectordaemon.eventsSent") - self._discardedEvents = Metrology.meter( - "collectordaemon.discardedEvent" - ) - self._eventTimer = Metrology.timer("collectordaemon.eventTimer") - metricNames = {x[0] for x in registry} - if "collectordaemon.eventQueue" not in metricNames: - queue = self - - class EventQueueGauge(Gauge): - @property - def value(self): - return queue.event_queue_length - - Metrology.gauge("collectordaemon.eventQueue", EventQueueGauge()) - - def _initQueues(self): - maxlen = self.options.maxqueuelen - queue_type = ( - DeDupingEventQueue - if self.options.deduplicate_events - else DequeEventQueue - ) - self.event_queue = queue_type(maxlen) - self.perf_event_queue = queue_type(maxlen) - self.heartbeat_event_queue = collections.deque(maxlen=1) - - def _transformEvent(self, event): - for transformer in self.transformers: - result = transformer.transform(event) - if result == TRANSFORM_DROP: - self.log.debug( - "Event dropped by transform %s: %s", transformer, event - ) - return None - if result == TRANSFORM_STOP: - break - return event - - def _clearFingerprint(self, event): - return tuple( - event.get(field, "") for field in self.CLEAR_FINGERPRINT_FIELDS + self.__eventclient = EventClient( + self.options, + self._eventqueue, + self.generateEvent, + lambda: self.getService("EventService"), ) - - def _removeDiscardedEventFromClearState(self, discarded): - # - # There is a particular condition that could cause clear events to - # never be sent until a collector restart. - # Consider the following sequence: - # - # 1) Clear event added to queue. This is the first clear event of - # this type and so it is added to the clear_events_count - # dictionary with a count of 1. - # 2) A large number of additional events are queued until maxqueuelen - # is reached, and so the queue starts to discard events including - # the clear event from #1. - # 3) The same clear event in #1 is sent again, however this time it - # is dropped because allowduplicateclears is False and the event - # has a > 0 count. - # - # To resolve this, we are careful to track all discarded events, and - # remove their state from the clear_events_count dictionary. - # - opts = self.options - if not opts.allowduplicateclears and opts.duplicateclearinterval == 0: - severity = discarded.get("severity", -1) - if severity == Clear: - clear_fingerprint = self._clearFingerprint(discarded) - if clear_fingerprint in self.clear_events_count: - self.clear_events_count[clear_fingerprint] -= 1 - - def _addEvent(self, queue, event): - if self._transformEvent(event) is None: - return - - allowduplicateclears = self.options.allowduplicateclears - duplicateclearinterval = self.options.duplicateclearinterval - if not allowduplicateclears or duplicateclearinterval > 0: - clear_fingerprint = self._clearFingerprint(event) - severity = event.get("severity", -1) - if severity != Clear: - # A non-clear event - clear out count if it exists - self.clear_events_count.pop(clear_fingerprint, None) - else: - current_count = self.clear_events_count.get( - clear_fingerprint, 0 - ) - self.clear_events_count[clear_fingerprint] = current_count + 1 - if not allowduplicateclears and current_count != 0: - self.log.debug( - "allowduplicateclears dropping clear event %r", event - ) - return - if ( - duplicateclearinterval > 0 - and current_count % duplicateclearinterval != 0 - ): - self.log.debug( - "duplicateclearinterval dropping clear event %r", event - ) - return - - discarded = queue.append(event) - self.log.debug( - "Queued event (total of %d) %r", len(self.event_queue), event + self.__recordQueuedEventsCountLoop = task.LoopingCall( + self.__record_queued_events_count ) - if discarded: - self.log.warn("Discarded event - queue overflow: %r", discarded) - self._removeDiscardedEventFromClearState(discarded) - self.discarded_events += 1 - self._discardedEvents.mark() - - def addEvent(self, event): - self._addEvent(self.event_queue, event) - - def addPerformanceEvent(self, event): - self._addEvent(self.perf_event_queue, event) - - def addHeartbeatEvent(self, heartbeat_event): - self.heartbeat_event_queue.append(heartbeat_event) - - @defer.inlineCallbacks - def sendEvents(self, event_sender_fn): - # Create new queues - we will flush the current queues and don't want - # to get in a loop sending events that are queued while we send this - # batch (the event sending is asynchronous). - prev_heartbeat_event_queue = self.heartbeat_event_queue - prev_perf_event_queue = self.perf_event_queue - prev_event_queue = self.event_queue - self._initQueues() - - perf_events = [] - events = [] - sent = 0 - try: - - def chunk_events(): - chunk_remaining = self.options.eventflushchunksize - heartbeat_events = [] - num_heartbeat_events = min( - chunk_remaining, len(prev_heartbeat_event_queue) - ) - for i in xrange(num_heartbeat_events): - heartbeat_events.append( - prev_heartbeat_event_queue.popleft() - ) - chunk_remaining -= num_heartbeat_events - - perf_events = [] - num_perf_events = min( - chunk_remaining, len(prev_perf_event_queue) - ) - for i in xrange(num_perf_events): - perf_events.append(prev_perf_event_queue.popleft()) - chunk_remaining -= num_perf_events - - events = [] - num_events = min(chunk_remaining, len(prev_event_queue)) - for i in xrange(num_events): - events.append(prev_event_queue.popleft()) - return heartbeat_events, perf_events, events - - heartbeat_events, perf_events, events = chunk_events() - while heartbeat_events or perf_events or events: - self.log.debug( - "Sending %d events, %d perf events, %d heartbeats", - len(events), - len(perf_events), - len(heartbeat_events), - ) - start = time.time() - yield event_sender_fn(heartbeat_events + perf_events + events) - duration = int((time.time() - start) * 1000) - self._eventTimer.update(duration) - sent += len(events) + len(perf_events) + len(heartbeat_events) - self._eventsSent.mark(len(events)) - self._eventsSent.mark(len(perf_events)) - self._eventsSent.mark(len(heartbeat_events)) - heartbeat_events, perf_events, events = chunk_events() - - defer.returnValue(sent) - except Exception: - # Restore performance events that failed to send - perf_events.extend(prev_perf_event_queue) - discarded_perf_events = self.perf_event_queue.extendleft( - perf_events - ) - self.discarded_events += len(discarded_perf_events) - self._discardedEvents.mark(len(discarded_perf_events)) - - # Restore events that failed to send - events.extend(prev_event_queue) - discarded_events = self.event_queue.extendleft(events) - self.discarded_events += len(discarded_events) - self._discardedEvents.mark(len(discarded_events)) - - # Remove any clear state for events that were discarded - for discarded in chain(discarded_perf_events, discarded_events): - self.log.debug( - "Discarded event - queue overflow: %r", discarded - ) - self._removeDiscardedEventFromClearState(discarded) - raise @property - def event_queue_length(self): - return ( - len(self.event_queue) - + len(self.perf_event_queue) - + len(self.heartbeat_event_queue) - ) + def services(self): + return self.__zhclient.services + def __record_queued_events_count(self): + if self.rrdStats.name: + self.rrdStats.gauge("eventQueueLength", len(self._eventqueue)) -class PBDaemon(ZenDaemon, pb.Referenceable): - - name = "pbdaemon" - initialServices = ["EventService"] - heartbeatEvent = {"eventClass": Heartbeat} - heartbeatTimeout = 60 * 3 - _customexitcode = 0 - _pushEventsDeferred = None - _eventHighWaterMark = None - _healthMonitorInterval = 30 - - def __init__(self, noopts=0, keeproot=False, name=None): - # if we were provided our collector name via the constructor instead of - # via code, be sure to store it correctly. - if name is not None: - self.name = name - self.mname = name - - try: - ZenDaemon.__init__(self, noopts, keeproot) - - except IOError: - import traceback - - self.log.critical(traceback.format_exc(0)) - sys.exit(1) - - self._thresholds = None - self._threshold_notifier = None - self.rrdStats = DaemonStats() - self.lastStats = 0 - self.perspective = None - self.services = {} - self.eventQueueManager = EventQueueManager(self.options, self.log) - self.startEvent = startEvent.copy() - self.stopEvent = stopEvent.copy() - details = dict(component=self.name, device=self.options.monitor) - for evt in self.startEvent, self.stopEvent, self.heartbeatEvent: - evt.update(details) - self.initialConnect = defer.Deferred() - self.stopped = False - self.counters = collections.Counter() - self._pingedZenhub = None - self._connectionTimeout = None - self._publisher = None - self._internal_publisher = None - self._metric_writer = None - self._derivative_tracker = None - self._metrologyReporter = None - # Add a shutdown trigger to send a stop event and flush the event queue - reactor.addSystemEventTrigger("before", "shutdown", self._stopPbDaemon) + def generateEvent(self, event, **kw): + """ + Return a 'filled out' version of the given event. + """ + eventCopy = {} + for k, v in chain(event.items(), kw.items()): + if isinstance(v, basestring): + # default max size is 512k + size = LIMITS.get(k, DEFAULT_LIMIT) + eventCopy[k] = v[0:size] if len(v) > size else v + else: + eventCopy[k] = v - # Set up a looping call to support the health check. - self.healthMonitor = task.LoopingCall(self._checkZenHub) - self.healthMonitor.start(self._healthMonitorInterval) + eventCopy["agent"] = self.name + eventCopy["monitor"] = self.options.monitor + return eventCopy def publisher(self): - if not self._publisher: + if not self.__publisher: host, port = urlparse(self.options.redisUrl).netloc.split(":") try: port = int(port) @@ -707,28 +173,31 @@ def publisher(self): publisher.defaultRedisPort, ) port = publisher.defaultRedisPort - self._publisher = publisher.RedisListPublisher( + self.__publisher = publisher.RedisListPublisher( host, port, self.options.metricBufferSize, channel=self.options.metricsChannel, maxOutstandingMetrics=self.options.maxOutstandingMetrics, ) - return self._publisher + return self.__publisher + + def setInternalPublisher(self, publisher): + self.__internal_publisher = publisher def internalPublisher(self): - if not self._internal_publisher: + if not self.__internal_publisher: url = os.environ.get("CONTROLPLANE_CONSUMER_URL", None) username = os.environ.get("CONTROLPLANE_CONSUMER_USERNAME", "") password = os.environ.get("CONTROLPLANE_CONSUMER_PASSWORD", "") if url: - self._internal_publisher = publisher.HttpPostPublisher( + self.__internal_publisher = publisher.HttpPostPublisher( username, password, url ) - return self._internal_publisher + return self.__internal_publisher def metricWriter(self): - if not self._metric_writer: + if not self.__metric_writer: publisher = self.publisher() metric_writer = MetricWriter(publisher) if os.environ.get("CONTROLPLANE", "0") == "1": @@ -741,239 +210,88 @@ def metricWriter(self): internal_metric_writer = FilteredMetricWriter( internal_publisher, internal_metric_filter ) - self._metric_writer = AggregateMetricWriter( + self.__metric_writer = AggregateMetricWriter( [metric_writer, internal_metric_writer] ) else: - self._metric_writer = metric_writer - return self._metric_writer + self.__metric_writer = metric_writer + return self.__metric_writer def derivativeTracker(self): - if not self._derivative_tracker: - self._derivative_tracker = DerivativeTracker() - return self._derivative_tracker - - def connecting(self): - """ - Called when about to connect to zenhub - """ - self.log.info("Attempting to connect to zenhub") - - def getZenhubInstanceId(self): - """ - Called after we connected to zenhub. - """ - - def callback(result): - self.log.info("Connected to the zenhub/%s instance", result) - - def errback(result): - self.log.info( - "Unexpected error appeared while getting zenhub " - "instance number %s", - result, - ) - - d = self.perspective.callRemote("getHubInstanceId") - d.addCallback(callback) - d.addErrback(errback) - return d - - def gotPerspective(self, perspective): - """ - This gets called every time we reconnect. - - @parameter perspective: Twisted perspective object - @type perspective: Twisted perspective object - """ - self.perspective = perspective - self.getZenhubInstanceId() - # Cancel the connection timeout timer as it's no longer needed. - if self._connectionTimeout: - try: - self._connectionTimeout.cancel() - except AlreadyCalled: - pass - self._connectionTimeout = None - d2 = self.getInitialServices() - if self.initialConnect: - self.log.debug("Chaining getInitialServices with d2") - self.initialConnect, d = None, self.initialConnect - d2.chainDeferred(d) - - def connect(self): - pingInterval = self.options.zhPingInterval - factory = ReconnectingPBClientFactory( - connectTimeout=60, - pingPerspective=self.options.pingPerspective, - pingInterval=pingInterval, - pingtimeout=pingInterval * 5, - ) - self.log.info( - "Connecting to %s:%d", self.options.hubhost, self.options.hubport - ) - factory.connectTCP(self.options.hubhost, self.options.hubport) - username = self.options.hubusername - password = self.options.hubpassword - self.log.debug("Logging in as %s", username) - c = credentials.UsernamePassword(username, password) - factory.gotPerspective = self.gotPerspective - factory.connecting = self.connecting - factory.setCredentials(c) - - def timeout(d): - if not d.called: - self.connectTimeout() - - self._connectionTimeout = reactor.callLater( - self.options.hubtimeout, timeout, self.initialConnect - ) - return self.initialConnect - - def connectTimeout(self): - self.log.error("Timeout connecting to zenhub: is it running?") - pass + if not self.__derivative_tracker: + self.__derivative_tracker = DerivativeTracker() + return self.__derivative_tracker def eventService(self): return self.getServiceNow("EventService") + def sendEvents(self, events): + return self.__eventclient.sendEvents(events) + + @defer.inlineCallbacks + def sendEvent(self, event, **kw): + yield self.__eventclient.sendEvent(event, **kw) + def getServiceNow(self, svcName): - if svcName not in self.services: + svc = self.__zhclient.services.get(svcName) + if svc is None: self.log.warning( - "No service named %r: ZenHub may be disconnected", svcName + "no service named %r: ZenHub may be disconnected", svcName ) - return self.services.get(svcName, None) or FakeRemote() + return svc or FakeRemote() - def getService(self, serviceName, serviceListeningInterface=None): + @defer.inlineCallbacks + def getService(self, name, serviceListeningInterface=None): """ - Attempt to get a service from zenhub. Returns a deferred. - When service is retrieved it is stashed in self.services with - serviceName as the key. When getService is called it will first - check self.services and if serviceName is already there it will return - the entry from self.services wrapped in a defer.succeed + Attempt to get a service from ZenHub. + + @rtype: Deferred """ - if serviceName in self.services: - return defer.succeed(self.services[serviceName]) - - def removeService(ignored): - self.log.debug("Removing service %s", serviceName) - if serviceName in self.services: - del self.services[serviceName] - - def callback(result, serviceName): - self.log.debug("Loaded service %s from zenhub", serviceName) - self.services[serviceName] = result - result.notifyOnDisconnect(removeService) - return result - - def errback(error, serviceName): - self.log.debug("errback after getting service %s", serviceName) - self.log.error("Could not retrieve service %s", serviceName) - if serviceName in self.services: - del self.services[serviceName] - return error - - d = self.perspective.callRemote( - "getService", - serviceName, + svc = yield self.__zhclient.get_service( + name, self.options.monitor, serviceListeningInterface or self, self.options.__dict__, ) - d.addCallback(callback, serviceName) - d.addErrback(errback, serviceName) - return d - - def getInitialServices(self): - """ - After connecting to zenhub, gather our initial list of services. - """ + defer.returnValue(svc) - def errback(error): - if isinstance(error, Failure): - self.log.critical( - "Invalid monitor: %s: %s", self.options.monitor, error - ) - reactor.stop() - return defer.fail( - RemoteBadMonitor( - "Invalid monitor: %s" % self.options.monitor, "" - ) - ) - return error - - self.log.debug( - "Setting up initial services: %s", ", ".join(self.initialServices) - ) - d = defer.DeferredList( - [self.getService(name) for name in self.initialServices], - fireOnOneErrback=True, - consumeErrors=True, - ) - d.addErrback(errback) - return d + def connect(self): + self.__zenhub_ready = self.__zhclient.start() + return self.__zenhub_ready def connected(self): - pass + """ + Invoked after a ZenHub connection is established and the + initial set of services have been loaded. - def _getThresholdNotifier(self): - if not self._threshold_notifier: - self._threshold_notifier = ThresholdNotifier( - self.sendEvent, self.getThresholds() - ) - return self._threshold_notifier + Sub-classes should override this method to add their own + functionality. + + @rtype: Deferred + """ def getThresholds(self): - if not self._thresholds: - self._thresholds = Thresholds() return self._thresholds def run(self): - def stopReporter(): - if self._metrologyReporter: - return self._metrologyReporter.stop() - - # Order of the shutdown triggers matter. Want to stop reporter first, - # calling self.metricWriter() below registers shutdown triggers for - # the actual metric http and redis publishers. - reactor.addSystemEventTrigger("before", "shutdown", stopReporter) + # Start the connection to zenhub + self.connect() - threshold_notifier = self._getThresholdNotifier() self.rrdStats.config( self.name, self.options.monitor, self.metricWriter(), - threshold_notifier, + self._threshold_notifier, self.derivativeTracker(), ) - self.log.debug("Starting PBDaemon initialization") - d = self.connect() - - def callback(result): - self.sendEvent(self.startEvent) - self.pushEventsLoop() - self.log.debug("Calling connected.") - self.connected() - return result - - def startStatsLoop(): - self.log.debug("Starting Statistic posting") - loop = task.LoopingCall(self.postStatistics) - loop.start(self.options.writeStatistics, now=False) - daemonTags = { - "zenoss_daemon": self.name, - "zenoss_monitor": self.options.monitor, - "internal": True, - } - self._metrologyReporter = TwistedMetricReporter( - self.options.writeStatistics, self.metricWriter(), daemonTags - ) - self._metrologyReporter.start() - if self.options.cycle: - reactor.callWhenRunning(startStatsLoop) - d.addCallback(callback) - d.addErrback(twisted.python.log.err) + reactor.addSystemEventTrigger( + "after", + "shutdown", + lambda: self.log.info("%s shutting down", self.name), + ) + + reactor.callWhenRunning(self._started) reactor.run() if self._customexitcode: sys.exit(self._customexitcode) @@ -986,201 +304,112 @@ def stop(self, ignored=""): try: reactor.stop() except ReactorNotRunning: - self.log.debug("Tried to stop reactor that was stopped") + self.log.debug("tried to stop reactor that was stopped") else: self.log.debug("stop() called when not running") - def _stopPbDaemon(self): - if self.stopped: - return - self.stopped = True - if "EventService" in self.services: - # send stop event if we don't have an implied --cycle, - # or if --cycle has been specified - if not hasattr(self.options, "cycle") or getattr( - self.options, "cycle", True - ): - self.sendEvent(self.stopEvent) - self.log.debug("Sent a 'stop' event") - if self._pushEventsDeferred: - self.log.debug("Currently sending events. Queueing next call") - d = self._pushEventsDeferred - # Schedule another call to flush any additional queued events - d.addBoth(lambda unused: self.pushEvents()) - else: - d = self.pushEvents() - return d - - self.log.debug("No event sent as no EventService available.") - - def sendEvents(self, events): - map(self.sendEvent, events) - - def sendEvent(self, event, **kw): - """Add event to queue of events to be sent. If we have an event - service then process the queue. - """ - generatedEvent = self.generateEvent(event, **kw) - self.eventQueueManager.addEvent(generatedEvent) - self.counters["eventCount"] += 1 - - if self._eventHighWaterMark: - return self._eventHighWaterMark - elif ( - self.eventQueueManager.event_queue_length - >= self.options.maxqueuelen * self.options.queueHighWaterMark - ): - return self.pushEvents() - else: - return defer.succeed(None) - - def generateEvent(self, event, **kw): - """Add event to queue of events to be sent. If we have an event - service then process the queue. - """ - if not reactor.running: - return - eventCopy = {} - for k, v in chain(event.items(), kw.items()): - if isinstance(v, basestring): - # default max size is 512k - size = LIMITS.get(k, DEFAULT_LIMIT) - eventCopy[k] = v[0:size] if len(v) > size else v - else: - eventCopy[k] = v - - eventCopy["agent"] = self.name - eventCopy["monitor"] = self.options.monitor - eventCopy["manager"] = self.fqdn - return eventCopy + _started_failures = { + "connect": "failed to connect to ZenHub", + "services": "failed to retrieve a service from ZenHub", + "eventclient": "failed to configure and start the event client", + "stats": "failed to configure and start statistics recording", + } @defer.inlineCallbacks - def pushEventsLoop(self): - """Periodially, wake up and flush events to ZenHub.""" - reactor.callLater(self.options.eventflushseconds, self.pushEventsLoop) - yield self.pushEvents() - - # Record the number of events in the queue up to every 2 seconds. - now = time.time() - if self.rrdStats.name and now >= (self.lastStats + 2): - self.lastStats = now - self.rrdStats.gauge( - "eventQueueLength", self.eventQueueManager.event_queue_length - ) + def _load_initial_services(self): + msg = self._started_failures["services"] + try: + for svcname in self.initialServices: + try: + yield self.getService(svcname) + except Exception as ex: + if self.options.cycle: + self.log.exception(msg) + else: + raise + else: + self.log.info("retrieved ZenHub service name=%s", svcname) + self.log.info("finished retrieving initial services") + except Exception as ex: + if self.options.cycle: + self.log.exception(msg) + else: + detail = ("%s %s" % (type(ex).__name__, ex)).strip() + self.log.critical("%s: %s", msg, detail) + self.stop() @defer.inlineCallbacks - def pushEvents(self): - """Flush events to ZenHub.""" - # are we already shutting down? - if not reactor.running: - self.log.debug("Skipping event sending - reactor not running.") - return - - if ( - self.eventQueueManager.event_queue_length - >= self.options.maxqueuelen * self.options.queueHighWaterMark - and not self._eventHighWaterMark - ): - self.log.debug( - "Queue length exceeded high water mark, %s ;" - "creating high water mark deferred", - self.eventQueueManager.event_queue_length, - ) - self._eventHighWaterMark = defer.Deferred() - - # are still connected to ZenHub? - evtSvc = self.services.get("EventService", None) - if not evtSvc: - self.log.error("No event service: %r", evtSvc) - yield task.deferLater(reactor, 0, lambda: None) - if self._eventHighWaterMark: - d, self._eventHighWaterMark = self._eventHighWaterMark, None - # not connected, release throttle and let things queue - d.callback("No Event Service") - defer.returnValue(None) - - if self._pushEventsDeferred: - self.log.debug("Skipping event sending - previous call active.") - defer.returnValue("Push Pending") - - sent = 0 + def _started(self): + # Called when the Twisted reactor is running. try: - # only set _pushEventsDeferred after we know we have - # an evtSvc/connectivity - self._pushEventsDeferred = defer.Deferred() - - def repush(val): - if ( - self.eventQueueManager.event_queue_length - >= self.options.eventflushchunksize - ): - self.pushEvents() - return val - - # conditionally push more events after this pushEvents - # call finishes - self._pushEventsDeferred.addCallback(repush) - - discarded_events = self.eventQueueManager.discarded_events - if discarded_events: - self.log.error( - "Discarded oldest %d events because maxqueuelen was " - "exceeded: %d/%d", - discarded_events, - discarded_events + self.options.maxqueuelen, - self.options.maxqueuelen, - ) - self.counters["discardedEvents"] += discarded_events - self.eventQueueManager.discarded_events = 0 + # Wait for the connection to zenhub + state = "connect" + self.log.info("waiting for zenhub") + ready, self.__zenhub_ready = self.__zenhub_ready, None + yield ready - send_events_fn = partial(evtSvc.callRemote, "sendEvents") - try: - sent = yield self.eventQueueManager.sendEvents(send_events_fn) - except ConnectionLost as ex: - self.log.error("Error sending event: %s", ex) - # let the reactor have time to clean up any connection - # errors and make callbacks - yield task.deferLater(reactor, 0, lambda: None) - except Exception as ex: - self.log.exception(ex) - # let the reactor have time to clean up any connection - # errors and make callbacks - yield task.deferLater(reactor, 0, lambda: None) - finally: - if self._pushEventsDeferred: - d, self._pushEventsDeferred = self._pushEventsDeferred, None - d.callback("sent %s" % sent) - if ( - self._eventHighWaterMark - and self.eventQueueManager.event_queue_length - < self.options.maxqueuelen * self.options.queueHighWaterMark - ): - self.log.debug( - "Queue restored to below high water mark: %s", - self.eventQueueManager.event_queue_length, - ) - d, self._eventHighWaterMark = self._eventHighWaterMark, None - d.callback("Queue length below high water mark") + if self.options.cycle: + state = "eventclient" + yield self._setup_event_client() - def heartbeat(self): - """if cycling, send a heartbeat, else, shutdown""" - if not self.options.cycle: + state = "stats" + yield self._setup_stats_recording() + + reactor.addSystemEventTrigger("before", "shutdown", self._stop) + + # Schedule the `connected` method to run + reactor.callLater(0, self.connected) + except Exception as ex: + msg = self._started_failures[state] + if self.options.cycle: + self.log.exception(msg) + else: + detail = ("%s %s" % (type(ex).__name__, ex)).strip() + self.log.critical("%s: %s", msg, detail) self.stop() - return - heartbeatEvent = self.generateEvent( - self.heartbeatEvent, timeout=self.heartbeatTimeout + + @defer.inlineCallbacks + def _stop(self): + if self.options.cycle: + self.__eventclient.sendEvent(self.stopEvent) + yield self.__eventclient.stop() + self.log.debug("stopped event client") + yield self.__zhclient.stop() + + def _setup_event_client(self): + self.__eventclient.start() + self.__recordQueuedEventsCountLoop.start(2.0, now=False) + self.__eventclient.sendEvent(self.startEvent) + self.log.debug("started event client") + + def _setup_stats_recording(self): + loop = task.LoopingCall(self.postStatistics) + loop.start(self.options.writeStatistics, now=False) + daemonTags = { + "zenoss_daemon": self.name, + "zenoss_monitor": self.options.monitor, + "internal": True, + } + self._metrologyReporter = TwistedMetricReporter( + self.options.writeStatistics, + self.metricWriter(), + daemonTags, ) - self.eventQueueManager.addHeartbeatEvent(heartbeatEvent) - # heartbeat is normally 3x cycle time - self.niceDoggie(self.heartbeatTimeout / 3) + self._metrologyReporter.start() + reactor.addSystemEventTrigger( + "before", "shutdown", self._metrologyReporter.stop + ) + self.log.debug("started statistics recording task") def postStatisticsImpl(self): pass def postStatistics(self): # save daemon counter stats - for name, value in self.counters.items(): - self.log.info("Counter %s, value %d", name, value) + for name, value in chain( + self.counters.items(), self.__eventclient.counters.items() + ): + self.log.debug("counter %s, value %d", name, value) self.rrdStats.counter(name, value) # persist counters values @@ -1204,96 +433,15 @@ def remote_setPropertyItems(self, items): def remote_updateThresholdClasses(self, classes): from Products.ZenUtils.Utils import importClass - self.log.debug("Loading classes %s", classes) for c in classes: try: importClass(c) + self.log.info("imported threshold class class=%r", c) except ImportError: - self.log.error("Unable to import class %s", c) - - def _checkZenHub(self): - """ - Check status of ZenHub (using ping method of service). - @return: if ping occurs, return deferred with result of ping attempt. - """ - self.log.debug("_checkZenHub: entry") - - def callback(result): - self.log.debug("ZenHub health check: Got result %s", result) - if result == "pong": - self.log.debug( - "ZenHub health check: " - "Success - received pong from ZenHub ping service." - ) - self._signalZenHubAnswering(True) - else: - self.log.error( - "ZenHub health check did not respond as expected." - ) - self._signalZenHubAnswering(False) - - def errback(error): - self.log.error( - "Error pinging ZenHub: %s (%s).", - error, - getattr(error, "message", ""), - ) - self._signalZenHubAnswering(False) - - try: - if self.perspective: - self.log.debug( - "ZenHub health check: " - "perspective found. attempting remote ping call." - ) - d = self.perspective.callRemote("ping") - d.addCallback(callback) - d.addErrback(errback) - return d - else: - self.log.debug("ZenHub health check: ZenHub may be down.") - self._signalZenHubAnswering(False) - except pb.DeadReferenceError: - self.log.warning( - "ZenHub health check: " - "DeadReferenceError - lost connection to ZenHub." - ) - self._signalZenHubAnswering(False) - except Exception as e: - self.log.error( - "ZenHub health check: caught %s exception: %s", - e.__class__, - e.message, - ) - self._signalZenHubAnswering(False) - - def _signalZenHubAnswering(self, answering): - """ - Write or remove file that the ZenHub_answering health check uses - to report status. - - @param answering: true if ZenHub is answering, False, otherwise. - """ - self.log.debug("_signalZenHubAnswering(%s)", answering) - filename = "zenhub_connected" - signalFilePath = zenPath("var", filename) - if answering: - self.log.debug("writing file at %s", signalFilePath) - atomicWrite(signalFilePath, "") - else: - try: - self.log.debug("removing file at %s", signalFilePath) - os.remove(signalFilePath) - except Exception as e: - self.log.debug( - "ignoring %s exception (%s) removing file %s", - e.__class__, - e.message, - signalFilePath, - ) + self.log.error("unable to import class %s", c) def buildOptions(self): - ZenDaemon.buildOptions(self) + super(PBDaemon, self).buildOptions() self.parser.add_option( "--hubhost", dest="hubhost", @@ -1439,3 +587,19 @@ def buildOptions(self): action="store_false", help="Enable or disable ping perspective", ) + + +def _getZenHubClient(app, options): + creds = UsernamePassword(options.hubusername, options.hubpassword) + endpointDescriptor = "tcp:{host}:{port}".format( + host=options.hubhost, port=options.hubport + ) + endpoint = clientFromString(reactor, endpointDescriptor) + return ZenHubClient( + reactor, + endpoint, + creds, + app, + options.hubtimeout, + options.zhPingInterval, + ) diff --git a/Products/ZenHub/configure.zcml b/Products/ZenHub/configure.zcml index 93e6a90254..a94114dd57 100644 --- a/Products/ZenHub/configure.zcml +++ b/Products/ZenHub/configure.zcml @@ -1,3 +1,4 @@ + - - + diff --git a/Products/ZenHub/errors.py b/Products/ZenHub/errors.py new file mode 100644 index 0000000000..5cf91e345e --- /dev/null +++ b/Products/ZenHub/errors.py @@ -0,0 +1,85 @@ +import traceback + +from twisted.spread import pb +from ZODB.POSException import ConflictError + + +class RemoteException(pb.Error, pb.Copyable, pb.RemoteCopy): + """Exception that can cross the PB barrier""" + + def __init__(self, msg, tb): + super(RemoteException, self).__init__(msg) + self.traceback = tb + + def getStateToCopy(self): + return { + "args": tuple(self.args), + "traceback": self.traceback, + } + + def setCopyableState(self, state): + self.args = state["args"] + self.traceback = state["traceback"] + + def __str__(self): + return "%s:%s" % ( + super(RemoteException, self).__str__(), + ("\n" + self.traceback) if self.traceback else " ", + ) + + +pb.setUnjellyableForClass(RemoteException, RemoteException) + + +# ZODB conflicts +class RemoteConflictError(RemoteException): + pass + + +pb.setUnjellyableForClass(RemoteConflictError, RemoteConflictError) + + +# Invalid monitor specified +class RemoteBadMonitor(RemoteException): + pass + + +pb.setUnjellyableForClass(RemoteBadMonitor, RemoteBadMonitor) + + +class HubDown(Exception): + """Raised when a connection to ZenHub is required but not available.""" + + def __init__(self, mesg="ZenHub is down"): + super(HubDown, self).__init__(mesg) + + +def translateError(callable): + """ + Decorator function to wrap remote exceptions into something + understandable by our daemon. + + @parameter callable: function to wrap + @type callable: function + @return: function's return or an exception + @rtype: various + """ + + def inner(*args, **kw): + """ + Interior decorator + """ + try: + return callable(*args, **kw) + except ConflictError as ex: + raise RemoteConflictError( + "Remote exception: %s: %s" % (ex.__class__, ex), + traceback.format_exc(), + ) + except Exception as ex: + raise RemoteException( + "Remote exception: %s: %s" % (ex.__class__, ex), + traceback.format_exc(), + ) + + return inner diff --git a/Products/ZenHub/events/__init__.py b/Products/ZenHub/events/__init__.py new file mode 100644 index 0000000000..bc956ee39b --- /dev/null +++ b/Products/ZenHub/events/__init__.py @@ -0,0 +1,13 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from .client import EventClient +from .queue.manager import EventQueueManager + +__all__ = ("EventClient", "EventQueueManager") diff --git a/Products/ZenHub/events/client.py b/Products/ZenHub/events/client.py new file mode 100644 index 0000000000..6865627db1 --- /dev/null +++ b/Products/ZenHub/events/client.py @@ -0,0 +1,149 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import collections +import logging + +from functools import partial + +from twisted.internet import defer, reactor, task + +from ..errors import HubDown + +log = logging.getLogger("zen.eventclient") + +# field size limits for events +DEFAULT_LIMIT = 524288 # 512k +LIMITS = {"summary": 256, "message": 4096} + + +class EventClient(object): + """ + Manages sending events to ZenHub's event service. + """ + + def __init__(self, options, queue, builder, servicefactory): + """ + Initialize an EventClient instance. + """ + self.__queue = queue + self.__builder = builder + self.__factory = servicefactory + + self.__flushinterval = options.eventflushseconds + self.__flushchunksize = options.eventflushchunksize + self.__maxqueuelength = options.maxqueuelen + self.__limit = options.maxqueuelen * options.queueHighWaterMark + + self.__task = task.LoopingCall(self._push) + self.__taskd = None + self.__pause = None + self.__pushing = False + self.__stopping = False + + self.counters = collections.Counter() + + def start(self): # type: () -> None + """Start the event client.""" + # Note: the __taskd deferred is called when __task is stopped + self.__taskd = self.__task.start(self.__flushinterval, now=False) + self.__taskd.addCallback(self._last_push) + + def stop(self): # type: () -> defer.Deferred + """Stop the event client.""" + self.__stopping = True + if self.__pause is None: + self.__pause = defer.Deferred() + self.__task.stop() + return self.__pause + + def sendEvents(self, events): # (Sequence[dict]) -> defer.DeferredList + return defer.DeferredList([self.sendEvent(event) for event in events]) + + @defer.inlineCallbacks + def sendEvent(self, event, **kw): + """ + Add event to queue of events to be sent. + If we have an event service then process the queue. + """ + if not reactor.running: + defer.returnValue(None) + + # If __pause is not None, yield it which blocks this + # method until the deferred is called and the yield returns. + if self.__pause: + yield self.__pause + + built_event = self.__builder(event, **kw) + self.__queue.addEvent(built_event) + self.counters["eventCount"] += 1 + + @defer.inlineCallbacks + def _last_push(self, task): + yield self._push() + + @defer.inlineCallbacks + def _push(self): + """ + Flush events to ZenHub. + """ + if len(self.__queue) >= self.__limit and not self.__pause: + log.debug( + "pause accepting new events; queue length at or " + "exceeds high water mark (%s >= %s)", + len(self.__queue), + self.__limit, + ) + self.__pause = defer.Deferred() + + if self.__pushing: + log.debug("skipping event sending - previous call active.") + defer.returnValue("push pending") + + try: + self.__pushing = True + + discarded_events = self.__queue.discarded_events + if discarded_events: + log.error( + "discarded oldest %d events because maxqueuelen was " + "exceeded: %d/%d", + discarded_events, + discarded_events + self.__maxqueuelength, + self.__maxqueuelength, + ) + self.counters["discardedEvents"] += discarded_events + self.__queue.discarded_events = 0 + + eventsvc = yield self.__factory() + send_events_fn = partial(eventsvc.callRemote, "sendEvents") + count = yield self.__queue.sendEvents(send_events_fn) + if count > 0: + log.debug("sent %d event%s", count, "s" if count > 1 else "") + except HubDown as ex: + log.warn("event service unavailable: %s", ex) + except Exception as ex: + log.exception("failed to send event: %s", ex) + # let the reactor have time to clean up any connection + # errors and make callbacks + yield task.deferLater(reactor, 0, lambda: None) + finally: + self.__pushing = False + if self.__pause and len(self.__queue) < self.__limit: + # Don't log the 'resume' message during a shutdown is + # confusing to avoid confusion. + if not self.__stopping: + log.debug( + "resume accepting new events; queue length below " + "high water mark (%s < %s)", + len(self.__queue), + self.__limit, + ) + pause, self.__pause = self.__pause, None + pause.callback("Queue length below high water mark") diff --git a/Products/ZenHub/events/queue/__init__.py b/Products/ZenHub/events/queue/__init__.py new file mode 100644 index 0000000000..8a09ea402e --- /dev/null +++ b/Products/ZenHub/events/queue/__init__.py @@ -0,0 +1,4 @@ + +from .manager import EventQueueManager + +__all__ = ("EventQueueManager",) diff --git a/Products/ZenHub/events/queue/base.py b/Products/ZenHub/events/queue/base.py new file mode 100644 index 0000000000..071104d822 --- /dev/null +++ b/Products/ZenHub/events/queue/base.py @@ -0,0 +1,62 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +class BaseEventQueue(object): + def __init__(self, maxlen): + self.maxlen = maxlen + + def append(self, event): + """ + Appends the event to the queue. + + @param event: The event. + @return: If the queue is full, this will return the oldest event + which was discarded when this event was added. + """ + raise NotImplementedError() + + def popleft(self): + """ + Removes and returns the oldest event from the queue. If the queue + is empty, raises IndexError. + + @return: The oldest event from the queue. + @raise IndexError: If the queue is empty. + """ + raise NotImplementedError() + + def extendleft(self, events): + """ + Appends the events to the beginning of the queue (they will be the + first ones removed with calls to popleft). The list of events are + expected to be in order, with the earliest queued events listed + first. + + @param events: The events to add to the beginning of the queue. + @type events: list + @return A list of discarded events that didn't fit on the queue. + @rtype list + """ + raise NotImplementedError() + + def __len__(self): + """ + Returns the length of the queue. + + @return: The length of the queue. + """ + raise NotImplementedError() + + def __iter__(self): + """ + Returns an iterator over the elements in the queue (oldest events + are returned first). + """ + raise NotImplementedError() diff --git a/Products/ZenHub/events/queue/deduping.py b/Products/ZenHub/events/queue/deduping.py new file mode 100644 index 0000000000..52fd3abc46 --- /dev/null +++ b/Products/ZenHub/events/queue/deduping.py @@ -0,0 +1,117 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import time + +from collections import OrderedDict + +from Products.ZenHub.interfaces import ICollectorEventFingerprintGenerator + +from .base import BaseEventQueue +from .fingerprint import DefaultFingerprintGenerator +from .misc import load_utilities + + +class DeDupingEventQueue(BaseEventQueue): + """ + Event queue implementation backed by a OrderedDict. This queue performs + de-duplication of events (when an event with the same fingerprint is + seen, the 'count' field of the event is incremented by one instead of + sending an additional event). + """ + + def __init__(self, maxlen): + super(DeDupingEventQueue, self).__init__(maxlen) + self.__fingerprinters = load_utilities( + ICollectorEventFingerprintGenerator + ) + if not self.__fingerprinters: + self.__fingerprinters = [DefaultFingerprintGenerator()] + self.__queue = OrderedDict() + + def append(self, event): + # Make sure every processed event specifies the time it was queued. + if "rcvtime" not in event: + event["rcvtime"] = time.time() + + fingerprint = self._fingerprint_event(event) + if fingerprint in self.__queue: + # Remove the currently queued item - we will insert again which + # will move to the end. + current_event = self.__queue.pop(fingerprint) + event["count"] = current_event.get("count", 1) + 1 + event["firstTime"] = self._first_time(current_event, event) + self.__queue[fingerprint] = event + return + + discarded = None + if len(self.__queue) == self.maxlen: + discarded = self.popleft() + + self.__queue[fingerprint] = event + return discarded + + def popleft(self): + try: + return self.__queue.popitem(last=False)[1] + except KeyError: + # Re-raise KeyError as IndexError for common interface across + # queues. + raise IndexError() + + def extendleft(self, events): + # Attempt to de-duplicate with events currently in queue + events_to_add = [] + for event in events: + fingerprint = self._fingerprint_event(event) + if fingerprint in self.__queue: + current_event = self.__queue[fingerprint] + current_event["count"] = current_event.get("count", 1) + 1 + current_event["firstTime"] = self._first_time( + current_event, event + ) + else: + events_to_add.append(event) + + if not events_to_add: + return events_to_add + available = self.maxlen - len(self.__queue) + if not available: + return events_to_add + to_discard = 0 + if available < len(events_to_add): + to_discard = len(events_to_add) - available + old_queue, self.__queue = self.__queue, OrderedDict() + for event in events_to_add[to_discard:]: + self.__queue[self._fingerprint_event(event)] = event + for fingerprint, event in old_queue.iteritems(): + self.__queue[fingerprint] = event + return events_to_add[:to_discard] + + def __contains__(self, event): + return self._fingerprint_event(event) in self.__queue + + def __len__(self): + return len(self.__queue) + + def __iter__(self): + return self.__queue.itervalues() + + def _fingerprint_event(self, event): + for fingerprinter in self.__fingerprinters: + fingerprint = fingerprinter.generate(event) + if fingerprint is not None: + break + return fingerprint + + def _first_time(self, event1, event2): + def first(evt): + return evt.get("firstTime", evt["rcvtime"]) + + return min(first(event1), first(event2)) diff --git a/Products/ZenHub/events/queue/deque.py b/Products/ZenHub/events/queue/deque.py new file mode 100644 index 0000000000..512e42bb40 --- /dev/null +++ b/Products/ZenHub/events/queue/deque.py @@ -0,0 +1,60 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import time + +from collections import deque + +from .base import BaseEventQueue + + +class DequeEventQueue(BaseEventQueue): + """ + Event queue implementation backed by a deque. This queue does not + perform de-duplication of events. + """ + + def __init__(self, maxlen): + super(DequeEventQueue, self).__init__(maxlen) + self.__queue = deque() + + def append(self, event): + # Make sure every processed event specifies the time it was queued. + if "rcvtime" not in event: + event["rcvtime"] = time.time() + + discarded = None + if len(self.__queue) == self.maxlen: + discarded = self.popleft() + self.__queue.append(event) + return discarded + + def popleft(self): + return self.__queue.popleft() + + def extendleft(self, events): + if not events: + return events + available = self.maxlen - len(self.__queue) + if not available: + return events + to_discard = 0 + if available < len(events): + to_discard = len(events) - available + self.__queue.extendleft(reversed(events[to_discard:])) + return events[:to_discard] + + def __contains__(self, event): + return event in self.__queue + + def __len__(self): + return len(self.__queue) + + def __iter__(self): + return iter(self.__queue) diff --git a/Products/ZenHub/events/queue/fingerprint.py b/Products/ZenHub/events/queue/fingerprint.py new file mode 100644 index 0000000000..b1000b79ac --- /dev/null +++ b/Products/ZenHub/events/queue/fingerprint.py @@ -0,0 +1,25 @@ +from hashlib import sha1 + +from zope.interface import implementer + +from Products.ZenHub.interfaces import ICollectorEventFingerprintGenerator + + +@implementer(ICollectorEventFingerprintGenerator) +class DefaultFingerprintGenerator(object): + """Generates a fingerprint using a checksum of properties of the event.""" + + weight = 100 + + _IGNORE_FIELDS = ("rcvtime", "firstTime", "lastTime") + + def generate(self, event): + fields = [] + for k, v in sorted(event.iteritems()): + if k not in DefaultFingerprintGenerator._IGNORE_FIELDS: + if isinstance(v, unicode): + v = v.encode("utf-8") + else: + v = str(v) + fields.extend((k, v)) + return sha1("|".join(fields)).hexdigest() diff --git a/Products/ZenHub/events/queue/manager.py b/Products/ZenHub/events/queue/manager.py new file mode 100644 index 0000000000..ec7d78d388 --- /dev/null +++ b/Products/ZenHub/events/queue/manager.py @@ -0,0 +1,255 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import time + +from collections import deque +from itertools import chain + +from metrology import Metrology +from metrology.instruments import Gauge +from metrology.registry import registry +from twisted.internet import defer + +from Products.ZenEvents.ZenEventClasses import Clear +from Products.ZenHub.interfaces import ( + ICollectorEventTransformer, + TRANSFORM_DROP, + TRANSFORM_STOP, +) + +from .misc import load_utilities +from .deduping import DeDupingEventQueue +from .deque import DequeEventQueue + + +class EventQueueManager(object): + + CLEAR_FINGERPRINT_FIELDS = ( + "device", + "component", + "eventKey", + "eventClass", + ) + + def __init__(self, options, log): + self.options = options + self.transformers = load_utilities(ICollectorEventTransformer) + self.log = log + self.discarded_events = 0 + # TODO: Do we want to limit the size of the clear event dictionary? + self.clear_events_count = {} + self._initQueues() + self._eventsSent = Metrology.meter("collectordaemon.eventsSent") + self._discardedEvents = Metrology.meter( + "collectordaemon.discardedEvent" + ) + self._eventTimer = Metrology.timer("collectordaemon.eventTimer") + metricNames = {x[0] for x in registry} + if "collectordaemon.eventQueue" not in metricNames: + queue = self + + class EventQueueGauge(Gauge): + @property + def value(self): + return len(queue) + + Metrology.gauge("collectordaemon.eventQueue", EventQueueGauge()) + + def __len__(self): + return ( + len(self.event_queue) + + len(self.perf_event_queue) + + len(self.heartbeat_event_queue) + ) + + def _initQueues(self): + maxlen = self.options.maxqueuelen + queue_type = ( + DeDupingEventQueue + if self.options.deduplicate_events + else DequeEventQueue + ) + self.event_queue = queue_type(maxlen) + self.perf_event_queue = queue_type(maxlen) + self.heartbeat_event_queue = deque(maxlen=1) + + def _transformEvent(self, event): + for transformer in self.transformers: + result = transformer.transform(event) + if result == TRANSFORM_DROP: + self.log.debug( + "event dropped by transform %s: %s", transformer, event + ) + return None + if result == TRANSFORM_STOP: + break + return event + + def _clearFingerprint(self, event): + return tuple( + event.get(field, "") for field in self.CLEAR_FINGERPRINT_FIELDS + ) + + def _removeDiscardedEventFromClearState(self, discarded): + # + # There is a particular condition that could cause clear events to + # never be sent until a collector restart. + # Consider the following sequence: + # + # 1) Clear event added to queue. This is the first clear event of + # this type and so it is added to the clear_events_count + # dictionary with a count of 1. + # 2) A large number of additional events are queued until maxqueuelen + # is reached, and so the queue starts to discard events including + # the clear event from #1. + # 3) The same clear event in #1 is sent again, however this time it + # is dropped because allowduplicateclears is False and the event + # has a > 0 count. + # + # To resolve this, we are careful to track all discarded events, and + # remove their state from the clear_events_count dictionary. + # + opts = self.options + if not opts.allowduplicateclears and opts.duplicateclearinterval == 0: + severity = discarded.get("severity", -1) + if severity == Clear: + clear_fingerprint = self._clearFingerprint(discarded) + if clear_fingerprint in self.clear_events_count: + self.clear_events_count[clear_fingerprint] -= 1 + + def _addEvent(self, queue, event): + if self._transformEvent(event) is None: + return + + allowduplicateclears = self.options.allowduplicateclears + duplicateclearinterval = self.options.duplicateclearinterval + if not allowduplicateclears or duplicateclearinterval > 0: + clear_fingerprint = self._clearFingerprint(event) + severity = event.get("severity", -1) + if severity != Clear: + # A non-clear event - clear out count if it exists + self.clear_events_count.pop(clear_fingerprint, None) + else: + current_count = self.clear_events_count.get( + clear_fingerprint, 0 + ) + self.clear_events_count[clear_fingerprint] = current_count + 1 + if not allowduplicateclears and current_count != 0: + self.log.debug( + "allowduplicateclears dropping clear event %r", event + ) + return + if ( + duplicateclearinterval > 0 + and current_count % duplicateclearinterval != 0 + ): + self.log.debug( + "duplicateclearinterval dropping clear event %r", event + ) + return + + discarded = queue.append(event) + self.log.debug("queued event (total of %d) %r", len(queue), event) + if discarded: + self.log.warn("discarded event - queue overflow: %r", discarded) + self._removeDiscardedEventFromClearState(discarded) + self.discarded_events += 1 + self._discardedEvents.mark() + + def addEvent(self, event): + self._addEvent(self.event_queue, event) + + def addPerformanceEvent(self, event): + self._addEvent(self.perf_event_queue, event) + + def addHeartbeatEvent(self, heartbeat_event): + self.heartbeat_event_queue.append(heartbeat_event) + + @defer.inlineCallbacks + def sendEvents(self, event_sender_fn): + # Create new queues - we will flush the current queues and don't want + # to get in a loop sending events that are queued while we send this + # batch (the event sending is asynchronous). + prev_heartbeat_event_queue = self.heartbeat_event_queue + prev_perf_event_queue = self.perf_event_queue + prev_event_queue = self.event_queue + self._initQueues() + + perf_events = [] + events = [] + sent = 0 + try: + def chunk_events(): + chunk_remaining = self.options.eventflushchunksize + heartbeat_events = [] + num_heartbeat_events = min( + chunk_remaining, len(prev_heartbeat_event_queue) + ) + for i in xrange(num_heartbeat_events): + heartbeat_events.append( + prev_heartbeat_event_queue.popleft() + ) + chunk_remaining -= num_heartbeat_events + + perf_events = [] + num_perf_events = min( + chunk_remaining, len(prev_perf_event_queue) + ) + for i in xrange(num_perf_events): + perf_events.append(prev_perf_event_queue.popleft()) + chunk_remaining -= num_perf_events + + events = [] + num_events = min(chunk_remaining, len(prev_event_queue)) + for i in xrange(num_events): + events.append(prev_event_queue.popleft()) + return heartbeat_events, perf_events, events + + heartbeat_events, perf_events, events = chunk_events() + while heartbeat_events or perf_events or events: + self.log.debug( + "sending %d events, %d perf events, %d heartbeats", + len(events), + len(perf_events), + len(heartbeat_events), + ) + start = time.time() + yield event_sender_fn(heartbeat_events + perf_events + events) + duration = int((time.time() - start) * 1000) + self._eventTimer.update(duration) + sent += len(events) + len(perf_events) + len(heartbeat_events) + self._eventsSent.mark(len(events)) + self._eventsSent.mark(len(perf_events)) + self._eventsSent.mark(len(heartbeat_events)) + heartbeat_events, perf_events, events = chunk_events() + + defer.returnValue(sent) + except Exception: + # Restore performance events that failed to send + perf_events.extend(prev_perf_event_queue) + discarded_perf_events = self.perf_event_queue.extendleft( + perf_events + ) + self.discarded_events += len(discarded_perf_events) + self._discardedEvents.mark(len(discarded_perf_events)) + + # Restore events that failed to send + events.extend(prev_event_queue) + discarded_events = self.event_queue.extendleft(events) + self.discarded_events += len(discarded_events) + self._discardedEvents.mark(len(discarded_events)) + + # Remove any clear state for events that were discarded + for discarded in chain(discarded_perf_events, discarded_events): + self.log.debug( + "discarded event - queue overflow: %r", discarded + ) + self._removeDiscardedEventFromClearState(discarded) + raise diff --git a/Products/ZenHub/events/queue/misc.py b/Products/ZenHub/events/queue/misc.py new file mode 100644 index 0000000000..21de0f07df --- /dev/null +++ b/Products/ZenHub/events/queue/misc.py @@ -0,0 +1,21 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from zope.component import getUtilitiesFor + + +def load_utilities(utility_class): + """ + Loads ZCA utilities of the specified class. + + @param utility_class: The type of utility to load. + @return: A list of utilities, sorted by their 'weight' attribute. + """ + utilities = (f for n, f in getUtilitiesFor(utility_class)) + return sorted(utilities, key=lambda f: getattr(f, "weight", 100)) diff --git a/Products/ZenHub/events/queue/tests/__init__.py b/Products/ZenHub/events/queue/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Products/ZenHub/events/queue/tests/test_base.py b/Products/ZenHub/events/queue/tests/test_base.py new file mode 100644 index 0000000000..7665e91e5d --- /dev/null +++ b/Products/ZenHub/events/queue/tests/test_base.py @@ -0,0 +1,32 @@ +from unittest import TestCase + +from ..base import BaseEventQueue + + +class BaseEventQueueTest(TestCase): + def setUp(t): + t.beq = BaseEventQueue(maxlen=5) + + def test_init(t): + base_event_queue = BaseEventQueue(maxlen=5) + t.assertEqual(base_event_queue.maxlen, 5) + + def test_append(t): + with t.assertRaises(NotImplementedError): + t.beq.append("event") + + def test_popleft(t): + with t.assertRaises(NotImplementedError): + t.beq.popleft() + + def test_extendleft(t): + with t.assertRaises(NotImplementedError): + t.beq.extendleft(["event_a", "event_b"]) + + def test___len__(t): + with t.assertRaises(NotImplementedError): + len(t.beq) + + def test___iter__(t): + with t.assertRaises(NotImplementedError): + [i for i in t.beq] diff --git a/Products/ZenHub/events/queue/tests/test_deduping.py b/Products/ZenHub/events/queue/tests/test_deduping.py new file mode 100644 index 0000000000..f95a8b3fa3 --- /dev/null +++ b/Products/ZenHub/events/queue/tests/test_deduping.py @@ -0,0 +1,234 @@ +from mock import Mock, patch +from unittest import TestCase + +from ..deduping import DeDupingEventQueue +from ..fingerprint import DefaultFingerprintGenerator + +PATH = {"src": "Products.ZenHub.events.queue.deduping"} + + +class DeDupingEventQueueTest(TestCase): + def setUp(t): + t.ddeq = DeDupingEventQueue(maxlen=10) + t.event_a, t.event_b = {"name": "event_a"}, {"name": "event_b"} + + @patch("{src}.load_utilities".format(**PATH)) + def test_init(t, load_utilities): + load_utilities.return_value = [] + ddeq = DeDupingEventQueue(maxlen=10) + t.assertEqual(ddeq.maxlen, 10) + + default = DefaultFingerprintGenerator() + expected = default.generate(t.event_a) + actual = ddeq._fingerprint_event(t.event_a) + + t.assertEqual(actual, expected) + + def test_fingerprint_event(t): + t.ddeq.fingerprinters = [] + + ret = t.ddeq._fingerprint_event(t.event_a) + expected = DefaultFingerprintGenerator().generate(t.event_a) + t.assertEqual(ret, expected) + + # Identical events generate the same fingerprint + event_2 = t.event_a.copy() + ret = t.ddeq._fingerprint_event(event_2) + t.assertEqual(ret, expected) + + @patch("{src}.load_utilities".format(**PATH)) + def test_fingerprint_event_fingerprinters_list(t, load_utilities): + """_fingerprint_event will attempt to generate a fingerprint from + each ICollectorEventFingerprintGenerator it loaded, + and return the first non-falsey value from them + """ + fp1 = Mock(spec_set=["generate"]) + fp1.generate.return_value = None + fp2 = Mock(spec_set=["generate"]) + fp2.generate.side_effect = lambda x: str(x) + # fp2 returns a value, so fp3 is never called + fp3 = Mock(spec_set=["generate"]) + fp3.generate.side_effect = lambda x: 1 / 0 + load_utilities.return_value = [fp1, fp2, fp3] + ddeq = DeDupingEventQueue(maxlen=10) + + ret = ddeq._fingerprint_event(t.event_a) + + fp1.generate.assert_called_with(t.event_a) + fp2.generate.assert_called_with(t.event_a) + fp3.generate.assert_not_called() + t.assertEqual(ret, str(t.event_a)) + + def test_first_time(t): + """given 2 events, retrun the earliest timestamp of the two + use 'firstTime' if available, else 'rcvtime' + """ + event1 = {"firstTime": 1, "rcvtime": 0} + event2 = {"rcvtime": 2} + + ret = t.ddeq._first_time(event1, event2) + t.assertEqual(ret, 1) + + event1 = {"firstTime": 3, "rcvtime": 1} + event2 = {"rcvtime": 2} + + ret = t.ddeq._first_time(event1, event2) + t.assertEqual(ret, 2) + + @patch("{src}.time".format(**PATH)) + def test_append_timestamp(t, time): + """Make sure every processed event specifies the time it was queued.""" + t.ddeq.append(t.event_a) + event = t.ddeq.popleft() + + t.assertEqual(event["rcvtime"], time.time.return_value) + + @patch("{src}.time".format(**PATH)) + def test_append_deduplication(t, time): + """The same event cannot be added to the queue twice + appending a duplicate event replaces the original + """ + event1 = {"data": "some data"} + event2 = {"data": "some data"} + t.assertEqual(event1, event2) + + t.ddeq.append(event1) + t.ddeq.append(event2) + + t.assertEqual(len(t.ddeq), 1) + + ret = t.ddeq.popleft() + # The new event replaces the old one + t.assertIs(ret, event2) + t.assertEqual(event2["count"], 2) + + @patch("{src}.time".format(**PATH)) + def test_append_deduplicates_and_counts_events(t, time): + time.time.side_effect = (t for t in range(100)) + t.ddeq.append({"name": "event_a"}) + t.assertEqual(list(t.ddeq), [{"rcvtime": 0, "name": "event_a"}]) + t.ddeq.append({"name": "event_a"}) + t.assertEqual( + list(t.ddeq), + [{"rcvtime": 1, "firstTime": 0, "count": 2, "name": "event_a"}], + ) + t.ddeq.append({"name": "event_a"}) + t.assertEqual( + list(t.ddeq), + [{"rcvtime": 2, "firstTime": 0, "count": 3, "name": "event_a"}], + ) + t.ddeq.append({"name": "event_a"}) + t.assertEqual( + list(t.ddeq), + [{"rcvtime": 3, "firstTime": 0, "count": 4, "name": "event_a"}], + ) + + def test_append_pops_and_returns_leftmost_if_full(t): + t.ddeq.maxlen = 1 + + t.ddeq.append(t.event_a) + ret = t.ddeq.append(t.event_b) + + # NOTE: events are stored in a dict, key=fingerprint + t.assertIn(t.event_b, t.ddeq) + t.assertNotIn(t.event_a, t.ddeq) + t.assertEqual(ret, t.event_a) + + def test_popleft(t): + t.ddeq.append(t.event_a) + t.ddeq.append(t.event_b) + + ret = t.ddeq.popleft() + + t.assertEqual(ret, t.event_a) + + def test_popleft_raises_IndexError(t): + """Raises IndexError instead of KeyError, for api compatability""" + with t.assertRaises(IndexError): + t.ddeq.popleft() + + @patch("{src}.time".format(**PATH)) + def test_extendleft(t, time): + """WARNING: extendleft does NOT add timestamps, as .append does + is this behavior is intentional? + """ + event_c = {"name": "event_c"} + t.ddeq.append(event_c) + t.assertEqual(list(t.ddeq), [event_c]) + events = [t.event_a, t.event_b] + + ret = t.ddeq.extendleft(events) + + t.assertEqual(ret, []) + t.assertEqual(list(t.ddeq), [t.event_a, t.event_b, event_c]) + """ + # to validate all events get timestamps + t.assertEqual( + list(t.ddeq), + [{'name': 'event_a', 'rcvtime': time.time.return_value}, + {'name': 'event_b', 'rcvtime': time.time.return_value}, + {'name': 'event_c', 'rcvtime': time.time.return_value}, + ] + ) + """ + + @patch("{src}.time".format(**PATH)) + def test_extendleft_counts_events_BUG(t, time): + time.time.side_effect = (t for t in range(100)) + t.ddeq.extendleft([{"name": "event_a"}, {"name": "event_b"}]) + t.assertEqual( + list(t.ddeq), + # This should work + # [{'rcvtime': 0, 'name': 'event_a'}] + # current behavior + [{"name": "event_a"}, {"name": "event_b"}], + ) + # rcvtime is required, but is not set by extendleft + with t.assertRaises(KeyError): + t.ddeq.extendleft([{"name": "event_a"}, {"name": "event_b"}]) + """ + Test Breaks Here due to missing rcvtime + t.assertEqual( + list(t.ddeq), + [{'rcvtime': 1, 'firstTime': 0, 'count': 2, 'name': 'event_a'}, + {'rcvtime': 1, 'firstTime': 0, 'count': 2, 'name': 'event_b'}] + ) + t.ddeq.extendleft([{'name': 'event_a'}, {'name': 'event_b'}]) + t.assertEqual( + list(t.ddeq), + [{'rcvtime': 2, 'firstTime': 0, 'count': 3, 'name': 'event_a'}, + {'rcvtime': 2, 'firstTime': 0, 'count': 3, 'name': 'event_b'}] + ) + t.ddeq.extendleft([{'name': 'event_a'}, {'name': 'event_b'}]) + t.assertEqual( + list(t.ddeq), + [{'rcvtime': 3, 'firstTime': 0, 'count': 4, 'name': 'event_a'}, + {'rcvtime': 3, 'firstTime': 0, 'count': 4, 'name': 'event_b'}] + ) + """ + + def test_extendleft_returns_events_if_empty(t): + ret = t.ddeq.extendleft([]) + t.assertEqual(ret, []) + + def test_extendleft_returns_extra_events_if_nearly_full(t): + t.ddeq.maxlen = 3 + t.ddeq.extendleft([t.event_a, t.event_b]) + event_c, event_d = {"name": "event_c"}, {"name": "event_d"} + events = [event_c, event_d] + + ret = t.ddeq.extendleft(events) + + t.assertEqual(list(t.ddeq), [event_d, t.event_a, t.event_b]) + t.assertEqual(ret, [event_c]) + + def test___len__(t): + ret = len(t.ddeq) + t.assertEqual(ret, 0) + t.ddeq.extendleft([t.event_a, t.event_b]) + t.assertEqual(len(t.ddeq), 2) + + def test___iter__(t): + t.ddeq.extendleft([t.event_a, t.event_b]) + ret = [event for event in t.ddeq] + t.assertEqual(ret, [t.event_a, t.event_b]) diff --git a/Products/ZenHub/events/queue/tests/test_deque.py b/Products/ZenHub/events/queue/tests/test_deque.py new file mode 100644 index 0000000000..593a94ac2f --- /dev/null +++ b/Products/ZenHub/events/queue/tests/test_deque.py @@ -0,0 +1,102 @@ +from mock import patch +from unittest import TestCase + +from ..deque import DequeEventQueue + +PATH = {"src": "Products.ZenHub.events.queue.deque"} + + +class DequeEventQueueTest(TestCase): + def setUp(t): + t.deq = DequeEventQueue(maxlen=10) + t.event_a, t.event_b = {"name": "event_a"}, {"name": "event_b"} + + def test_init(t): + maxlen = 100 + deq = DequeEventQueue(maxlen=maxlen) + t.assertEqual(deq.maxlen, maxlen) + + @patch("{src}.time".format(**PATH)) + def test_append(t, time): + event = {} + deq = DequeEventQueue(maxlen=10) + + ret = deq.append(event) + + # append sets the time the event was added to the queue + t.assertEqual(event["rcvtime"], time.time()) + t.assertEqual(ret, None) + + def test_append_pops_and_returns_leftmost_if_full(t): + event_a, event_b = {"name": "event_a"}, {"name": "event_b"} + deq = DequeEventQueue(maxlen=1) + + deq.append(event_a) + ret = deq.append(event_b) + + t.assertIn(event_b, deq) + t.assertNotIn(event_a, deq) + t.assertEqual(ret, event_a) + + @patch("{src}.time".format(**PATH)) + def test_popleft(t, time): + t.deq.append(t.event_a) + t.deq.append(t.event_b) + + ret = t.deq.popleft() + + t.assertEqual(ret, t.event_a) + + @patch("{src}.time".format(**PATH)) + def test_extendleft(t, time): + """WARNING: extendleft does NOT add timestamps, as .append does + is this behavior is intentional? + """ + event_c = {"name": "event_c"} + t.deq.append(event_c) + t.assertEqual(list(t.deq), [event_c]) + events = [t.event_a, t.event_b] + + ret = t.deq.extendleft(events) + + t.assertEqual(ret, []) + t.assertEqual(list(t.deq), [t.event_a, t.event_b, event_c]) + """ + # to validate all events get timestamps + t.assertEqual( + list(t.deq), + [{'name': 'event_a', 'rcvtime': time.time.return_value}, + {'name': 'event_b', 'rcvtime': time.time.return_value}, + {'name': 'event_c', 'rcvtime': time.time.return_value}, + ] + """ + + def test_extendleft_returns_events_if_falsey(t): + ret = t.deq.extendleft(False) + t.assertEqual(ret, False) + ret = t.deq.extendleft([]) + t.assertEqual(ret, []) + ret = t.deq.extendleft(0) + t.assertEqual(ret, 0) + + def test_extendleft_returns_extra_events_if_nearly_full(t): + t.deq.maxlen = 3 + t.deq.extendleft([t.event_a, t.event_b]) + event_c, event_d = {"name": "event_c"}, {"name": "event_d"} + events = [event_c, event_d] + + ret = t.deq.extendleft(events) + + t.assertEqual(list(t.deq), [event_d, t.event_a, t.event_b]) + t.assertEqual(ret, [event_c]) + + def test___len__(t): + ret = len(t.deq) + t.assertEqual(ret, 0) + t.deq.extendleft([t.event_a, t.event_b]) + t.assertEqual(len(t.deq), 2) + + def test___iter__(t): + t.deq.extendleft([t.event_a, t.event_b]) + ret = [event for event in t.deq] + t.assertEqual(ret, [t.event_a, t.event_b]) diff --git a/Products/ZenHub/events/queue/tests/test_fingerprint.py b/Products/ZenHub/events/queue/tests/test_fingerprint.py new file mode 100644 index 0000000000..f22ccaa8b2 --- /dev/null +++ b/Products/ZenHub/events/queue/tests/test_fingerprint.py @@ -0,0 +1,55 @@ +from unittest import TestCase + +from zope.interface.verify import verifyObject + +from ..fingerprint import ( + DefaultFingerprintGenerator, + ICollectorEventFingerprintGenerator, + sha1, +) + + +class DefaultFingerprintGeneratorTest(TestCase): + def test_init(t): + fingerprint_generator = DefaultFingerprintGenerator() + + # the class Implements the Interface + t.assertTrue( + ICollectorEventFingerprintGenerator.implementedBy( + DefaultFingerprintGenerator + ) + ) + # the object provides the interface + t.assertTrue( + ICollectorEventFingerprintGenerator.providedBy( + fingerprint_generator + ) + ) + # Verify the object implments the interface properly + verifyObject( + ICollectorEventFingerprintGenerator, fingerprint_generator + ) + + def test_generate(t): + """Takes an event, chews it up and spits out a sha1 hash + without an intermediate function that returns its internal fields list + we have to duplicate the entire function in test. + REFACTOR: split this up so we can test the fields list generator + and sha generator seperately. + Any method of generating the a hash from the dict should work so long + as its the same hash for the event with the _IGNORE_FILEDS stripped off + """ + event = {"k%s" % i: "v%s" % i for i in range(3)} + fields = [] + for k, v in sorted(event.iteritems()): + fields.extend((k, v)) + expected = sha1("|".join(fields)).hexdigest() + + # any keys listed in _IGNORE_FIELDS are not hashed + for key in DefaultFingerprintGenerator._IGNORE_FIELDS: + event[key] = "IGNORE ME!" + + fingerprint_generator = DefaultFingerprintGenerator() + out = fingerprint_generator.generate(event) + + t.assertEqual(out, expected) diff --git a/Products/ZenHub/events/queue/tests/test_manager.py b/Products/ZenHub/events/queue/tests/test_manager.py new file mode 100644 index 0000000000..394bb1a4e8 --- /dev/null +++ b/Products/ZenHub/events/queue/tests/test_manager.py @@ -0,0 +1,328 @@ +import collections + +from unittest import TestCase +from mock import MagicMock, Mock, create_autospec, call + +# Breaks Test Isolation. Products/ZenHub/metricpublisher/utils.py:15 +# ImportError: No module named eventlet +from Products.ZenHub.PBDaemon import Clear, defer +from ..deduping import DeDupingEventQueue +from ..manager import EventQueueManager, TRANSFORM_DROP, TRANSFORM_STOP + +PATH = {"src": "Products.ZenHub.PBDaemon"} + + +class EventQueueManagerTest(TestCase): + def setUp(t): + options = Mock( + name="options", + spec_set=[ + "maxqueuelen", + "deduplicate_events", + "allowduplicateclears", + "duplicateclearinterval", + "eventflushchunksize", + ], + ) + options.deduplicate_events = True + log = Mock(name="logger.log", spec_set=["debug", "warn"]) + + t.eqm = EventQueueManager(options, log) + t.eqm._initQueues() + + def test_initQueues(t): + options = Mock( + name="options", spec_set=["maxqueuelen", "deduplicate_events"] + ) + options.deduplicate_events = True + log = Mock(name="logger.log", spec_set=[]) + + eqm = EventQueueManager(options, log) + eqm._initQueues() + + t.assertIsInstance(eqm.event_queue, DeDupingEventQueue) + t.assertEqual(eqm.event_queue.maxlen, options.maxqueuelen) + t.assertIsInstance(eqm.perf_event_queue, DeDupingEventQueue) + t.assertEqual(eqm.perf_event_queue.maxlen, options.maxqueuelen) + t.assertIsInstance(eqm.heartbeat_event_queue, collections.deque) + t.assertEqual(eqm.heartbeat_event_queue.maxlen, 1) + + def test_transformEvent(t): + """a transformer mutates and returns an event""" + + def transform(event): + event["transformed"] = True + return event + + transformer = Mock(name="transformer", spec_set=["transform"]) + transformer.transform.side_effect = transform + t.eqm.transformers = [transformer] + + event = {} + ret = t.eqm._transformEvent(event) + + t.assertEqual(ret, event) + t.assertEqual(event, {"transformed": True}) + + def test_transformEvent_drop(t): + """if a transformer returns TRANSFORM_DROP + stop running the event through transformer, and return None + """ + + def transform_drop(event): + return TRANSFORM_DROP + + def transform_bomb(event): + 0 / 0 + + transformer = Mock(name="transformer", spec_set=["transform"]) + transformer.transform.side_effect = transform_drop + transformer_2 = Mock(name="transformer", spec_set=["transform"]) + transformer_2.transform.side_effect = transform_bomb + + t.eqm.transformers = [transformer, transformer_2] + + event = {} + ret = t.eqm._transformEvent(event) + t.assertEqual(ret, None) + + def test_transformEvent_stop(t): + """if a transformer returns TRANSFORM_STOP + stop running the event through transformers, and return the event + """ + + def transform_drop(event): + return TRANSFORM_STOP + + def transform_bomb(event): + 0 / 0 + + transformer = Mock(name="transformer", spec_set=["transform"]) + transformer.transform.side_effect = transform_drop + transformer_2 = Mock(name="transformer", spec_set=["transform"]) + transformer_2.transform.side_effect = transform_bomb + + t.eqm.transformers = [transformer, transformer_2] + + event = {} + ret = t.eqm._transformEvent(event) + t.assertIs(ret, event) + + def test_clearFingerprint(t): + event = {k: k + "_v" for k in t.eqm.CLEAR_FINGERPRINT_FIELDS} + + ret = t.eqm._clearFingerprint(event) + + t.assertEqual( + ret, ("device_v", "component_v", "eventKey_v", "eventClass_v") + ) + + def test__removeDiscardedEventFromClearState(t): + """if the event's fingerprint is in clear_events_count + decrement its value + """ + t.eqm.options.allowduplicateclears = False + t.eqm.options.duplicateclearinterval = 0 + + discarded = {"severity": Clear} + clear_fingerprint = t.eqm._clearFingerprint(discarded) + t.eqm.clear_events_count[clear_fingerprint] = 3 + + t.eqm._removeDiscardedEventFromClearState(discarded) + + t.assertEqual(t.eqm.clear_events_count[clear_fingerprint], 2) + + def test__addEvent(t): + """remove the event from clear_events_count + and append it to the queue + """ + t.eqm.options.allowduplicateclears = False + + queue = MagicMock(name="queue", spec_set=["append", "__len__"]) + event = {} + clear_fingerprint = t.eqm._clearFingerprint(event) + t.eqm.clear_events_count = {clear_fingerprint: 3} + + t.eqm._addEvent(queue, event) + + t.assertNotIn(clear_fingerprint, t.eqm.clear_events_count) + queue.append.assert_called_with(event) + + def test__addEvent_status_clear(t): + t.eqm.options.allowduplicateclears = False + t.eqm.options.duplicateclearinterval = 0 + + queue = MagicMock(name="queue", spec_set=["append", "__len__"]) + event = {"severity": Clear} + clear_fingerprint = t.eqm._clearFingerprint(event) + + t.eqm._addEvent(queue, event) + + t.assertEqual(t.eqm.clear_events_count[clear_fingerprint], 1) + queue.append.assert_called_with(event) + + def test__addEvent_drop_duplicate_clear_events(t): + t.eqm.options.allowduplicateclears = False + clear_count = 1 + + queue = MagicMock(name="queue", spec_set=["append", "__len__"]) + event = {"severity": Clear} + clear_fingerprint = t.eqm._clearFingerprint(event) + t.eqm.clear_events_count = {clear_fingerprint: clear_count} + + t.eqm._addEvent(queue, event) + + # non-clear events are not added to the clear_events_count dict + t.assertNotIn(t.eqm.clear_events_count, clear_fingerprint) + + queue.append.assert_not_called() + + def test__addEvent_drop_duplicate_clear_events_interval(t): + t.eqm.options.allowduplicateclears = False + clear_count = 3 + t.eqm.options.duplicateclearinterval = clear_count + + queue = MagicMock(name="queue", spec_set=["append", "__len__"]) + event = {"severity": Clear} + clear_fingerprint = t.eqm._clearFingerprint(event) + t.eqm.clear_events_count = {clear_fingerprint: clear_count} + + t.eqm._addEvent(queue, event) + + # non-clear events are not added to the clear_events_count dict + t.assertNotIn(t.eqm.clear_events_count, clear_fingerprint) + queue.append.assert_not_called() + + def test__addEvent_counts_discarded_events(t): + queue = MagicMock(name="queue", spec_set=["append", "__len__"]) + event = {} + discarded_event = {"name": "event"} + queue.append.return_value = discarded_event + + t.eqm._removeDiscardedEventFromClearState = create_autospec( + t.eqm._removeDiscardedEventFromClearState, + ) + t.eqm._discardedEvents.mark = create_autospec( + t.eqm._discardedEvents.mark + ) + + t.eqm._addEvent(queue, event) + + t.eqm._removeDiscardedEventFromClearState.assert_called_with( + discarded_event + ) + t.eqm._discardedEvents.mark.assert_called_with() + t.assertEqual(t.eqm.discarded_events, 1) + + def test_addEvent(t): + t.eqm._addEvent = create_autospec(t.eqm._addEvent) + event = {} + t.eqm.addEvent(event) + + t.eqm._addEvent.assert_called_with(t.eqm.event_queue, event) + + def test_addPerformanceEvent(t): + t.eqm._addEvent = create_autospec(t.eqm._addEvent) + event = {} + t.eqm.addPerformanceEvent(event) + + t.eqm._addEvent.assert_called_with(t.eqm.perf_event_queue, event) + + def test_addHeartbeatEvent(t): + heartbeat_event_queue = Mock(spec_set=t.eqm.heartbeat_event_queue) + t.eqm.heartbeat_event_queue = heartbeat_event_queue + heartbeat_event = {} + t.eqm.addHeartbeatEvent(heartbeat_event) + + heartbeat_event_queue.append.assert_called_with(heartbeat_event) + + def test_sendEvents(t): + """chunks events from EventManager's queues + yields them to the event_sender_fn + and returns a deffered with a result of events sent count + """ + t.eqm.options.eventflushchunksize = 3 + t.eqm.options.maxqueuelen = 5 + t.eqm._initQueues() + heartbeat_events = [{"heartbeat": i} for i in range(2)] + perf_events = [{"perf_event": i} for i in range(2)] + events = [{"event": i} for i in range(2)] + + t.eqm.heartbeat_event_queue.extendleft(heartbeat_events) + # heartbeat_event_queue set to static maxlen=1 + t.assertEqual(len(t.eqm.heartbeat_event_queue), 1) + t.eqm.perf_event_queue.extendleft(perf_events) + t.eqm.event_queue.extendleft(events) + + event_sender_fn = Mock(name="event_sender_fn") + + ret = t.eqm.sendEvents(event_sender_fn) + + # Priority: heartbeat, perf, event + event_sender_fn.assert_has_calls( + [ + call([heartbeat_events[1], perf_events[0], perf_events[1]]), + call([events[0], events[1]]), + ] + ) + t.assertIsInstance(ret, defer.Deferred) + t.assertEqual(ret.result, 5) + + def test_sendEvents_exception_handling(t): + """In case of exception, places events back in the queue, + and remove clear state for any discarded events + """ + t.eqm.options.eventflushchunksize = 3 + t.eqm.options.maxqueuelen = 5 + t.eqm._initQueues() + heartbeat_events = [{"heartbeat": i} for i in range(2)] + perf_events = [{"perf_event": i} for i in range(2)] + events = [{"event": i} for i in range(2)] + + t.eqm.heartbeat_event_queue.extendleft(heartbeat_events) + t.eqm.perf_event_queue.extendleft(perf_events) + t.eqm.event_queue.extendleft(events) + + def event_sender_fn(args): + raise Exception("event_sender_fn failed") + + ret = t.eqm.sendEvents(event_sender_fn) + # validate Exception was raised + t.assertEqual(ret.result.check(Exception), Exception) + # quash the unhandled error in defferd exception + ret.addErrback(Mock()) + + # Heartbeat events get dropped + t.assertNotIn(heartbeat_events[1], t.eqm.heartbeat_event_queue) + # events and perf_events are returned to the queues + t.assertIn(perf_events[0], t.eqm.perf_event_queue) + t.assertIn(events[0], t.eqm.event_queue) + + def test_sendEvents_exception_removes_clear_state_for_discarded(t): + t.eqm.options.eventflushchunksize = 3 + t.eqm.options.maxqueuelen = 2 + t.eqm._initQueues() + events = [{"event": i} for i in range(2)] + + t.eqm.event_queue.extendleft(events) + + def send(args): + t.eqm.event_queue.append({"new_event": 0}) + raise Exception("event_sender_fn failed") + + event_sender_fn = Mock(name="event_sender_fn", side_effect=send) + + t.eqm._removeDiscardedEventFromClearState = create_autospec( + t.eqm._removeDiscardedEventFromClearState, + name="_removeDiscardedEventFromClearState", + ) + + ret = t.eqm.sendEvents(event_sender_fn) + # validate Exception was raised + t.assertEqual(ret.result.check(Exception), Exception) + # quash the unhandled error in differd exception + ret.addErrback(Mock()) + + event_sender_fn.assert_called_with([events[0], events[1]]) + + t.eqm._removeDiscardedEventFromClearState.assert_called_with(events[0]) diff --git a/Products/ZenHub/events/queue/tests/test_misc.py b/Products/ZenHub/events/queue/tests/test_misc.py new file mode 100644 index 0000000000..4a5292a579 --- /dev/null +++ b/Products/ZenHub/events/queue/tests/test_misc.py @@ -0,0 +1,29 @@ +from mock import patch +from unittest import TestCase + +from ..misc import load_utilities + +PATH = {"src": "Products.ZenHub.events.queue.misc"} + + +class load_utilities_Test(TestCase): + @patch("{src}.getUtilitiesFor".format(**PATH), autospec=True) + def test_load_utilities(t, getUtilitiesFor): + ICollectorEventTransformer = "some transform function" + + def func1(): + pass + + def func2(): + pass + + func1.weight = 100 + func2.weight = 50 + getUtilitiesFor.return_value = (("func1", func1), ("func2", func2)) + + ret = load_utilities(ICollectorEventTransformer) + + getUtilitiesFor.assert_called_with(ICollectorEventTransformer) + # NOTE: lower weight comes first in the sorted list + # Is this intentional? + t.assertEqual(ret, [func2, func1]) diff --git a/Products/ZenHub/events/tests/__init__.py b/Products/ZenHub/events/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Products/ZenHub/hub.zcml b/Products/ZenHub/hub.zcml index f58deafd0e..414c9f4609 100644 --- a/Products/ZenHub/hub.zcml +++ b/Products/ZenHub/hub.zcml @@ -9,10 +9,7 @@ # ############################################################################## --> - - - + - - - - - - - - str + """ + Return ZenHub's Control Center instance ID. + + The value is a string containing a number or "unknown". + """ + return self.__instanceId + + @property + def services(self): + # type: () -> collections.Mapping[str, pb.Referenceable] + return _FrozenDictProxy(self.__services) + + def start(self): + # type: () -> defer.Deferred + """ + Start connecting to ZenHub. + + On a successful connection, the returned Deferred's callback is + invoked with the ZenHub broker instance. On failure, the errback + is invoked with the error. + + :rtype: defer.Deferred + """ + if self.__service.running: + log.warn("service already running service=%r", self.__service) + return + self.__service.startService() + log.debug("started client service service=%r", self.__service) + return self.__service.whenConnected() + + def stop(self): + # type: () -> defer.Deferred + """ + Stop connecting to ZenHub. + + When the connection is closed, the returned Deferred is called. + + :rtype: defer.Deferred + """ + self.__stopping = True + self._reset() + return self.__service.stopService() + + def notifyOnConnect(self, f): + self.__connected_callbacks.append(f) + + def notifyOnDisconnect(self, f): + self.__disconnected_callbacks.append(f) + + @defer.inlineCallbacks + def ping(self): + response = yield self.__zenhub.callRemote("ping") + defer.returnValue(response) + + @defer.inlineCallbacks + def register_worker(self, worker, instanceId, worklistId): + try: + yield self.__zenhub.callRemote( + "reportingForWork", + worker, + workerId=instanceId, + worklistId=worklistId, + ) + except pb.RemoteError as ex: + six.reraise(_remoteErrorType(ex), ex.args[0], tb=sys.exc_info()[2]) + + @defer.inlineCallbacks + def get_service(self, name, monitor, listener, options): + # type: (str, str, object, collections.Mapping) -> defer.Deferred + """ + Return a reference to the named ZenHub service. + + :param name: Name of the service + :param monitor: Name of the collector + :param listener: Object reference to caller + :param options: key/value data relevant to the service + """ + if name in self.__services: + defer.returnValue(self.__services[name]) + + if self.__zenhub is None: + raise HubDown("not connected to ZenHub") + + try: + service_ref = yield self.__zenhub.callRemote( + "getService", name, monitor, listener, options + ) + self.__services[name] = service_ref + log.debug( + "retrieved remote reference to ZenHub service " + "name=%s monitor=%s service=%r", + name, + monitor, + service_ref, + ) + defer.returnValue(service_ref) + except pb.RemoteError as ex: + six.reraise(_remoteErrorType(ex), ex.args[0], tb=sys.exc_info()[2]) + + @defer.inlineCallbacks + def _new_connection(self, broker): + log.debug("connected to ZenHub broker=%r", broker) + try: + if hasattr(broker.transport, "socket"): + setKeepAlive(broker.transport.socket) + else: + log.warn("broker.transport.socket attribute is missing") + + self.__zenhub = yield self._login(broker) + + ping = PingZenHub(self.__zenhub, self) + self.__pinger = task.LoopingCall(ping) + d = self.__pinger.start(self.__ping_interval, now=False) + d.addErrback(self._pingFail) # Catch and pass on errors + log.debug("started ZenHub pinger pinger=%r", self.__pinger) + + self.__instanceId = yield self.__zenhub.callRemote( + "getHubInstanceId" + ) + except defer.CancelledError: + log.error("timed out trying to login to ZenHub") + self._reset() + raise RuntimeError("reject connection") + except pb.RemoteError as ex: + log.error("login rejected by ZenHub: %s", _fromRemoteError(ex)) + self._reset() + raise RuntimeError("reject connection") + # defer.returnValue(None) + except Exception: + log.exception("unexpected error while logging into ZenHub") + self.__reactor.stop() + else: + log.debug("logged into ZenHub instance-id=%s", self.__instanceId) + try: + # Connection complete; install a listener to be notified if + # the connection is lost. + broker.notifyOnDisconnect(self._disconnected) + + log.debug( + "calling %d on-connect callbacks", + len(self.__connected_callbacks), + ) + for callback in self.__connected_callbacks: + try: + yield defer.maybeDeferred(callback) + except Exception: + log.exception( + "disconnect callback error callback=%r", callback + ) + except Exception: + log.exception("boom") + + def _login(self, broker): + d = broker.factory.login(self.__credentials, self.__app) + timeoutCall = self.__reactor.callLater(self.__timeout, d.cancel) + + def completedLogin(arg): + if timeoutCall.active(): + timeoutCall.cancel() + return arg + + d.addBoth(completedLogin) + return d + + def _disconnected(self, *args): + # Called when the connection to ZenHub is lost. + # Ensures that processing resumes when the connection to ZenHub + # is restored. + log.warn( + "disconnected from ZenHub%s", + ": %s" % (args[0],) if args else "", + ) + self._reset() + for callback in self.__disconnected_callbacks: + try: + callback(*args) + except Exception: + log.exception( + "disconnect callback error callback=%r", callback + ) + self.__disconnected_callbacks = [] + + def _reset(self): + self.__zenhub = None + self.__services = {} + if self.__pinger: + self.__pinger.stop() + self.__pinger = None + log.debug("stopped and removed ZenHub pinger") + + def _pingFail(self, ex): + log.error("pinger failed: %s", ex) + + +class PingZenHub(object): + """Simple task to ping ZenHub. + + PingZenHub's real purpose is to allow the ZenHubWorker to detect when + ZenHub is no longer responsive (for whatever reason). + """ + + def __init__(self, zenhub, client): + """Initialize a PingZenHub instance.""" + self.__zenhub = zenhub + self.__client = client + self.__log = log.getChild("ping") + + @defer.inlineCallbacks + def __call__(self): + # type: () -> defer.Deferred + """Ping zenhub. + + If the ping fails, causes the connection to ZenHub to reset. + """ + try: + response = yield self.__zenhub.callRemote("ping") + self.__log.debug("pinged zenhub: %s", response) + except Exception as ex: + self.__log.error("ping failed: %s", ex) + + +class _FrozenDictProxy(collections.Mapping): + def __init__(self, data): + self.__data = data + + def __getitem__(self, key): + return self.__data[key] + + def __contains__(self, key): + return key in self.__data + + def __len__(self): + return len(self.__data) + + def __iter__(self): + return iter(self.__data) + + +def _getBackoffPolicy(*args, **kw): + policy = backoffPolicy(*args, **kw) + + def _policy(attempt): + log.info( + "no connection to ZenHub; is ZenHub running? attempt=%s", attempt + ) + return policy(attempt) + + return _policy + + +def _remoteErrorType(ex): + modpath, clsname = ex.remoteType.rsplit(".", 1) + mod = importlib.import_module(modpath) + return getattr(mod, clsname) + + +def _fromRemoteError(ex): + return _remoteErrorType(ex)(*ex.args) diff --git a/Products/ZenHub/zenhubworker.py b/Products/ZenHub/zenhubworker.py index 3fa6f9d99d..29a9400b30 100755 --- a/Products/ZenHub/zenhubworker.py +++ b/Products/ZenHub/zenhubworker.py @@ -39,7 +39,7 @@ UnknownServiceError, ZenPBClientFactory, ) -from Products.ZenHub.PBDaemon import RemoteBadMonitor +from Products.ZenHub.errors import RemoteBadMonitor from Products.ZenUtils.debugtools import ContinuousProfiler from Products.ZenUtils.PBUtil import setKeepAlive from Products.ZenUtils.Time import isoDateTime diff --git a/Products/ZenModel/ThresholdInstance.py b/Products/ZenModel/ThresholdInstance.py index ee59fa1500..07e97fd32c 100644 --- a/Products/ZenModel/ThresholdInstance.py +++ b/Products/ZenModel/ThresholdInstance.py @@ -7,19 +7,16 @@ # ############################################################################## +import logging -import os +from twisted.spread import pb from Products.ZenModel.PerformanceConf import PerformanceConf from Products.ZenModel.MonitorClass import MonitorClass -from Products.ZenUtils.Utils import unused, rrd_daemon_args, rrd_daemon_retry - -from twisted.spread import pb - -import logging from Products.ZenUtils.deprecated import deprecated +from Products.ZenUtils.Utils import unused -log = logging.getLogger('zen.ThresholdInstance') +log = logging.getLogger("zen.ThresholdInstance") class ThresholdContext(pb.Copyable, pb.RemoteCopy): @@ -31,31 +28,35 @@ class ThresholdContext(pb.Copyable, pb.RemoteCopy): def __init__(self, context): self.metricMetaData = {} if isinstance(context, MonitorClass): - self.deviceName = "{context.id} hub".format(context=context) - self.componentName = '' - self.deviceUrl = 'zport/dmd/Monitors/Hub/{context.id}/viewHubPerformance'.format(context=context) - self.devicePath = 'Monitors/Hub/{context.id}'.format(context=context) - self._contextKey = '/'.join(('Daemons', context.id)) + self.deviceName = "{ctx.id} hub".format(ctx=context) + self.componentName = "" + self.deviceUrl = ( + "zport/dmd/Monitors/Hub/{ctx.id}/viewHubPerformance" + ).format(ctx=context) + self.devicePath = "Monitors/Hub/{ctx.id}".format(ctx=context) + self._contextKey = "/".join(("Daemons", context.id)) elif isinstance(context, PerformanceConf): - self.deviceName = "{context.id} collector".format(context=context) - self.componentName = '' - self.deviceUrl = 'zport/dmd/Monitors/Performance/{context.id}/viewDaemonPerformance'.format(context=context) - self.devicePath = 'Monitors/Performance/{context.id}'.format(context=context) - self._contextKey = '/'.join(('Daemons', context.id)) + self.deviceName = "{ctx.id} collector".format(ctx=context) + self.componentName = "" + self.deviceUrl = ( + "zport/dmd/Monitors/Performance/{ctx.id}/viewDaemonPerformance" + ).format(ctx=context) + self.devicePath = "Monitors/Performance/{ctx.id}".format( + ctx=context + ) + self._contextKey = "/".join(("Daemons", context.id)) else: self.deviceName = context.device().id self.componentName = context.id if self.componentName == self.deviceName: - self.componentName = '' + self.componentName = "" self._contextKey = context.getUUID() self.metricMetaData = context.getMetricMetadata() self._contextUid = context.getPrimaryId() - - def key(self): "Unique data that refers this context" return self.deviceName, self.componentName @@ -83,9 +84,9 @@ def fileKey(self, dataPoint): # return os.path.join(self.rrdPath, dataPoint) - pb.setUnjellyableForClass(ThresholdContext, ThresholdContext) + class ThresholdInstance(pb.Copyable, pb.RemoteCopy): """A ThresholdInstance is a threshold to be evaluated in a collector within a given context.""" @@ -106,7 +107,6 @@ def key(self): def dataPoints(self): "Returns the names of the datapoints used to compute the threshold" - def checkValue(self, dataPoint, timestamp, value): """ Check if the value violates the threshold. @@ -124,7 +124,6 @@ def check(self, dataPoints): returns events or an empty sequence""" raise NotImplementedError() - @deprecated def checkRaw(self, dataPoint, timeOf, value): """A new datapoint has been collected, use the given _raw_ @@ -133,8 +132,9 @@ def checkRaw(self, dataPoint, timeOf, value): """ raise NotImplementedError() - def getGraphElements(self, template, context, gopts, namespace, color, - legend, relatedGps): + def getGraphElements( + self, template, context, gopts, namespace, color, legend, relatedGps + ): """Produce a visual indication on the graph of where the threshold applies.""" unused(template, context, gopts, namespace, color, legend, relatedGps) @@ -143,16 +143,17 @@ def getGraphElements(self, template, context, gopts, namespace, color, pb.setUnjellyableForClass(ThresholdInstance, ThresholdInstance) + class RRDThresholdInstance(ThresholdInstance): """ Deprecated """ + pb.setUnjellyableForClass(RRDThresholdInstance, RRDThresholdInstance) class MetricThresholdInstance(ThresholdInstance): - def __init__(self, id, context, dpNames, eventClass, severity): self._context = context self.id = id @@ -172,7 +173,6 @@ def dataPoints(self): "Returns the names of the datapoints used to compute the threshold" return self.dataPointNames - def checkValue(self, dataPoint, timestamp, value): return self._checkImpl(dataPoint, value) @@ -187,4 +187,5 @@ def _checkImpl(self, dataPoint, value): """ raise NotImplementedError() + pb.setUnjellyableForClass(MetricThresholdInstance, MetricThresholdInstance) diff --git a/Products/ZenModel/ValueChangeThreshold.py b/Products/ZenModel/ValueChangeThreshold.py index 10e91a2436..75ee6b79fe 100644 --- a/Products/ZenModel/ValueChangeThreshold.py +++ b/Products/ZenModel/ValueChangeThreshold.py @@ -7,52 +7,56 @@ # ############################################################################## - -from Products.ZenModel.ThresholdInstance import MetricThresholdInstance - -__doc__= """Threshold to track when a value changes. -""" +import logging from AccessControl.class_init import InitializeClass -from ThresholdClass import ThresholdClass -from ThresholdInstance import ThresholdContext +from twisted.spread import pb from zenoss.protocols.protobufs.zep_pb2 import SEVERITY_INFO + from Products.ZenEvents.ZenEventClasses import Status_Perf from Products.ZenUtils import Map -import logging -log = logging.getLogger('zen.MinMaxCheck') +from .ThresholdClass import ThresholdClass +from .ThresholdInstance import MetricThresholdInstance, ThresholdContext +log = logging.getLogger("zen.MinMaxCheck") + +NaN = float("nan") -NaN = float('nan') class ValueChangeThreshold(ThresholdClass): """ - Threshold that can watch changes in a value + Threshold that can watch changes in a value. """ eventClass = Status_Perf severity = SEVERITY_INFO def createThresholdInstance(self, context): - """Return the config used by the collector to process change thresholds """ - mmt = ValueChangeThresholdInstance(self.id, - ThresholdContext(context), - self.dsnames, - eventClass=self.eventClass, - severity=self.severity) + Return the config used by the collector to process change thresholds. + """ + mmt = ValueChangeThresholdInstance( + self.id, + ThresholdContext(context), + self.dsnames, + eventClass=self.eventClass, + severity=self.severity, + ) return mmt + InitializeClass(ValueChangeThreshold) ValueChangeThresholdClass = ValueChangeThreshold + class ValueChangeThresholdInstance(MetricThresholdInstance): """ - Threshold that emits an event when a value changes from its previous value. Does not send clear events. + Threshold that emits an event when a value changes from its + previous value. Does not send clear events. """ - lastValues = Map.Locked(Map.Timed({}, 60*60*24)) # 24-hour timeout + lastValues = Map.Locked(Map.Timed({}, 60 * 60 * 24)) # 24-hour timeout def _checkImpl(self, dataPoint, value): dpKey = self._getDpKey(dataPoint) @@ -62,7 +66,7 @@ def _checkImpl(self, dataPoint, value): # Update the value in the map. ValueChangeThresholdInstance.lastValues[dpKey] = value # .. Only create a change event if this isn't the first collection - if lastValue != None: + if lastValue is not None: event = dict( device=self.context().deviceName, summary="Value changed from %s to %s" % (lastValue, value), @@ -71,16 +75,19 @@ def _checkImpl(self, dataPoint, value): component=self.context().componentName, current=value, previous=lastValue, - severity=self.severity) + severity=self.severity, + ) return (event,) return tuple() def _getDpKey(self, dp): - return ':'.join(self.context().key()) + ':' + dp + return ":".join(self.context().key()) + ":" + dp def getGraphValues(self, relatedGps, context): # currently, no visualization implemented for this threshold type return () -from twisted.spread import pb -pb.setUnjellyableForClass(ValueChangeThresholdInstance, ValueChangeThresholdInstance) + +pb.setUnjellyableForClass( + ValueChangeThresholdInstance, ValueChangeThresholdInstance +) diff --git a/Products/ZenModel/migrate/addConfigCacheProperties.py b/Products/ZenModel/migrate/addConfigCacheProperties.py new file mode 100644 index 0000000000..aa8bd71d62 --- /dev/null +++ b/Products/ZenModel/migrate/addConfigCacheProperties.py @@ -0,0 +1,70 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from Products.ZenRelations.zPropertyCategory import setzPropertyCategory + +from . import Migrate +from Products.ZenModel.ZMigrateVersion import SCHEMA_MAJOR, SCHEMA_MINOR, SCHEMA_REVISION + + +_properties = ( + ( + ("zDeviceConfigTTL", 43200), + { + "type": "int", + "label": "Device configuration expiration", + "description": ( + "The number of seconds to wait before rebuilding a " + "device configuration." + ), + } + ), + ( + ("zDeviceConfigBuildTimeout", 7200), + { + "type": "int", + "label": "Device configuration build timeout", + "description": ( + "The number of seconds allowed for building a device " + "configuration." + ), + } + ), + ( + ("zDeviceConfigPendingTimeout", 7200), + { + "type": "int", + "label": "Device configuration build queued timeout", + "description": ( + "The number of seconds a device configuration build may be " + "queued before a timeout." + ), + } + ), +) + + +class addConfigCacheProperties(Migrate.Step): + """ + Add the zDeviceConfigTTL, zDeviceConfigBuildTimeout, and + zDeviceConfigPendingTimeout z-properties to /Devices. + """ + + version = Migrate.Version(SCHEMA_MAJOR, SCHEMA_MINOR, SCHEMA_REVISION) + + def cutover(self, dmd): + for args, kwargs in _properties: + if not dmd.Devices.hasProperty(args[0]): + dmd.Devices._setProperty(*args, **kwargs) + setzPropertyCategory(args[0], "Config Cache") + + +addConfigCacheProperties() diff --git a/Products/ZenRRD/RRDDaemon.py b/Products/ZenRRD/RRDDaemon.py index 195b28ddb8..0a21204ca4 100644 --- a/Products/ZenRRD/RRDDaemon.py +++ b/Products/ZenRRD/RRDDaemon.py @@ -18,7 +18,6 @@ from Products.ZenEvents import Event from Products.ZenHub.PBDaemon import FakeRemote, PBDaemon -from Products.ZenUtils.Utils import unused from .Thresholds import Thresholds @@ -52,8 +51,8 @@ def __init__(self, name, noopts=False): @param noopts: process command-line arguments? @type noopts: boolean """ + super(RRDDaemon, self).__init__(noopts, name=name) self.events = [] - PBDaemon.__init__(self, noopts, name=name) self.thresholds = Thresholds() def getDevicePingIssues(self): @@ -75,16 +74,6 @@ def remote_setPropertyItems(self, items): self.log.debug("Async update of collection properties") self.setPropertyItems(items) - def remote_updateDeviceList(self, devices): - """ - Callable from zenhub. - - @param devices: list of devices (unused) - @type devices: list - """ - unused(devices) - self.log.debug("Async update of device list") - def setPropertyItems(self, items): """ Set zProperties @@ -113,7 +102,7 @@ def buildOptions(self): """ Command-line options to add """ - PBDaemon.buildOptions(self) + super(RRDDaemon, self).buildOptions() self.parser.add_option( "-d", "--device", diff --git a/Products/ZenRelations/ZenPropertyManager.py b/Products/ZenRelations/ZenPropertyManager.py index 1f88de36bf..8233511fec 100644 --- a/Products/ZenRelations/ZenPropertyManager.py +++ b/Products/ZenRelations/ZenPropertyManager.py @@ -44,6 +44,30 @@ # define all the zProperties. The values are set on dmd.Devices in the # buildDeviceTreeProperties of DeviceClass Z_PROPERTIES = [ + # Config Cache properties + ( + "zDeviceConfigBuildTimeout", + 7200, + "int", + "Device configuration build timeout", + "The number of seconds before timing out a device configuration build." + ), + ( + "zDeviceConfigPendingTimeout", + 7200, + "int", + "Device configuration build queued timeout", + "The number of seconds a device configuration build may be queued " + "before a timeout." + ), + ( + "zDeviceConfigTTL", + 43200, + "int", + "Device configuration expiration", + "The number of seconds to wait before rebuilding a " + "device configuration." + ), # zPythonClass maps device class to python classs (separate from device # class name) ( @@ -668,18 +692,18 @@ class ZenPropertyManager(object, PropertyManager): the actual value the popup will have. It also has management for zenProperties which are properties that can be - inherited long the acquision chain. All properties are for a branch are - defined on a "root node" specified by the function which must be returned - by the function getZenRootNode that should be over ridden in a sub class. - Prperties can then be added further "down" the aq_chain by calling - setZenProperty on any contained node. + inherited along the acquisition chain. All properties are for a branch + are defined on a "root node" specified by the function which must be + returned by the function getZenRootNode that should be over ridden in a + sub class. Properties can then be added further "down" the aq_chain by + calling setZenProperty on any contained node. ZenProperties all have the same prefix which is defined by iszprop this can be overridden in a subclass. ZenPropertyManager overrides getProperty and getPropertyType from PropertyManager to support acquisition. If you want to query an object - about a property, but do not want it to search the acquistion chain then + about a property, but do not want it to search the acquisition chain then use the super classes method or aq_base. Example: # acquires property from dmd.Devices @@ -692,7 +716,7 @@ class ZenPropertyManager(object, PropertyManager): aq_base(dmd.Devices.Server).getProperty('zSnmpCommunity') The properties are stored as attributes which is convenient, but can be - confusing. Attribute access always uses acquistion. Setting an + confusing. Attribute access always uses acquisition. Setting an attribute, will not add it to the list of properties, so subsquent calls to hasProperty or getProperty won't return it. @@ -1077,7 +1101,7 @@ def getProperty(self, id, d=None): security.declareProtected(ZEN_ZPROPERTIES_VIEW, "getPropertyType") def getPropertyType(self, id): - """Overrides methods from PropertyManager to support acquistion.""" + """Overrides methods from PropertyManager to support acquisition.""" ob = self._findParentWithProperty(id) if ob is not None: return PropertyManager.getPropertyType(ob, id) diff --git a/Products/ZenRelations/zPropertyCategory.py b/Products/ZenRelations/zPropertyCategory.py index 77bf4adefe..fc1eb7f06e 100644 --- a/Products/ZenRelations/zPropertyCategory.py +++ b/Products/ZenRelations/zPropertyCategory.py @@ -42,6 +42,12 @@ "zCommandUsername": "zencommand", "zKeyPath": "zencommand", # + # Configuration Cache + # ------------------- + "zDeviceConfigTTL": "Config Cache", + "zDeviceConfigBuildTimeout": "Config Cache", + "zDeviceConfigPendingTimeout": "Config Cache", + # # Misc # --------- "zDeviceTemplates": "Misc", diff --git a/Products/ZenUtils/DaemonStats.py b/Products/ZenUtils/DaemonStats.py index 51f98cf129..687264db8d 100644 --- a/Products/ZenUtils/DaemonStats.py +++ b/Products/ZenUtils/DaemonStats.py @@ -7,13 +7,13 @@ # ############################################################################## - -import time, os +import time +import os class DaemonStats(object): """ - Utility for a daemon to write out internal performance statistics + Utility for a daemon to write out internal performance statistics. """ def __init__(self): @@ -26,8 +26,14 @@ def __init__(self): self._tenant_id = None self._instance_id = None - def config(self, name, monitor, metric_writer, threshold_notifier, - derivative_tracker): + def config( + self, + name, + monitor, + metric_writer, + threshold_notifier, + derivative_tracker, + ): """ Initialize the object. We could do this in __init__, but that would delay creation to after configuration time, which @@ -42,66 +48,74 @@ def config(self, name, monitor, metric_writer, threshold_notifier, self._threshold_notifier = threshold_notifier self._derivative_tracker = derivative_tracker - # when running inside control plane pull the service id from the environment - if os.environ.get( 'CONTROLPLANE', "0") == "1": - self._tenant_id = os.environ.get('CONTROLPLANE_TENANT_ID') - self._service_id = os.environ.get('CONTROLPLANE_SERVICE_ID') - self._instance_id = os.environ.get('CONTROLPLANE_INSTANCE_ID') + # when running inside control plane pull the service id from the + # environment. + if os.environ.get("CONTROLPLANE", "0") == "1": + self._tenant_id = os.environ.get("CONTROLPLANE_TENANT_ID") + self._service_id = os.environ.get("CONTROLPLANE_SERVICE_ID") + self._instance_id = os.environ.get("CONTROLPLANE_INSTANCE_ID") def _context_id(self): return self.name + "-" + self.monitor def _contextKey(self): - return "/".join(('Daemons', self.monitor)) + return "/".join(("Daemons", self.monitor)) def _tags(self, metric_type): tags = { - 'daemon': self.name, - 'monitor': self.monitor, - 'metricType': metric_type, - 'internal': True + "daemon": self.name, + "monitor": self.monitor, + "metricType": metric_type, + "internal": True, } if self._service_id: - tags['serviceId'] = self._service_id + tags["serviceId"] = self._service_id if self._tenant_id: - tags['tenantId'] = self._tenant_id + tags["tenantId"] = self._tenant_id if self._instance_id: - tags['instance'] = self._instance_id + tags["instance"] = self._instance_id return tags def derive(self, name, value): """Write a DERIVE value and post any relevant events""" - self.post_metrics(name, value, 'DERIVE') + self.post_metrics(name, value, "DERIVE") def counter(self, name, value): """Write a COUNTER value and post any relevant events""" - self.post_metrics(name, value, 'COUNTER') + self.post_metrics(name, value, "COUNTER") def gauge(self, name, value): """Write a GAUGE value and post any relevant events""" - self.post_metrics(name, value, 'GAUGE') + self.post_metrics(name, value, "GAUGE") def post_metrics(self, name, value, metric_type): tags = self._tags(metric_type) timestamp = time.time() context_id = self._context_id() - if metric_type in {'DERIVE', 'COUNTER'}: + if metric_type in {"DERIVE", "COUNTER"}: # compute (and cache) a rate for COUNTER/DERIVE - if metric_type == 'COUNTER': + if metric_type == "COUNTER": metric_min = 0 else: - metric_min = 'U' + metric_min = "U" value = self._derivative_tracker.derivative( - '%s:%s' % (context_id, name), (float(value), timestamp), - min=metric_min) + "%s:%s" % (context_id, name), + (float(value), timestamp), + min=metric_min, + ) if value is not None: self._metric_writer.write_metric(name, value, timestamp, tags) # check for threshold breaches and send events when needed self._threshold_notifier.notify( - self._contextKey(), context_id, self.name+'_'+name, timestamp, value) + self._contextKey(), + context_id, + self.name + "_" + name, + timestamp, + value, + ) diff --git a/Products/ZenUtils/MetricServiceRequest.py b/Products/ZenUtils/MetricServiceRequest.py index 5fda9f0dfe..1ef46fb50c 100644 --- a/Products/ZenUtils/MetricServiceRequest.py +++ b/Products/ZenUtils/MetricServiceRequest.py @@ -98,7 +98,7 @@ def fetchMetrics(self, metrics, start="1h-ago", end=None, returnSet="EXACT"): """ metricQueries = [] for metric in metrics: - log.info("fetchMetrics metrics %s", metric) + log.debug("fetchMetrics metrics %s", metric) cf = metric.get('cf', 'average') rpn = metric.get('rpn', '') rate = metric.get('rate', False) @@ -124,6 +124,6 @@ def fetchMetrics(self, metrics, start="1h-ago", end=None, returnSet="EXACT"): queries=metricQueries ) body = FileBodyProducer(StringIO(json.dumps(request))) - log.info("POST %s %s %s", self._metric_url_v2, self._headers, json.dumps(request)) + log.debug("POST %s %s %s", self._metric_url_v2, self._headers, json.dumps(request)) d = self.agent.request('POST', self._metric_url_v2, self._headers, body) return d diff --git a/Products/ZenUtils/PBUtil.py b/Products/ZenUtils/PBUtil.py index c7a4cff93c..3ff95dfd43 100644 --- a/Products/ZenUtils/PBUtil.py +++ b/Products/ZenUtils/PBUtil.py @@ -1,34 +1,42 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2017, all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## -import logging -log = logging.getLogger("zen.pbclientfactory") +import logging -from twisted.spread.pb import PBClientFactory from twisted.internet import protocol, reactor, defer, task from twisted.internet.error import ConnectionClosed -import socket +from twisted.spread.pb import PBClientFactory + +log = logging.getLogger("zen.pbclientfactory") OPTION_STATE = 1 CONNECT_TIMEOUT = 60 -class ReconnectingPBClientFactory(PBClientFactory, - protocol.ReconnectingClientFactory): +class ReconnectingPBClientFactory( + PBClientFactory, protocol.ReconnectingClientFactory +): maxDelay = 60 - def __init__(self, connectTimeout=30, pingPerspective=True, pingInterval=30, pingtimeout=120): + def __init__( + self, + connectTimeout=30, + pingPerspective=True, + pingInterval=30, + pingtimeout=120, + ): PBClientFactory.__init__(self) self._creds = None self._scheduledConnectTimeout = None self._connectTimeout = connectTimeout - # should the perspective be pinged. Perspective must have a ping method. Deprecated => Always False. + # should the perspective be pinged. Perspective must have a ping + # method. Deprecated => Always False. self._shouldPingPerspective = pingPerspective # how often to ping self._pingInterval = pingInterval @@ -61,14 +69,20 @@ def clientConnectionFailed(self, connector, reason): self._perspective = None self._cancelConnectTimeout() PBClientFactory.clientConnectionFailed(self, connector, reason) - protocol.ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) + protocol.ReconnectingClientFactory.clientConnectionFailed( + self, connector, reason + ) def clientConnectionLost(self, connector, reason): log.debug("clientConnectionLost %s", reason) self._perspective = None self._cancelConnectTimeout() - PBClientFactory.clientConnectionLost(self, connector, reason, reconnecting=1) - protocol.ReconnectingClientFactory.clientConnectionLost(self, connector, reason) + PBClientFactory.clientConnectionLost( + self, connector, reason, reconnecting=1 + ) + protocol.ReconnectingClientFactory.clientConnectionLost( + self, connector, reason + ) def clientConnectionMade(self, broker): log.debug("clientConnectionMade") @@ -96,11 +110,13 @@ def gotPerspective(self, perspective): def gotPerspectiveFailed(self, reason): self._cancelConnectTimeout() - if reason.type == 'twisted.cred.error.UnauthorizedLogin': + if reason.type == "twisted.cred.error.UnauthorizedLogin": log.critical("zenhub username/password combination is incorrect!") # Don't exit as Enterprise caches info and can survive else: - log.critical("Unknown connection problem to zenhub %s", reason.type) + log.critical( + "Unknown connection problem to zenhub %s", reason.type + ) def _gotPerspective(self, perspective): self._cancelConnectTimeout() @@ -117,21 +133,26 @@ def _disconnect(self): try: self.connector.disconnect() except Exception: - log.exception('Could not disconnect') + log.exception("Could not disconnect") else: - log.debug('No connector or broker to disconnect') - + log.debug("No connector or broker to disconnect") + # methods for connecting and login timeout def _startConnectTimeout(self, msg): self._cancelConnectTimeout() - self._scheduledConnectTimeout = reactor.callLater(self._connectTimeout, self._timeoutConnect, msg) + self._scheduledConnectTimeout = reactor.callLater( + self._connectTimeout, self._timeoutConnect, msg + ) def _timeoutConnect(self, msg): log.info("%s timed out after %s seconds", msg, self._connectTimeout) self._disconnect() def _cancelConnectTimeout(self): - self._scheduledConnectTimeout, timeout = None, self._scheduledConnectTimeout + self._scheduledConnectTimeout, timeout = ( + None, + self._scheduledConnectTimeout, + ) if timeout and timeout.active(): log.debug("Cancelling connect timeout") timeout.cancel() @@ -139,8 +160,9 @@ def _cancelConnectTimeout(self): # methods to check connection is active def _startPingTimeout(self): if not self._pingTimeout: - self._pingTimeout = reactor.callLater(self._pingTimeoutTime, - self._doPingTimeout) + self._pingTimeout = reactor.callLater( + self._pingTimeoutTime, self._doPingTimeout + ) def _cancelPingTimeout(self): self._pingTimeout, timeout = None, self._pingTimeout @@ -150,7 +172,10 @@ def _cancelPingTimeout(self): def _doPingTimeout(self): if self._perspective: - log.warn("Perspective ping timed out after %s seconds", self._pingTimeoutTime) + log.warn( + "Perspective ping timed out after %s seconds", + self._pingTimeoutTime, + ) self._disconnect() @defer.inlineCallbacks @@ -170,12 +195,12 @@ def _startPingCycle(self): def _pingPerspective(self): try: if self._perspective: - log.debug('pinging perspective') + log.debug("pinging perspective") self._startPingTimeout() - response = yield self._perspective.callRemote('ping') + response = yield self._perspective.callRemote("ping") log.debug("perspective %sed", response) else: - log.debug('skipping perspective ping') + log.debug("skipping perspective ping") self._cancelPingTimeout() except ConnectionClosed: log.info("Connection was closed") @@ -187,6 +212,7 @@ def _pingPerspective(self): def setKeepAlive(sock): """Configure a socket for a longer keep-alive interval.""" import socket + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, OPTION_STATE) sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, CONNECT_TIMEOUT) interval = max(CONNECT_TIMEOUT / 4, 10) diff --git a/Products/ZenUtils/ZCmdBase.py b/Products/ZenUtils/ZCmdBase.py index 8b831ae516..7aaad6c956 100644 --- a/Products/ZenUtils/ZCmdBase.py +++ b/Products/ZenUtils/ZCmdBase.py @@ -88,7 +88,7 @@ class ZCmdBase(ZenDaemon): """Base class for daemons that need ZODB access.""" def __init__(self, noopts=0, app=None, keeproot=False): - ZenDaemon.__init__(self, noopts, keeproot) + super(ZCmdBase, self).__init__(noopts=noopts, keeproot=keeproot) self.dataroot = None self.app = app self.db = None diff --git a/Products/ZenUtils/ZenDaemon.py b/Products/ZenUtils/ZenDaemon.py index 60ca76db1f..49b5ad35ea 100755 --- a/Products/ZenUtils/ZenDaemon.py +++ b/Products/ZenUtils/ZenDaemon.py @@ -337,7 +337,7 @@ def sigTerm(self, signum=None, frame=None): if self.pidfile and os.path.exists(self.pidfile): self.log.info("Deleting PID file %s ...", self.pidfile) os.remove(self.pidfile) - self.log.info("Daemon %s shutting down", type(self).__name__) + self.log.info("received signal to shut down") self.audit("Stop") def watchdogCycleTime(self): @@ -422,7 +422,7 @@ def niceDoggie(self, timeout): self.reporter.niceDoggie(timeout) def buildOptions(self): - CmdBase.buildOptions(self) + super(ZenDaemon, self).buildOptions() self.parser.add_option( "--uid", dest="uid", @@ -497,7 +497,7 @@ def buildOptions(self): "--heartbeattimeout", dest="heartbeatTimeout", type="int", - default=900, + default=getattr(self, "heartbeatTimeout", 900), help="Set a heartbeat timeout in seconds for a daemon; " "default %default", ) diff --git a/Products/ZenUtils/metricwriter.py b/Products/ZenUtils/metricwriter.py index a14b98adec..5c20bd4ce6 100644 --- a/Products/ZenUtils/metricwriter.py +++ b/Products/ZenUtils/metricwriter.py @@ -30,13 +30,20 @@ def write_metric(self, metric, value, timestamp, tags): @return deferred: metric was published or queued """ try: - if tags and 'mtrace' in tags.keys(): - log.info("mtrace: publishing metric %s %s %s %s", - metric, value, timestamp, tags) - log.debug("publishing metric %s %s %s %s", metric, value, - timestamp, tags) - val = defer.maybeDeferred(self._publisher.put, metric, value, - timestamp, tags) + if tags and "mtrace" in tags.keys(): + log.info( + "mtrace: publishing metric %s %s %s %s", + metric, + value, + timestamp, + tags, + ) + log.debug( + "publishing metric %s %s %s %s", metric, value, timestamp, tags + ) + val = defer.maybeDeferred( + self._publisher.put, metric, value, timestamp, tags + ) self._datapoints += 1 return val except Exception as x: @@ -69,13 +76,24 @@ def write_metric(self, metric, value, timestamp, tags): """ try: if self._test_filter(metric, value, timestamp, tags): - if tags and 'mtrace' in tags.keys(): - log.info("mtrace: publishing metric %s %s %s %s", - metric, value, timestamp, tags) - log.debug("publishing metric %s %s %s %s", metric, value, - timestamp, tags) - val = defer.maybeDeferred(self._publisher.put, metric, value, - timestamp, tags) + if tags and "mtrace" in tags.keys(): + log.info( + "mtrace: publishing metric %s %s %s %s", + metric, + value, + timestamp, + tags, + ) + log.debug( + "publishing metric %s %s %s %s", + metric, + value, + timestamp, + tags, + ) + val = defer.maybeDeferred( + self._publisher.put, metric, value, timestamp, tags + ) self._datapoints += 1 return val except Exception as x: @@ -108,7 +126,11 @@ def write_metric(self, metric, value, timestamp, tags): dList = [] for writer in self._writers: try: - dList.append(defer.maybeDeferred(writer.write_metric, metric, value, timestamp, tags)) + dList.append( + defer.maybeDeferred( + writer.write_metric, metric, value, timestamp, tags + ) + ) except Exception as x: log.exception(x) self._datapoints += 1 @@ -127,7 +149,7 @@ class DerivativeTracker(object): def __init__(self): self._timed_metric_cache = {} - def derivative(self, name, timed_metric, min='U', max='U'): + def derivative(self, name, timed_metric, min="U", max="U"): """ Tracks a metric value over time and returns deltas @@ -148,8 +170,9 @@ def derivative(self, name, timed_metric, min='U', max='U'): # in an infinity/nan rate. return None else: - delta = float(timed_metric[0] - last_timed_metric[0]) / \ - float(timed_metric[1] - last_timed_metric[1]) + delta = float(timed_metric[0] - last_timed_metric[0]) / float( + timed_metric[1] - last_timed_metric[1] + ) # Get min/max into a usable float or None state. min, max = map(constraint_value, (min, max)) @@ -189,7 +212,7 @@ def constraint_value(value): elif isinstance(value, int): return float(value) elif isinstance(value, types.StringTypes): - if value in ('U', ''): + if value in ("U", ""): return None try: @@ -219,7 +242,15 @@ def updateThresholds(self, thresholds): self._thresholds.updateList(thresholds) @defer.inlineCallbacks - def notify(self, context_uuid, context_id, metric, timestamp, value, thresh_event_data={}): + def notify( + self, + context_uuid, + context_id, + metric, + timestamp, + value, + thresh_event_data={}, + ): """ Check the specified value against thresholds and send any generated events @@ -233,22 +264,23 @@ def notify(self, context_uuid, context_id, metric, timestamp, value, thresh_even @return: """ if self._thresholds and value is not None: - if 'eventKey' in thresh_event_data: - eventKeyPrefix = [thresh_event_data['eventKey']] + if "eventKey" in thresh_event_data: + eventKeyPrefix = [thresh_event_data["eventKey"]] else: eventKeyPrefix = [metric] - for ev in self._thresholds.check(context_uuid, metric, timestamp, value): + for ev in self._thresholds.check( + context_uuid, metric, timestamp, value + ): parts = eventKeyPrefix[:] - if 'eventKey' in ev: - parts.append(ev['eventKey']) - ev['eventKey'] = '|'.join(parts) + if "eventKey" in ev: + parts.append(ev["eventKey"]) + ev["eventKey"] = "|".join(parts) # add any additional values for this threshold # (only update if key is not in event, or if # the event's value is blank or None) for key, value in thresh_event_data.items(): - if ev.get(key, None) in ('', None): + if ev.get(key, None) in ("", None): ev[key] = value if ev.get("component", None): - ev['component_guid'] = context_uuid + ev["component_guid"] = context_uuid yield defer.maybeDeferred(self._send_callback, ev) - diff --git a/Products/ZenUtils/terminal_size.py b/Products/ZenUtils/terminal_size.py new file mode 100644 index 0000000000..9f7325b006 --- /dev/null +++ b/Products/ZenUtils/terminal_size.py @@ -0,0 +1,125 @@ +"""This is a backport of shutil.get_terminal_size from Python 3.3. + +The original implementation is in C, but here we use the ctypes and +fcntl modules to create a pure Python version of os.get_terminal_size. + +Pulled from https://github.com/chrippa/backports.shutil_get_terminal_size + + +The MIT License (MIT) + +Copyright (c) 2014 Christopher Rosell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import os +import struct +import sys + +from collections import namedtuple + +__all__ = ["get_terminal_size"] + + +terminal_size = namedtuple("terminal_size", "columns lines") + +try: + from ctypes import windll, create_string_buffer, WinError + + _handle_ids = { + 0: -10, + 1: -11, + 2: -12, + } + + def _get_terminal_size(fd): + handle = windll.kernel32.GetStdHandle(_handle_ids[fd]) + if handle == 0: + raise OSError('handle cannot be retrieved') + if handle == -1: + raise WinError() + csbi = create_string_buffer(22) + res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi) + if res: + res = struct.unpack("hhhhHhhhhhh", csbi.raw) + left, top, right, bottom = res[5:9] + columns = right - left + 1 + lines = bottom - top + 1 + return terminal_size(columns, lines) + else: + raise WinError() + +except ImportError: + import fcntl + import termios + + def _get_terminal_size(fd): + try: + res = fcntl.ioctl(fd, termios.TIOCGWINSZ, b"\x00" * 4) + except IOError as e: + raise OSError(e) + lines, columns = struct.unpack("hh", res) + + return terminal_size(columns, lines) + + +def get_terminal_size(fallback=(80, 24)): + """Get the size of the terminal window. + + For each of the two dimensions, the environment variable, COLUMNS + and LINES respectively, is checked. If the variable is defined and + the value is a positive integer, it is used. + + When COLUMNS or LINES is not defined, which is the common case, + the terminal connected to sys.__stdout__ is queried + by invoking os.get_terminal_size. + + If the terminal size cannot be successfully queried, either because + the system doesn't support querying, or because we are not + connected to a terminal, the value given in fallback parameter + is used. Fallback defaults to (80, 24) which is the default + size used by many terminal emulators. + + The value returned is a named tuple of type os.terminal_size. + """ + # Try the environment first + try: + columns = int(os.environ["COLUMNS"]) + except (KeyError, ValueError): + columns = 0 + + try: + lines = int(os.environ["LINES"]) + except (KeyError, ValueError): + lines = 0 + + # Only query if necessary + if columns <= 0 or lines <= 0: + try: + size = _get_terminal_size(sys.__stdout__.fileno()) + except (NameError, OSError): + size = terminal_size(*fallback) + + if columns <= 0: + columns = size.columns + if lines <= 0: + lines = size.lines + + return terminal_size(columns, lines) diff --git a/setup.py b/setup.py index 5b3f274e00..967d4d2432 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,24 @@ +from __future__ import print_function + from os import path # , walk from distutils.command.build import build from setuptools import setup, find_packages from setuptools.command.develop import develop from setuptools.command.install import install -from setuptools.command.sdist import sdist _here = path.abspath(path.dirname(__file__)) with open(path.join(_here, "VERSION"), "r") as _f: - _version = ''.join(_f.readlines()).strip() + _version = "".join(_f.readlines()).strip() class ZenInstallCommand(install): """Used to disable installs.""" def run(self): - print "Installation disabled" + print("Installation disabled") import sys + sys.exit(1) @@ -24,8 +26,9 @@ class ZenBuildCommand(build): """Used to disable builds.""" def run(self): - print "Build disabled" + print("Build disabled") import sys + sys.exit(1) @@ -41,13 +44,6 @@ class ZenDevelopCommand(develop): ) -def applySchemaVersion(*args, **kw): - print("Applied: %s %s" % (args, kw)) - - -sdist.sub_commands.append(("apply_schema_version", applySchemaVersion)) - - setup( name="Zenoss", version=_version, @@ -76,6 +72,9 @@ def applySchemaVersion(*args, **kw): "install": ZenInstallCommand, }, entry_points={ + "console_scripts": [ + "configcache=Products.ZenCollector.configcache.__main__:main", + ], "celery.commands": [ "monitor=Products.Jobber.monitor:ZenJobsMonitor", ], From 70b1d116a1fb16f8c2c62bf74d09db27c30f2899 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 16 Jan 2024 14:06:08 -0600 Subject: [PATCH 035/147] Include setup.py, VERSION files in the archive. ZEN-34639 --- .gitignore | 1 - install-zenoss.mk.in | 29 ----------------------------- javascript.mk | 4 ++-- jenkins_build.sh | 17 +++++++++++------ makefile | 38 +++++++++----------------------------- migration.mk | 7 +++++-- 6 files changed, 27 insertions(+), 69 deletions(-) delete mode 100644 install-zenoss.mk.in diff --git a/.gitignore b/.gitignore index e301a9d6b7..53f6eb72ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ Products/ZenUI3/node_modules dist build Zenoss.egg-info -install-zenoss.mk lib/python2.7/site-packages/Zenoss-nspkg.pth lib/python2.7/site-packages/Zenoss.egg-link diff --git a/install-zenoss.mk.in b/install-zenoss.mk.in deleted file mode 100644 index 21ca9184eb..0000000000 --- a/install-zenoss.mk.in +++ /dev/null @@ -1,29 +0,0 @@ -TARGET = /mnt -ZENHOME = /opt/zenoss - -SITE_PACKAGES = lib/python2.7/site-packages -PTH_FILE = $(SITE_PACKAGES)/Zenoss-nspkg.pth -EGG_LINK = $(SITE_PACKAGES)/Zenoss.egg-link - -.PHONY: install configure-user - -install: $(TARGET)/$(PTH_FILE) $(TARGET)/$(EGG_LINK) $(TARGET)/Zenoss.egg-info - -configure-user: - groupmod -g %GID% zenoss - usermod -u %UID% zenoss - chown -R zenoss:zenoss $(ZENHOME) - -$(TARGET)/Zenoss.egg-info: $(ZENHOME)/$(PTH_FILE) - -$(ZENHOME)/$(PTH_FILE): | configure-user - su - zenoss -c "cd $(TARGET); python setup.py develop" - -$(TARGET)/$(SITE_PACKAGES): - su - zenoss -c "mkdir -p $@" - -$(TARGET)/$(PTH_FILE): $(ZENHOME)/$(PTH_FILE) | $(TARGET)/$(SITE_PACKAGES) - su - zenoss -c "cp $< $@" - -$(TARGET)/$(EGG_LINK): | $(TARGET)/$(SITE_PACKAGES) - su - zenoss -c "printf \"/opt/zenoss\n.\n\" > $@" diff --git a/javascript.mk b/javascript.mk index 41c884c42a..15e833025f 100644 --- a/javascript.mk +++ b/javascript.mk @@ -47,10 +47,10 @@ JSBUILD_COMMAND = java -jar $(JSBUILDER) -p $(JSB_FILE) -d $(JS_BASEDIR) -v JSB_SOURCES = $(shell python2 -c "import json, sys, os.path; d=sys.stdin.read(); p=json.loads(d)['pkgs'][0]['fileIncludes']; print ' '.join(os.path.join('$(JS_BASEDIR)', e['path'], e['text']) for e in p)" < $(JSB_FILE)) JSB_TARGETS = $(JS_OUTPUT_DIR)/zenoss-compiled.js $(JS_OUTPUT_DIR)/zenoss-compiled-debug.js -.PHONY: clean-javascript build-javascript - +.PHONY: build-javascript build-javascript: $(JSB_TARGETS) +.PHONY: clean-javascript clean-javascript: @-rm -vrf $(JS_OUTPUT_DIR) diff --git a/jenkins_build.sh b/jenkins_build.sh index 46e7024b39..fee23544be 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -49,11 +49,8 @@ REPO_PATH=${ZENDEV_ROOT}/src/github.com/zenoss/${REPO_NAME} cleanup() { RC="$?" - if [[ $RC == 0 ]]; then - zendev drop ${ZENDEV_ENV} - docker image rm -f zendev/devimg:${ZENDEV_ENV} zendev/product-base:${ZENDEV_ENV} - docker image rm -f zendev/mariadb:${ZENDEV_ENV} zendev/mariadb-base:${ZENDEV_ENV} - fi + zendev drop ${ZENDEV_ENV} + docker image rm -f zendev/devimg:${ZENDEV_ENV} zendev/product-base:${ZENDEV_ENV} zendev/mariadb:${ZENDEV_ENV} zendev/mariadb-base:${ZENDEV_ENV} } trap cleanup INT TERM EXIT @@ -132,4 +129,12 @@ if [ "$1" != "--no-tests" ]; then fi echo Building the artifacts... -cdz ${REPO_NAME};make clean build +docker run --rm \ + -v ${HOME}/.m2:/home/zenoss/.m2 \ + -v ${ZENDEV_ROOT}/zenhome:/opt/zenoss \ + -v ${ZENDEV_ROOT}/src/github.com/zenoss:/mnt/src \ + -w /mnt/src/zenoss-prodbin \ + --env ZENHOME=/opt/zenoss \ + --env SRCROOT=/mnt/src \ + zendev/devimg:${ZENDEV_ENV} \ + make clean build diff --git a/makefile b/makefile index 1b3a4174ab..340bd70e5f 100644 --- a/makefile +++ b/makefile @@ -3,23 +3,11 @@ BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) ARTIFACT_TAG ?= $(shell echo $(BRANCH) | sed 's/\//-/g') ARTIFACT = prodbin-$(VERSION)-$(ARTIFACT_TAG).tar.gz -IMAGE = zenoss/zenoss-centos-base:1.4.0.devtools - -USER_ID := $(shell id -u) -GROUP_ID := $(shell id -g) - -DOCKER = $(shell which docker 2>/dev/null) -ifneq ($(DOCKER),) -_common_cmd = $(DOCKER) run --rm -v $(PWD):/mnt -w /mnt -DOCKER_USER = $(_common_cmd) --user $(USER_ID):$(GROUP_ID) $(IMAGE) -DOCKER_ROOT = $(_common_cmd) $(IMAGE) -endif +IMAGE = zenoss/zenpackbuild:ubuntu2204-5 ZENHOME = $(shell echo $$ZENHOME) -.PHONY: default test clean build javascript build-javascript - -default: $(ARTIFACT) +.DEFAULT_GOAL := $(ARTIFACT) include javascript.mk include migration.mk @@ -27,31 +15,23 @@ include migration.mk EXCLUSIONS = *.pyc $(MIGRATE_VERSION).in Products/ZenModel/migrate/tests Products/ZenUITests ARCHIVE_EXCLUSIONS = $(foreach item,$(EXCLUSIONS),--exclude=$(item)) -ARCHIVE_INCLUSIONS = Products bin lib etc share Zenoss.egg-info +ARCHIVE_INCLUSIONS = Products bin etc share VERSION setup.py +.PHONY: build build: $(ARTIFACT) # equivalent to python setup.py develop +.PHONY: install install: setup.py $(JSB_TARGETS) $(MIGRATE_VERSION) ifeq ($(ZENHOME),/opt/zenoss) - @python setup.py develop + @pip install --prefix /opt/zenoss -e . else @echo "Please execute this target in a devshell container (where ZENHOME=/opt/zenoss)." endif +.PHONY: clean clean: clean-javascript clean-migration - rm -f $(ARTIFACT) install-zenoss.mk - rm -rf Zenoss.egg-info lib + rm -f $(ARTIFACT) -$(ARTIFACT): $(JSB_TARGETS) $(MIGRATE_VERSION) Zenoss.egg-info +$(ARTIFACT): $(JSB_TARGETS) $(MIGRATE_VERSION) VERSION setup.py tar cvfz $@ $(ARCHIVE_EXCLUSIONS) $(ARCHIVE_INCLUSIONS) - -Zenoss.egg-info: install-zenoss.mk setup.py -ifneq ($(DOCKER),) - $(DOCKER_ROOT) make -f install-zenoss.mk install -else - $(error The $@ target requires Docker) -endif - -install-zenoss.mk: install-zenoss.mk.in - sed -e "s/%GID%/$(GROUP_ID)/" -e "s/%UID%/$(USER_ID)/" $< > $@ diff --git a/migration.mk b/migration.mk index c1f5c4d2e6..e3a1c25f21 100644 --- a/migration.mk +++ b/migration.mk @@ -11,16 +11,17 @@ SCHEMA_MAJOR = $(call pick_version_part,1,$(SCHEMA_VERSION)) SCHEMA_MINOR = $(call pick_version_part,2,$(SCHEMA_VERSION)) SCHEMA_REVISION = $(call pick_version_part,3,$(SCHEMA_VERSION)) -.PHONY: clean-migration generate-zversion generate-zmigrateversion - +.PHONY: clean-migration clean-migration: rm -f $(MIGRATE_VERSION) # Exists for backward compatibility +.PHONY: generate-zversion generate-zversion: generate-zmigrateversion # See the topic "Managing Migrate.Version" in Products/ZenModel/migrate/README.md # for more information about setting the SCHEMA_* values. +.PHONY: generate-zmigrateversion generate-zmigrateversion: $(MIGRATE_VERSION) $(MIGRATE_VERSION): $(MIGRATE_VERSION).in SCHEMA_VERSION @@ -33,6 +34,7 @@ $(MIGRATE_VERSION): $(MIGRATE_VERSION).in SCHEMA_VERSION # The target replace-zmigrationversion should be used just prior to release to lock # down the schema versions for a particular release +.PHONY: replace-zmigrateversion replace-zmigrateversion: @echo Replacing SCHEMA_MAJOR with $(SCHEMA_MAJOR) @echo Replacing SCHEMA_MINOR with $(SCHEMA_MINOR) @@ -52,6 +54,7 @@ SCHEMA_FOUND = $(shell grep Migrate.Version Products/ZenModel/migrate/*.py | gr # The target verify-explicit-zmigrateversion should be invoked as a first step in all release # builds to verify that all of the SCHEMA_* variables were replaced with an actual numeric value. +.PHONY: verify-explicit-zmigrateversion verify-explicit-zmigrateversion: ifeq ($(SCHEMA_FOUND),) @echo "Good - no SCHEMA_* variables found: $(SCHEMA_FOUND)" From 52e9ff3c0bea0307690f9e0851a05c48ae9d5d43 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 16 Jan 2024 16:22:08 -0600 Subject: [PATCH 036/147] Add BRANCH env to artifact build container. ZEN-34639 --- jenkins_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins_build.sh b/jenkins_build.sh index fee23544be..1400318340 100755 --- a/jenkins_build.sh +++ b/jenkins_build.sh @@ -134,6 +134,7 @@ docker run --rm \ -v ${ZENDEV_ROOT}/zenhome:/opt/zenoss \ -v ${ZENDEV_ROOT}/src/github.com/zenoss:/mnt/src \ -w /mnt/src/zenoss-prodbin \ + --env BRANCH=${BRANCH} \ --env ZENHOME=/opt/zenoss \ --env SRCROOT=/mnt/src \ zendev/devimg:${ZENDEV_ENV} \ From fbb238398b6d974065855e3c2d3cdc52d20fc223 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 16 Jan 2024 09:00:03 -0600 Subject: [PATCH 037/147] Removed code pushing device config updates to collector. Added code to create device configs when new devices show up. ZEN-34635 --- Products/ZenCollector/config/task.py | 10 +- .../ZenCollector/configcache/invalidator.py | 31 +- Products/ZenCollector/configcache/manager.py | 6 +- .../ZenCollector/configcache/modelchange.zcml | 10 +- .../configcache/modelchange/filters.py | 4 + .../configcache/modelchange/invalidation.py | 14 +- .../configcache/modelchange/oids.py | 76 ++++- .../configcache/modelchange/processor.py | 13 +- Products/ZenCollector/daemon.py | 35 +-- Products/ZenCollector/scheduler.py | 15 +- Products/ZenCollector/services/push.py | 267 +----------------- Products/ZenCollector/tasks.py | 21 -- Products/ZenCollector/tests/test_daemon.py | 15 +- Products/ZenCollector/utils/maintenance.py | 83 ++---- Products/ZenEvents/tests/test_zeneventd.py | 53 ++-- Products/ZenEvents/zeneventd.py | 197 ++++++++----- Products/ZenEvents/zentrap.py | 18 +- Products/ZenHub/HubService.py | 2 +- Products/ZenHub/services/ProcessConfig.py | 37 --- Products/ZenHub/services/SnmpTrapConfig.py | 21 +- Products/ZenHub/services/ThresholdMixin.py | 2 +- Products/ZenStatus/zenping.py | 4 +- 22 files changed, 350 insertions(+), 584 deletions(-) diff --git a/Products/ZenCollector/config/task.py b/Products/ZenCollector/config/task.py index aebc32d662..e1ff0b341b 100644 --- a/Products/ZenCollector/config/task.py +++ b/Products/ZenCollector/config/task.py @@ -110,7 +110,7 @@ def doTask(self): thresholds = yield proxy.getThresholds() self._processThresholds(thresholds) - self._collector.runPostConfigTasks() + yield self._collector.runPostConfigTasks() except Exception as ex: log.exception("task '%s' failed", self.name) @@ -195,11 +195,11 @@ def _processConfigs(self, config_data): yield self._callback(new, updated, removed) finally: self._update_local_cache(new, updated, removed) - log.info( + lengths = (len(new), len(updated), len(removed)) + logmethod = log.debug if lengths == (0, 0, 0) else log.info + logmethod( "processed %d new, %d updated, and %d removed device configs", - len(new), - len(updated), - len(removed), + *lengths ) def _update_local_cache(self, new, updated, removed): diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 74417c3e24..41407821d0 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -43,9 +43,7 @@ class Invalidator(object): "device configurations" ) - configs = ( - ("modelchange.zcml", CONFIGCACHE_MODULE), - ) + configs = (("modelchange.zcml", CONFIGCACHE_MODULE),) @staticmethod def add_arguments(parser, subparsers): @@ -102,14 +100,17 @@ def run(self): "polling for device changes every %s seconds", self.interval ) while not self.ctx.controller.shutdown: - for invalidation in poller.poll(): + result = poller.poll() + if result: + self.log.debug("found %d relevant invalidations", len(result)) + for invalidation in result: try: self._process(invalidation) except AttributeError: self.log.info( "invalidation device=%s reason=%s", invalidation.device, - invalidation.reason + invalidation.reason, ) self.log.exception("failed while processing invalidation") self.ctx.controller.wait(self.interval) @@ -128,7 +129,7 @@ def _synchronize(self): self.log, tool, timelimitmap, self.store, self.dispatcher ) if len(new_devices) == 0: - self.log.info("no missing configurations") + self.log.info("no missing configurations found") def _process(self, invalidation): device = invalidation.device @@ -137,7 +138,19 @@ def _process(self, invalidation): keys = list( self.store.search(ConfigQuery(monitor=monitor, device=device.id)) ) - if reason is InvalidationCause.Updated: + if not keys: + timelimitmap = DevicePropertyMap.from_organizer( + self.ctx.dmd.Devices, Constants.build_timeout_id + ) + uid = device.getPrimaryId() + timeout = timelimitmap.get(uid) + self.dispatcher.dispatch_all(monitor, device.id, timeout) + self.log.info( + "submitted build jobs for new device uid=%s monitor=%s", + uid, + monitor, + ) + elif reason is InvalidationCause.Updated: self.store.set_expired(*keys) for key in keys: self.log.info( @@ -148,7 +161,7 @@ def _process(self, invalidation): key.service, invalidation.oid, ) - elif reason is InvalidationCause.Deleted: + elif reason is InvalidationCause.Removed: self.store.remove(*keys) for key in keys: self.log.info( @@ -195,7 +208,7 @@ def _removeDeleted(log, tool, store): "device=%s monitor=%s service=%s", key.device, key.monitor, - key.device, + key.service, ) return len(devices_not_found) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index ac02738dc2..b6261c76ac 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -105,7 +105,7 @@ def _retry_pending_builds(self): duration = pendinglimitmap.get(uid) if status.submitted < (now - duration): self.store.set_expired(key) - self.log.debug( + self.log.info( "pending configuration build has timed out " "submitted=%s service=%s monitor=%s device=%s", datetime.fromtimestamp(status.submitted).strftime( @@ -152,7 +152,7 @@ def _rebuild_older_configs(self): key.service, key.monitor, key.device, timeout ) if isinstance(status, ConfigStatus.Expired): - self.log.debug( + self.log.info( "submitted job to rebuild expired config " "service=%s monitor=%s device=%s", key.service, @@ -160,7 +160,7 @@ def _rebuild_older_configs(self): key.device, ) else: - self.log.debug( + self.log.info( "submitted job to rebuild old config " "updated=%s %s=%s service=%s monitor=%s device=%s", datetime.fromtimestamp(status.updated).strftime( diff --git a/Products/ZenCollector/configcache/modelchange.zcml b/Products/ZenCollector/configcache/modelchange.zcml index 09c79fe93a..16afa81f0c 100644 --- a/Products/ZenCollector/configcache/modelchange.zcml +++ b/Products/ZenCollector/configcache/modelchange.zcml @@ -35,27 +35,27 @@ License.zenoss under the directory where your Zenoss product is installed. /> " % ( - self.__class__, + return "" % ( self.oid, self.device, self.reason, diff --git a/Products/ZenCollector/configcache/modelchange/oids.py b/Products/ZenCollector/configcache/modelchange/oids.py index cc96433ac3..959950ab7a 100644 --- a/Products/ZenCollector/configcache/modelchange/oids.py +++ b/Products/ZenCollector/configcache/modelchange/oids.py @@ -16,6 +16,7 @@ from zope.interface import implementer from Products.ZenHub.interfaces import IInvalidationOid +from Products.ZenRelations.RelationshipBase import IRelationship from Products.Zuul.catalog.interfaces import IModelCatalogTool log = logging.getLogger("zen.configcache.modelchange") @@ -27,31 +28,58 @@ def __init__(self, entity): @implementer(IInvalidationOid) -class DefaultOidTransform(BaseTransform): - """Default transformation returns the OID that was given.""" +class IdentityOidTransform(BaseTransform): + """Identity transformation returns the OID that was given.""" def transformOid(self, oid): + log.debug( + "[IdentityOidTransform] entity=%s oid=%r", self._entity, oid + ) return oid @implementer(IInvalidationOid) -class DeviceOidTransform(BaseTransform): +class ComponentOidTransform(BaseTransform): """ If the object has a relationship with a device, return the device's OID. """ def transformOid(self, oid): - # get device oid - result = oid - device = getattr(self._entity, "device", lambda: None)() - if device: - result = device._p_oid - log.debug( - "oid for %s transformed to device oid for %s", - self._entity, - device, - ) - return result + funcs = ( + lambda: self._getdevice(self._entity), + self._from_os, + self._from_hw, + ) + for fn in funcs: + device = fn() + if device: + log.debug( + "[ComponentOidTransform] transformed oid to device " + "entity=%s oid=%r device=%s", + self._entity, + oid, + device, + ) + return device._p_oid + log.debug( + "[ComponentOidTransform] oid not transformed entity=%s oid=%r", + self._entity, + oid, + ) + return oid + + def _from_os(self): + return self._getdevice(getattr(self._entity, "os", None)) + + def _from_hw(self): + return self._getdevice(getattr(self._entity, "hw", None)) + + def _getdevice(self, entity): + if entity is None: + return + if isinstance(entity, IRelationship): + entity = entity() + return getattr(entity, "device", lambda: None)() @implementer(IInvalidationOid) @@ -62,6 +90,11 @@ def transformOid(self, oid): ds = self._entity.datasource().primaryAq() template = ds.rrdTemplate().primaryAq() dc = template.deviceClass().primaryAq() + log.debug( + "[DataPointToDevice] return OIDs of devices associated " + "with DataPoint entity=%s ", + self._entity, + ) return _getDevicesFromDeviceClass(dc) @@ -72,6 +105,11 @@ class DataSourceToDevice(BaseTransform): def transformOid(self, oid): template = self._entity.rrdTemplate().primaryAq() dc = template.deviceClass().primaryAq() + log.debug( + "[DataSourceToDevice] return OIDs of devices associated " + "with DataSource entity=%s ", + self._entity, + ) return _getDevicesFromDeviceClass(dc) @@ -81,6 +119,11 @@ class TemplateToDevice(BaseTransform): def transformOid(self, oid): dc = self._entity.deviceClass().primaryAq() + log.debug( + "[TemplateToDevice] return OIDs of devices associated " + "with RRDTemplate entity=%s ", + self._entity, + ) return _getDevicesFromDeviceClass(dc) @@ -89,6 +132,11 @@ class DeviceClassToDevice(BaseTransform): """Return the device OIDs in the DeviceClass hierarchy.""" def transformOid(self, oid): + log.debug( + "[DeviceClassToDevice] return OIDs of devices associated " + "with DeviceClass entity=%s ", + self._entity, + ) return _getDevicesFromDeviceClass(self._entity) diff --git a/Products/ZenCollector/configcache/modelchange/processor.py b/Products/ZenCollector/configcache/modelchange/processor.py index dff7ec82ad..94d410d3e2 100644 --- a/Products/ZenCollector/configcache/modelchange/processor.py +++ b/Products/ZenCollector/configcache/modelchange/processor.py @@ -21,6 +21,7 @@ FILTER_EXCLUDE, IInvalidationOid, ) +from Products.ZenModel.Device import Device from Products.ZenModel.DeviceComponent import DeviceComponent from Products.ZenRelations.PrimaryPathObjectManager import ( PrimaryPathObjectManager, @@ -78,7 +79,7 @@ def apply(self, oid): class OidToObject(Action): """Validates the OID to ensure it references a device.""" - SINK = "sink" + SINK = "sink1" def __init__(self, app): """ @@ -142,7 +143,7 @@ def __call__(self, invalidation): log.debug( "invalidation excluded by filter filter=%r device=%s", fltr, - invalidation.device + invalidation.device, ) break else: @@ -160,7 +161,7 @@ class ApplyTransforms(Action): in its place. """ - SINK = "sink" + SINK = "sink2" def __call__(self, invalidation): # First, get any subscription adapters registered as transforms @@ -210,14 +211,18 @@ def __init__(self): def __call__(self, result): results = into_tuple(result) + devices = [] for result in results: + if not isinstance(result.device, Device): + continue + devices.append(result) log.debug( "collected an invalidation reason=%s device=%s oid=%r", result.reason, result.device, result.oid, ) - self._output.update(results) + self._output.update(devices) def pop(self): """Return the collected data, removing it from the set.""" diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 86c3ee9455..ece0d1e6a7 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -18,7 +18,6 @@ from metrology import Metrology from metrology.instruments import Gauge from twisted.internet import defer, reactor, task -from twisted.python.failure import Failure from zope.component import ( getUtilitiesFor, provideUtility, @@ -48,8 +47,6 @@ ITaskSplitter, ) from .listeners import ConfigListenerNotifier - -# from .statistics import StatisticsService from .utils.maintenance import MaintenanceCycle, ZenHubHeartbeatSender CONFIG_LOADER_NAME = "configLoader" @@ -284,7 +281,7 @@ def connected(self): try: yield defer.maybeDeferred(self._getInitializationCallback()) framework = _getFramework(self.frameworkFactoryName) - self.log.info("Using framework -> %r", framework) + self.log.debug("using framework factory %s", type(framework)) self._configProxy = framework.getConfigurationProxy() yield self._initEncryptionKey() yield self._startConfigCycle() @@ -345,7 +342,6 @@ def _startMaintenance(self): interval, heartbeatSender, self._maintenanceCallback ) self._maintenanceCycle.start() - self.log.debug("started maintenance cycle interval=%s", interval) def _startTaskStatsLogging(self): if not (self.options.cycle and self.options.logTaskStats): @@ -372,7 +368,7 @@ def _startDeviceConfigLoader(self): self._deviceloader = DeviceConfigLoader( self.options, self._configProxy, - self._deviceConfgCallback, + self._deviceConfigCallback, ) self._deviceloadertask = task.LoopingCall(self._deviceloader) self._deviceloadertaskd = self._deviceloadertask.start( @@ -616,7 +612,7 @@ def _taskCompleteCallback(self, taskName): self._displayStatistics() self.stop() - def _deviceConfgCallback(self, new, updated, removed): + def _deviceConfigCallback(self, new, updated, removed): """ Update the device configs for the devices this collector manages. @@ -640,6 +636,7 @@ def _deleteDevice(self, deviceId): self.log.debug("deleted device device-id=%s", deviceId) self._configListener.deleted(deviceId) self._scheduler.removeTasksForConfig(deviceId) + self._deviceGuids.pop(deviceId, None) def _updateConfig(self, cfg): """Update device configuration.""" @@ -660,6 +657,10 @@ def _updateConfig(self, cfg): configId = cfg.configId self.log.info("processing device config config-id=%s", configId) + guid = getattr(cfg, "_device_guid", None) + if guid is not None: + self._deviceGuids[configId] = guid + nextExpectedRuns = {} if configId in self._deviceloader.deviceIds: tasksToRemove = self._scheduler.getTasksForConfig(configId) @@ -747,18 +748,13 @@ def _maintenanceCallback(self, ignored=None): but afterward will self-schedule each run. """ try: - self.log.debug("performing periodic maintenance") - if not self.options.cycle: - ret = "No maintenance required" - elif getattr(self.preferences, "pauseUnreachableDevices", True): + if self.options.cycle and getattr( + self.preferences, "pauseUnreachableDevices", True + ): # TODO: handle different types of device issues - ret = yield self._pauseUnreachableDevices() - else: - ret = None - defer.returnValue(ret) + yield self._pauseUnreachableDevices() except Exception: self.log.exception("failure while running maintenance callback") - raise @defer.inlineCallbacks def _pauseUnreachableDevices(self): @@ -785,16 +781,13 @@ def _pauseUnreachableDevices(self): defer.returnValue(issues) - def runPostConfigTasks(self, result=None): + def runPostConfigTasks(self): """ Add post-startup tasks from the preferences. This may be called with the failure code as well. """ - if isinstance(result, Failure): - pass - - elif not self.addedPostStartupTasks: + if not self.addedPostStartupTasks: postStartupTasks = getattr( self.preferences, "postStartupTasks", lambda: [] ) diff --git a/Products/ZenCollector/scheduler.py b/Products/ZenCollector/scheduler.py index 114af272f8..69587abb30 100644 --- a/Products/ZenCollector/scheduler.py +++ b/Products/ZenCollector/scheduler.py @@ -316,6 +316,7 @@ def __init__(self, callableTaskFactory=CallableTaskFactory()): self._tasks = {} self._taskCallback = {} self._taskStats = {} + self._displaycounts = () self._callableTaskFactory = callableTaskFactory self._shuttingDown = False # create a cleanup task that will periodically sweep the @@ -754,9 +755,7 @@ def displayStatistics(self, verbose): totalStateStats.maxElapsedTime, stats.maxElapsedTime ) - log.info( - "Tasks: %d Successful_Runs: %d Failed_Runs: %d Missed_Runs: %d " - "Queued_Tasks: %d Running_Tasks: %d ", + counts = ( totalTasks, totalRuns, totalFailedRuns, @@ -764,6 +763,16 @@ def displayStatistics(self, verbose): self.executor.queued, self.executor.running, ) + if self._displaycounts != counts: + self._displaycounts = counts + logmethod = log.info + else: + logmethod = log.debug + logmethod( + "Tasks: %d Successful_Runs: %d Failed_Runs: %d " + "Missed_Runs: %d Queued_Tasks: %d Running_Tasks: %d ", + *counts + ) if verbose: buffer = "Task States Summary:\n" diff --git a/Products/ZenCollector/services/push.py b/Products/ZenCollector/services/push.py index 4f05a1aedf..413f309e2d 100644 --- a/Products/ZenCollector/services/push.py +++ b/Products/ZenCollector/services/push.py @@ -7,37 +7,15 @@ # ############################################################################## -from Acquisition import aq_parent -from twisted.internet import defer -from zope.component import getUtility - -from Products.ZenHub.interfaces import IBatchNotifier -from Products.ZenHub.services.Procrastinator import Procrastinate -from Products.ZenHub.zodb import onUpdate, onDelete -from Products.ZenModel.Device import Device -from Products.ZenModel.DeviceClass import DeviceClass +from Products.ZenHub.zodb import onUpdate from Products.ZenModel.PerformanceConf import PerformanceConf -from Products.ZenModel.privateobject import is_private -from Products.ZenModel.RRDTemplate import RRDTemplate from Products.ZenModel.ZenPack import ZenPack from Products.ZenUtils.AutoGCObjectReader import gc_cache_every -from Products.ZenUtils.picklezipper import Zipper - -from .error import trapException -from .optionsfilter import getOptionsFilter -class UpdateCollectorMixin: +class UpdateCollectorMixin(object): """Push data back to collection daemons.""" - def __init__(self): - # When about to notify daemons about device changes, wait for a little - # bit to batch up operations. - self._procrastinator = Procrastinate(self._pushConfig) - self._reconfigProcrastinator = Procrastinate(self._pushReconfigure) - - self._notifier = getUtility(IBatchNotifier) - @onUpdate(PerformanceConf) def perfConfUpdated(self, conf, event): with gc_cache_every(1000, db=self.dmd._p_jar._db): @@ -60,244 +38,3 @@ def zenPackUpdated(self, zenpack, event): self.log.warning( "Error notifying a listener of new classes" ) - - @onUpdate(Device) - def deviceUpdated(self, device, event): - with gc_cache_every(1000, db=self.dmd._p_jar._db): - self._notifyAll(device) - - @onUpdate(None) # Matches all - def notifyAffectedDevices(self, entity, event): - # FIXME: This is horrible - with gc_cache_every(1000, db=self.dmd._p_jar._db): - if isinstance(entity, self._getNotifiableClasses()): - self._reconfigureIfNotify(entity) - else: - if isinstance(entity, Device): - return - # Something else... mark the devices as out-of-date - template = None - while entity: - # Don't bother with privately managed objects; the ZenPack - # will handle them on its own - if is_private(entity): - return - # Walk up until you hit an organizer or a device - if isinstance(entity, RRDTemplate): - template = entity - if isinstance(entity, DeviceClass): - uid = (self.name(), self.instance) - devfilter = None - if template: - devfilter = _HasTemplate(template, self.log) - self._notifier.notify_subdevices( - entity, uid, self._notifyAll, devfilter - ) - break - if isinstance(entity, Device): - self._notifyAll(entity) - break - entity = aq_parent(entity) - - @onDelete(Device) - def deviceDeleted(self, device, event): - with gc_cache_every(1000, db=self.dmd._p_jar._db): - devid = device.id - collector = device.getPerformanceServer().getId() - # The invalidation is only sent to the collector where the - # deleted device was. - if collector == self.instance: - self.log.debug( - "Invalidation: Performing remote call to delete " - "device %s from collector %s", - devid, - self.instance, - ) - for listener in self.listeners: - listener.callRemote("deleteDevice", devid) - else: - self.log.debug( - "Invalidation: Skipping remote call to delete " - "device %s from collector %s", - devid, - self.instance, - ) - - def _notifyAll(self, device): - """Notify all instances (daemons) of a change for the device.""" - # procrastinator schedules a call to _pushConfig - self._procrastinator.doLater(device) - - def _pushConfig(self, device): - """Push device config and deletes to relevent collectors/instances.""" - deferreds = [] - - if self._perfIdFilter(device) and self._filterDevice(device): - proxies = trapException(self, self._createDeviceProxies, device) - if proxies: - trapException(self, self._postCreateDeviceProxy, proxies) - else: - proxies = None - - prev_collector = ( - device.dmd.Monitors.primaryAq().getPreviousCollectorForDevice( - device.id - ) - ) - for listener in self.listeners: - if not proxies: - if hasattr(device, "getPerformanceServer"): - # The invalidation is only sent to the previous and - # current collectors. - if self.instance in ( - prev_collector, - device.getPerformanceServer().getId(), - ): - self.log.debug( - "Invalidation: Performing remote call for " - "device %s on collector %s", - device.id, - self.instance, - ) - deferreds.append( - listener.callRemote("deleteDevice", device.id) - ) - else: - self.log.debug( - "Invalidation: Skipping remote call for " - "device %s on collector %s", - device.id, - self.instance, - ) - else: - deferreds.append( - listener.callRemote("deleteDevice", device.id) - ) - self.log.debug( - "Invalidation: Performing remote call for " - "device %s on collector %s", - device.id, - self.instance, - ) - else: - options = self.listenerOptions.get(listener, None) - deviceFilter = getOptionsFilter(options) - for proxy in proxies: - if deviceFilter(proxy): - deferreds.append( - self._sendDeviceProxy(listener, proxy) - ) - - return defer.DeferredList(deferreds) - - def _sendDeviceProxy(self, listener, proxy): - return listener.callRemote("updateDeviceConfig", proxy) - - # FIXME: Don't use _getNotifiableClasses, use @onUpdate(myclasses) - def _getNotifiableClasses(self): - """ - Return a tuple of classes. - - When any object of a type in the sequence is modified the collector - connected to the service will be notified to update its configuration. - - @rtype: tuple - """ - return () - - def _pushReconfigure(self, value): - """Notify the collector to reread the entire configuration.""" - # value is unused but needed for the procrastinator framework - for listener in self.listeners: - listener.callRemote("notifyConfigChanged") - self._reconfigProcrastinator.clear() - - def _reconfigureIfNotify(self, object): - ncc = self._notifyConfigChange(object) - self.log.debug( - "services/config.py _reconfigureIfNotify object=%r " - "_notifyConfigChange=%s", - object, - ncc, - ) - if ncc: - self.log.debug("scheduling collector reconfigure") - self._reconfigProcrastinator.doLater(True) - - def _notifyConfigChange(self, object): - """ - Called when an object of a type from _getNotifiableClasses is - encountered - - @return: should a notify config changed be sent - @rtype: boolean - """ - return True - - def sendDeviceConfigs(self, configs): - deferreds = [] - - def errback(failure): - self.log.critical( - "Unable to update configs for service instance %s: %s", - self.name(), - failure, - ) - - for listener in self.listeners: - options = self.listenerOptions.get(listener, None) - deviceFilter = getOptionsFilter(options) - filteredConfigs = filter(deviceFilter, configs) - args = Zipper.dump(filteredConfigs) - d = listener.callRemote("updateDeviceConfigs", args).addErrback( - errback - ) - deferreds.append(d) - return deferreds - - -class _HasTemplate(object): - """ - Predicate class that checks whether a given device has a template - matching the given template. - """ - - def __init__(self, template, log): - self.template = template - self.log = log - - def __call__(self, device): - if issubclass(self.template.getTargetPythonClass(), Device): - if self.template in device.getRRDTemplates(): - self.log.debug( - "%s bound to template %s", - device.getPrimaryId(), - self.template.getPrimaryId(), - ) - return True - else: - self.log.debug( - "%s not bound to template %s", - device.getPrimaryId(), - self.template.getPrimaryId(), - ) - return False - else: - # check components, Too expensive? - for comp in device.getMonitoredComponents( - type=self.template.getTargetPythonClass().meta_type - ): - if self.template in comp.getRRDTemplates(): - self.log.debug( - "%s bound to template %s", - comp.getPrimaryId(), - self.template.getPrimaryId(), - ) - return True - else: - self.log.debug( - "%s not bound to template %s", - comp.getPrimaryId(), - self.template.getPrimaryId(), - ) - return False diff --git a/Products/ZenCollector/tasks.py b/Products/ZenCollector/tasks.py index fa92c64aa3..5350c5c1eb 100644 --- a/Products/ZenCollector/tasks.py +++ b/Products/ZenCollector/tasks.py @@ -221,27 +221,6 @@ def reset(self): self.config = None -class RRDWriter(object): - def __init__(self, delegate): - self.delegate = delegate - - def writeRRD(self, counter, countervalue, countertype, **kwargs): - """ - write given data to RRD streaming files - """ - self.delegate.writeRRD(counter, countervalue, countertype, **kwargs) - - -class EventSender(object): - def __init__(self, delegate): - self.delegate = delegate - - def sendEvent(self, event, **eventData): - evt = event.copy() - evt.update(eventData) - self.delegate.sendEvent(evt) - - class TaskStates(object): STATE_IDLE = "IDLE" STATE_CONNECTING = "CONNECTING" diff --git a/Products/ZenCollector/tests/test_daemon.py b/Products/ZenCollector/tests/test_daemon.py index af8bfe42a4..fe7c6de7ee 100644 --- a/Products/ZenCollector/tests/test_daemon.py +++ b/Products/ZenCollector/tests/test_daemon.py @@ -1,4 +1,4 @@ -from mock import Mock, patch, create_autospec +from mock import ANY, Mock, patch, create_autospec from unittest import TestCase from Products.ZenHub.tests.mock_interface import create_interface_mock @@ -6,7 +6,6 @@ from Products.ZenCollector.daemon import ( CollectorDaemon, defer, - Failure, ICollectorPreferences, IConfigurationListener, ITaskSplitter, @@ -45,7 +44,7 @@ def test__maintenanceCallback(t): t.cd.log.debug.assert_called_with( "deviceIssues=%r", t.cd.getDevicePingIssues.return_value ) - t.assertEqual(ret.result, t.cd.getDevicePingIssues.return_value) + t.assertIsNone(ret.result) def test_ignores_unresponsive_devices(t): t.cd.log = Mock(name="log") @@ -62,7 +61,7 @@ def test_no_cycle_option(t): ret = t.cd._maintenanceCallback() - t.assertEqual(ret.result, "No maintenance required") + t.assertIsNone(ret.result) def test_handle_getDevicePingIssues_exception(t): t.cd.getDevicePingIssues.side_effect = Exception @@ -71,8 +70,8 @@ def test_handle_getDevicePingIssues_exception(t): ret = t.cd._maintenanceCallback() ret.addErrback(handler) - t.assertIsInstance(handler.err, Failure) - t.assertIsInstance(handler.err.value, Exception) + t.assertIsNone(handler.err) + t.cd.log.exception.assert_called_once_with(ANY) def test_handle__pauseUnreachableDevices_exception(t): t.cd._pauseUnreachableDevices = create_autospec( @@ -84,8 +83,8 @@ def test_handle__pauseUnreachableDevices_exception(t): ret = t.cd._maintenanceCallback() ret.addErrback(handler) - t.assertIsInstance(handler.err, Failure) - t.assertIsInstance(handler.err.value, Exception) + t.assertIsNone(handler.err) + t.cd.log.exception.assert_called_once_with(ANY) def test__pauseUnreachableDevices(t): t.cd._scheduler = Mock( diff --git a/Products/ZenCollector/utils/maintenance.py b/Products/ZenCollector/utils/maintenance.py index 8430540241..61b63488c7 100644 --- a/Products/ZenCollector/utils/maintenance.py +++ b/Products/ZenCollector/utils/maintenance.py @@ -10,7 +10,7 @@ import logging from twisted.internet import defer, reactor -from twisted.python.failure import Failure +from twisted.internet.task import LoopingCall from zenoss.protocols.protobufs.zep_pb2 import DaemonHeartbeat from zope.component import getUtility @@ -53,11 +53,11 @@ def heartbeat(self): daemon=self._daemon, timeout_seconds=self._timeout, ) - log.debug("sending heartbeat %s", heartbeat) publisher = getUtility(IQueuePublisher) publisher.publish( "$Heartbeats", "zenoss.heartbeat.%s" % heartbeat.monitor, heartbeat ) + log.debug("sent heartbeat %s", heartbeat) class ZenHubHeartbeatSender(object): @@ -82,61 +82,40 @@ class MaintenanceCycle(object): def __init__( self, cycleInterval, heartbeatSender=None, maintenanceCallback=None ): - self._cycleInterval = cycleInterval - self._heartbeatSender = heartbeatSender - self._callback = maintenanceCallback - self._stop = False + self.__interval = cycleInterval + self.__heartbeatSender = heartbeatSender + self.__callback = maintenanceCallback + self.__task = LoopingCall(self._maintenance) def start(self): - reactor.callWhenRunning(self._doMaintenance) + if self.__interval > 0: + interval = self.__interval + self.__task.start(interval, now=True) + else: + # maintenance is run only once if _interval <= 0. + interval = "run-once" + reactor.callWhenRunning(self._maintenance) + log.debug("maintenance started interval=%s", interval) def stop(self): + self.__task.stop() log.debug("maintenance stopped") - self._stop = True - def _doMaintenance(self): + @defer.inlineCallbacks + def _maintenance(self): """ - Perform daemon maintenance processing on a periodic schedule. Initially - called after the daemon configuration loader task is added, but - afterward will self-schedule each run. + Perform daemon maintenance processing on a periodic schedule. """ - if self._stop: - log.debug("skipping, maintenance stopped") - return - - log.info("performing periodic maintenance") - interval = self._cycleInterval - - def _maintenance(): - if self._heartbeatSender is not None: - log.debug("calling heartbeat sender") - d = defer.maybeDeferred(self._heartbeatSender.heartbeat) - d.addCallback(self._additionalMaintenance) - return d - else: - log.debug("skipping heartbeat: no sender configured") - return defer.maybeDeferred(self._additionalMaintenance) - - def _reschedule(result): - if isinstance(result, Failure): - # The full error message is actually the entire traceback, so - # just get the last line with the actual message. - log.error( - "maintenance failed. message from hub: (%s) %s", - result.type, result.getErrorMessage(), - ) - - if interval > 0: - log.debug("rescheduling maintenance in %ds", interval) - reactor.callLater(interval, self._doMaintenance) - - d = _maintenance() - d.addBoth(_reschedule) - - return d - - def _additionalMaintenance(self, result=None): - if self._callback: - log.debug("calling additional maintenance") - d = defer.maybeDeferred(self._callback, result) - return d + if self.__heartbeatSender is not None: + try: + yield self.__heartbeatSender.heartbeat() + log.debug("sent heartbeat") + except Exception: + log.exception("failed to send heartbeat") + if self.__callback is not None: + try: + yield self.__callback() + log.debug("executed maintenance callback") + except Exception: + log.exception("failed to execute maintenance callback") + log.debug("performed periodic maintanence") diff --git a/Products/ZenEvents/tests/test_zeneventd.py b/Products/ZenEvents/tests/test_zeneventd.py index c36108a699..10c1f4e607 100644 --- a/Products/ZenEvents/tests/test_zeneventd.py +++ b/Products/ZenEvents/tests/test_zeneventd.py @@ -2,34 +2,37 @@ from mock import patch, call, Mock, MagicMock from Products.ZenEvents.zeneventd import ( + CheckInputPipe, + Event, + EventContext, + EventPipelineProcessor, + time, Timeout, TimeoutError, - EventPipelineProcessor, - Event, ZepRawEvent, - CheckInputPipe, - EventContext, - time ) from Products.ZenEvents.events2.processing import EventProcessorPipe from zenoss.protocols.protobufs.zep_pb2 import EventActor, EventSeverity from zenoss.protocols.protobufs.model_pb2 import ModelElementType -PATH = {'zeneventd': 'Products.ZenEvents.zeneventd'} +PATH = {"zeneventd": "Products.ZenEvents.zeneventd"} class EventPipelineProcessorTest(TestCase): - def setUp(self): self.dmd = Mock() + self.log_patcher = patch( + "{zeneventd}.log".format(**PATH), autospec=True + ) self.manager_patcher = patch( - '{zeneventd}.Manager'.format(**PATH), autospec=True + "{zeneventd}.Manager".format(**PATH), autospec=True ) # silence 'new thread' error self.metric_reporter_patcher = patch( - '{zeneventd}.MetricReporter'.format(**PATH), autospec=True + "{zeneventd}.MetricReporter".format(**PATH), autospec=True ) + self.log_patcher.start() self.manager_patcher.start() self.metric_reporter_patcher.start() @@ -46,7 +49,7 @@ def setUp(self): element_sub_type_id=ModelElementType.COMPONENT, element_sub_identifier="zeneventd", ), - summary='Event Summary', + summary="Event Summary", severity=EventSeverity.SEVERITY_DEBUG, event_key="RMMonitor.collect.docker", agent="zenpython", @@ -55,11 +58,12 @@ def setUp(self): ) def tearDown(self): + self.log_patcher.stop() self.manager_patcher.stop() self.metric_reporter_patcher.stop() def test_processMessage(self): - self.epp._pipes = (CheckInputPipe(self.epp._manager), ) + self.epp._pipes = (CheckInputPipe(self.epp._manager),) zep_raw_event = self.epp.processMessage(self.message) @@ -69,7 +73,7 @@ def test_processMessage(self): def test_exception_in_pipe(self): error_pipe = self.ErrorPipe(self.epp._manager) - self.epp._pipes = (error_pipe, ) + self.epp._pipes = (error_pipe,) self.epp._pipe_timers[error_pipe.name] = MagicMock() zep_raw_event = self.epp.processMessage(self.message) @@ -82,12 +86,11 @@ def test_exception_in_pipe(self): ) self.assertEqual( - zep_raw_event.event.message, - exception_event.event.message + zep_raw_event.event.message, exception_event.event.message ) class ErrorPipe(EventProcessorPipe): - ERR = Exception('pipeline failure') + ERR = Exception("pipeline failure") def __call__(self, eventContext): raise self.ERR @@ -108,7 +111,7 @@ def test_synchronize_with_database_every_event(self): self.dmd._p_jar.sync.assert_called_once_with() def test_create_exception_event(self): - error = Exception('test exception') + error = Exception("test exception") event_context = self.epp.create_exception_event(self.message, error) self.assertIsInstance(event_context, EventContext) @@ -117,19 +120,17 @@ def test_create_exception_event(self): self.assertIsInstance(exception_event, Event) self.assertEqual( exception_event.summary, - "Internal exception processing event: Exception('test exception',)" - ) - self.assertTrue( - str(error) in exception_event.message + "Internal exception processing event: " + "Exception('test exception',)", ) + self.assertTrue(str(error) in exception_event.message) class TimeoutTest(TestCase): - def setUp(self): # Patch external dependencies self.signal_patcher = patch( - '{zeneventd}.signal'.format(**PATH), autospec=True + "{zeneventd}.signal".format(**PATH), autospec=True ) self.signal = self.signal_patcher.start() @@ -139,19 +140,17 @@ def tearDown(self): def test_context_manager(self): timeout_duration = 10 - with Timeout('event', timeout_duration) as ctx: + with Timeout("event", timeout_duration) as ctx: self.signal.signal.assert_called_with( self.signal.SIGALRM, ctx.handle_timeout ) - self.signal.alarm.assert_has_calls( - [call(timeout_duration), call(0)] - ) + self.signal.alarm.assert_has_calls([call(timeout_duration), call(0)]) def test_handle_timeout_raises_exception(self): with self.assertRaises(TimeoutError): with Timeout(1) as ctx: - ctx.handle_timeout(1, 'frame') + ctx.handle_timeout(1, "frame") class BaseQueueConsumerTaskTest(TestCase): diff --git a/Products/ZenEvents/zeneventd.py b/Products/ZenEvents/zeneventd.py index a3c5dbb8e8..399032dba7 100644 --- a/Products/ZenEvents/zeneventd.py +++ b/Products/ZenEvents/zeneventd.py @@ -21,22 +21,40 @@ from zenoss.protocols.interfaces import IAMQPConnectionInfo, IQueueSchema from zenoss.protocols.jsonformat import from_dict, to_dict from zenoss.protocols.protobufs.zep_pb2 import ( - STATUS_DROPPED, Event, ZepRawEvent + STATUS_DROPPED, + Event, + ZepRawEvent, ) from Products.ZenCollector.utils.maintenance import ( - MaintenanceCycle, QueueHeartbeatSender, maintenanceBuildOptions + MaintenanceCycle, + QueueHeartbeatSender, + maintenanceBuildOptions, ) from Products.ZenEvents.daemonlifecycle import ( - BuildOptionsEvent, DaemonCreatedEvent, DaemonStartRunEvent, SigTermEvent, - SigUsr1Event + BuildOptionsEvent, + DaemonCreatedEvent, + DaemonStartRunEvent, + SigTermEvent, + SigUsr1Event, ) from Products.ZenEvents.events2.processing import ( - AddDeviceContextAndTagsPipe, AssignDefaultEventClassAndTagPipe, - CheckHeartBeatPipe, CheckInputPipe, ClearClassRefreshPipe, DropEvent, - EventContext, EventPluginPipe, FingerprintPipe, IdentifierPipe, Manager, - ProcessingException, SerializeContextPipe, TransformAndReidentPipe, - TransformPipe, UpdateDeviceContextAndTagsPipe + AddDeviceContextAndTagsPipe, + AssignDefaultEventClassAndTagPipe, + CheckHeartBeatPipe, + CheckInputPipe, + ClearClassRefreshPipe, + DropEvent, + EventContext, + EventPluginPipe, + FingerprintPipe, + IdentifierPipe, + Manager, + ProcessingException, + SerializeContextPipe, + TransformAndReidentPipe, + TransformPipe, + UpdateDeviceContextAndTagsPipe, ) from Products.ZenEvents.interfaces import IPostEventPlugin, IPreEventPlugin from Products.ZenMessaging.queuemessaging.interfaces import IQueueConsumerTask @@ -51,9 +69,11 @@ def monkey_patch_rotatingfilehandler(): try: from cloghandler import ConcurrentRotatingFileHandler + logging.handlers.RotatingFileHandler = ConcurrentRotatingFileHandler except ImportError: from warnings import warn + warn( "ConcurrentLogHandler package not installed. Using" " RotatingFileLogHandler. While everything will still work fine," @@ -65,8 +85,8 @@ def monkey_patch_rotatingfilehandler(): log = logging.getLogger("zen.eventd") -EXCHANGE_ZEP_ZEN_EVENTS = '$ZepZenEvents' -QUEUE_RAW_ZEN_EVENTS = '$RawZenEvents' +EXCHANGE_ZEP_ZEN_EVENTS = "$ZepZenEvents" +QUEUE_RAW_ZEN_EVENTS = "$RawZenEvents" class EventPipelineProcessor(object): @@ -79,7 +99,7 @@ def __init__(self, dmd): self._manager = Manager(self.dmd) self._pipes = ( EventPluginPipe( - self._manager, IPreEventPlugin, 'PreEventPluginPipe' + self._manager, IPreEventPlugin, "PreEventPluginPipe" ), CheckInputPipe(self._manager), IdentifierPipe(self._manager), @@ -91,23 +111,23 @@ def __init__(self, dmd): UpdateDeviceContextAndTagsPipe(self._manager), IdentifierPipe(self._manager), AddDeviceContextAndTagsPipe(self._manager), - ] + ], ), AssignDefaultEventClassAndTagPipe(self._manager), FingerprintPipe(self._manager), SerializeContextPipe(self._manager), EventPluginPipe( - self._manager, IPostEventPlugin, 'PostEventPluginPipe' + self._manager, IPostEventPlugin, "PostEventPluginPipe" ), ClearClassRefreshPipe(self._manager), - CheckHeartBeatPipe(self._manager) + CheckHeartBeatPipe(self._manager), ) self._pipe_timers = {} for pipe in self._pipes: timer_name = pipe.name self._pipe_timers[timer_name] = Metrology.timer(timer_name) - self.reporter = MetricReporter(prefix='zenoss.zeneventd.') + self.reporter = MetricReporter(prefix="zenoss.zeneventd.") self.reporter.start() if not self.SYNC_EVERY_EVENT: @@ -133,8 +153,9 @@ class when it is done with the message eventContext = EventContext(log, zepevent) with Timeout( - zepevent, self.PROCESS_EVENT_TIMEOUT, - error_message='while processing event' + zepevent, + self.PROCESS_EVENT_TIMEOUT, + error_message="while processing event", ): for pipe in self._pipes: with self._pipe_timers[pipe.name]: @@ -142,12 +163,13 @@ class when it is done with the message if log.isEnabledFor(logging.DEBUG): # assume to_dict() is expensive. log.debug( - 'After pipe %s, event context is %s', - pipe.name, to_dict(eventContext.zepRawEvent) + "After pipe %s, event context is %s", + pipe.name, + to_dict(eventContext.zepRawEvent), ) if eventContext.event.status == STATUS_DROPPED: raise DropEvent( - 'Dropped by %s' % pipe, eventContext.event + "Dropped by %s" % pipe, eventContext.event ) except AttributeError: @@ -167,7 +189,7 @@ class when it is done with the message except Exception as error: log.info( "Failed to process event, forward original raw event: %s", - to_dict(zepevent.event) + to_dict(zepevent.event), ) # Pipes and plugins may raise ProcessingException's for their own # reasons. only log unexpected exceptions of other type @@ -179,15 +201,17 @@ class when it is done with the message if log.isEnabledFor(logging.DEBUG): # assume to_dict() is expensive. - log.debug("Publishing event: %s", to_dict(eventContext.zepRawEvent)) + log.debug( + "Publishing event: %s", to_dict(eventContext.zepRawEvent) + ) return eventContext.zepRawEvent def _synchronize_with_database(self): - '''sync() db if it has been longer than + """sync() db if it has been longer than self.syncInterval seconds since the last time, and no _synchronize has not been called for self.syncInterval seconds KNOWN ISSUE: ZEN-29884 - ''' + """ if self.SYNC_EVERY_EVENT: doSync = True else: @@ -204,23 +228,23 @@ def create_exception_event(self, message, exception): orig_zep_event = ZepRawEvent() orig_zep_event.event.CopyFrom(message) failure_event = { - 'uuid': guid.generate(), - 'created_time': int(time() * 1000), - 'fingerprint': - '|'.join(['zeneventd', 'processMessage', repr(exception)]), + "uuid": guid.generate(), + "created_time": int(time() * 1000), + "fingerprint": "|".join( + ["zeneventd", "processMessage", repr(exception)] + ), # Don't send the *same* event class or we loop endlessly - 'eventClass': '/', - 'summary': 'Internal exception processing event: %r' % exception, - 'message': - 'Internal exception processing event: %r/%s' % - (exception, to_dict(orig_zep_event.event)), - 'severity': 4, + "eventClass": "/", + "summary": "Internal exception processing event: %r" % exception, + "message": "Internal exception processing event: %r/%s" + % (exception, to_dict(orig_zep_event.event)), + "severity": 4, } zep_raw_event = ZepRawEvent() zep_raw_event.event.CopyFrom(from_dict(Event, failure_event)) event_context = EventContext(log, zep_raw_event) - event_context.eventProxy.device = 'zeneventd' - event_context.eventProxy.component = 'processMessage' + event_context.eventProxy.device = "zeneventd" + event_context.eventProxy.component = "processMessage" return event_context @@ -231,18 +255,19 @@ class BaseQueueConsumerTask(object): def __init__(self, processor): self.processor = processor self._queueSchema = getUtility(IQueueSchema) - self.dest_routing_key_prefix = 'zenoss.zenevent' + self.dest_routing_key_prefix = "zenoss.zenevent" self._dest_exchange = self._queueSchema.getExchange( EXCHANGE_ZEP_ZEN_EVENTS ) def _routing_key(self, event): - return (self.dest_routing_key_prefix + - event.event.event_class.replace('/', '.').lower()) + return ( + self.dest_routing_key_prefix + + event.event.event_class.replace("/", ".").lower() + ) class TwistedQueueConsumerTask(BaseQueueConsumerTask): - def __init__(self, processor): BaseQueueConsumerTask.__init__(self, processor) self.queue = self._queueSchema.getQueue(QUEUE_RAW_ZEN_EVENTS) @@ -264,16 +289,16 @@ def processMessage(self, message): EXCHANGE_ZEP_ZEN_EVENTS, self._routing_key(zepRawEvent), zepRawEvent, - declareExchange=False + declareExchange=False, ) yield self.queueConsumer.acknowledge(message) except DropEvent as e: if log.isEnabledFor(logging.DEBUG): # assume to_dict() is expensive. - log.debug('%s - %s', e.message, to_dict(e.event)) + log.debug("%s - %s", e.message, to_dict(e.event)) yield self.queueConsumer.acknowledge(message) except ProcessingException as e: - log.error('%s - %s', e.message, to_dict(e.event)) + log.error("%s - %s", e.message, to_dict(e.event)) log.exception(e) yield self.queueConsumer.reject(message) except Exception as e: @@ -296,7 +321,7 @@ def run(self): reactor.run() def _start(self): - reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) + reactor.addSystemEventTrigger("before", "shutdown", self._shutdown) self._consumer.run() @defer.inlineCallbacks @@ -315,20 +340,21 @@ def getConfig(self): class ZenEventD(ZCmdBase): - def __init__(self, *args, **kwargs): super(ZenEventD, self).__init__(*args, **kwargs) EventPipelineProcessor.SYNC_EVERY_EVENT = self.options.syncEveryEvent - EventPipelineProcessor.PROCESS_EVENT_TIMEOUT = self.options.process_event_timeout + EventPipelineProcessor.PROCESS_EVENT_TIMEOUT = ( + self.options.process_event_timeout + ) self._heartbeatSender = QueueHeartbeatSender( - 'localhost', 'zeneventd', self.options.heartbeatTimeout + "localhost", "zeneventd", self.options.heartbeatTimeout ) self._maintenanceCycle = MaintenanceCycle( self.options.maintenancecycle, self._heartbeatSender ) objectEventNotify(DaemonCreatedEvent(self)) config = ZenEventDConfig(self.options) - provideUtility(config, IDaemonConfig, 'zeneventd_config') + provideUtility(config, IDaemonConfig, "zeneventd_config") def sigTerm(self, signum=None, frame=None): log.info("Shutting down...") @@ -343,52 +369,69 @@ def run(self): def sighandler_USR1(self, signum, frame): super(ZenEventD, self).sighandler_USR1(signum, frame) - log.debug('sighandler_USR1 called %s', signum) + log.debug("sighandler_USR1 called %s", signum) objectEventNotify(SigUsr1Event(self, signum)) def buildOptions(self): super(ZenEventD, self).buildOptions() maintenanceBuildOptions(self.parser) self.parser.add_option( - '--synceveryevent', dest='syncEveryEvent', - action="store_true", default=False, - help=('Force sync() before processing every event; default is' - ' to sync() no more often than once every 1/2 second.') + "--synceveryevent", + dest="syncEveryEvent", + action="store_true", + default=False, + help=( + "Force sync() before processing every event; default is" + " to sync() no more often than once every 1/2 second." + ), ) self.parser.add_option( - '--process-event-timeout', dest='process_event_timeout', - type='int', default=0, - help=('Set the Timeout(in seconds) for processing each event.' - ' The timeout may be extended for a transforms using,' - 'signal.alarm() in the transform' - 'set to 0 to disable') + "--process-event-timeout", + dest="process_event_timeout", + type="int", + default=0, + help=( + "Set the Timeout(in seconds) for processing each event." + " The timeout may be extended for a transforms using," + "signal.alarm() in the transform" + "set to 0 to disable" + ), ) self.parser.add_option( - '--messagesperworker', dest='messagesPerWorker', default=1, + "--messagesperworker", + dest="messagesPerWorker", + default=1, type="int", - help=('Sets the number of messages each worker gets from the queue' - ' at any given time. Default is 1. Change this only if event' - ' processing is deemed slow. Note that increasing the value' - ' increases the probability that events will be processed' - ' out of order.') + help=( + "Sets the number of messages each worker gets from the queue" + " at any given time. Default is 1. Change this only if event" + " processing is deemed slow. Note that increasing the value" + " increases the probability that events will be processed" + " out of order." + ), ) self.parser.add_option( - '--maxpickle', dest='maxpickle', default=100, type="int", - help=('Sets the number of pickle files in' - ' var/zeneventd/failed_transformed_events.') + "--maxpickle", + dest="maxpickle", + default=100, + type="int", + help=( + "Sets the number of pickle files in" + " var/zeneventd/failed_transformed_events." + ), ) self.parser.add_option( - '--pickledir', dest='pickledir', - default=zenPath('var/zeneventd/failed_transformed_events'), + "--pickledir", + dest="pickledir", + default=zenPath("var/zeneventd/failed_transformed_events"), type="string", - help='Sets the path to save pickle files.' + help="Sets the path to save pickle files.", ) objectEventNotify(BuildOptionsEvent(self)) class Timeout: - - def __init__(self, event, seconds=1, error_message='Timeout'): + def __init__(self, event, seconds=1, error_message="Timeout"): self.seconds = seconds self.error_message = error_message self.event = event @@ -406,14 +449,14 @@ def __exit__(self, type, value, traceback): class TimeoutError(Exception): - def __init__(self, message, event=None): super(TimeoutError, self).__init__(message) self.event = event -if __name__ == '__main__': +if __name__ == "__main__": # explicit import of ZenEventD to activate enterprise extensions - from Products.ZenEvents.zeneventd import ZenEventD + from Products.ZenEvents.zeneventd import ZenEventD # noqa F811 + zed = ZenEventD() zed.run() diff --git a/Products/ZenEvents/zentrap.py b/Products/ZenEvents/zentrap.py index 3eebca92f6..02dc51bb81 100644 --- a/Products/ZenEvents/zentrap.py +++ b/Products/ZenEvents/zentrap.py @@ -27,7 +27,6 @@ from pynetsnmp import netsnmp, twistedsnmp from twisted.internet import defer, reactor -from twisted.python.failure import Failure from zope.component import queryUtility, getUtility, provideUtility from zope.interface import implementer @@ -1012,22 +1011,23 @@ def _initializeTrapFilter(self): self.setExitCode(1) self.stop() - def runPostConfigTasks(self, result=None): + @defer.inlineCallbacks + def runPostConfigTasks(self): # 1) super sets self._prefs.task with the call to postStartupTasks # 2) call remote createAllUsers # 3) service in turn walks DeviceClass tree and returns users - CollectorDaemon.runPostConfigTasks(self, result) - if not isinstance(result, Failure) and self._prefs.task is not None: - service = self.getRemoteConfigServiceProxy() - log.debug('TrapDaemon.runPostConfigTasks callRemote createAllUsers') - d = service.callRemote("createAllUsers") - d.addCallback(self._createUsers) + super(TrapDaemon, self).runPostConfigTasks() + if self._prefs.task is not None: + service = yield self.getRemoteConfigServiceProxy() + log.debug('callRemote createAllUsers') + users = yield service.callRemote('createAllUsers') + self._createUsers(users) def remote_createUser(self, user): reactor.callInThread(self._createUsers, [user]) def _createUsers(self, users): - log.debug('TrapDaemon._createUsers %s users', len(users)) + log.debug('_createUsers %s users', len(users)) if self._prefs.task.session is None: log.debug("No session created, so unable to create users") else: diff --git a/Products/ZenHub/HubService.py b/Products/ZenHub/HubService.py index 97b259de7c..b187822197 100644 --- a/Products/ZenHub/HubService.py +++ b/Products/ZenHub/HubService.py @@ -16,7 +16,7 @@ from Products.ZenUtils.deprecated import deprecated -class HubService(pb.Referenceable): +class HubService(object, pb.Referenceable): """ The base class for a ZenHub service class. diff --git a/Products/ZenHub/services/ProcessConfig.py b/Products/ZenHub/services/ProcessConfig.py index 965c9d5e51..f6296c9e8c 100644 --- a/Products/ZenHub/services/ProcessConfig.py +++ b/Products/ZenHub/services/ProcessConfig.py @@ -16,10 +16,6 @@ from Products.ZenCollector.services.config import CollectorConfigService from Products.ZenEvents import Event -from Products.ZenHub.zodb import onUpdate -from Products.ZenModel.OSProcessClass import OSProcessClass -from Products.ZenModel.OSProcessOrganizer import OSProcessOrganizer -from Products.Zuul.catalog.interfaces import IModelCatalogTool # DeviceProxy must be present for twisted PB serialization to work. from Products.ZenCollector.services.config import DeviceProxy # noqa F401 @@ -168,39 +164,6 @@ def _createDeviceProxy(self, device): if proxy.processes: return proxy - @onUpdate(OSProcessClass) - def processClassUpdated(self, object, event): - devices = set() - for process in object.instances(): - device = process.device() - if not device: - continue - device = device.primaryAq() - device_path = device.getPrimaryUrlPath() - if device_path not in devices: - self._notifyAll(device) - devices.add(device_path) - - @onUpdate(OSProcessOrganizer) - def processOrganizerUpdated(self, object, event): - catalog = IModelCatalogTool(object.primaryAq()) - results = catalog.search(OSProcessClass) - if not results.total: - return - devices = set() - for organizer in results: - if results.areBrains: - organizer = organizer.getObject() - for process in organizer.instances(): - device = process.device() - if not device: - continue - device = device.primaryAq() - device_path = device.getPrimaryUrlPath() - if device_path not in devices: - self._notifyAll(device) - devices.add(device_path) - if __name__ == "__main__": from Products.ZenHub.ServiceTester import ServiceTester diff --git a/Products/ZenHub/services/SnmpTrapConfig.py b/Products/ZenHub/services/SnmpTrapConfig.py index 5d71d825b6..ac4518ed90 100644 --- a/Products/ZenHub/services/SnmpTrapConfig.py +++ b/Products/ZenHub/services/SnmpTrapConfig.py @@ -7,22 +7,21 @@ # ############################################################################## -from __future__ import print_function - """SnmpTrapConfig Provides configuration for an OID translation service. """ +from __future__ import print_function + import logging from twisted.spread import pb from Products.ZenCollector.services.config import CollectorConfigService -from Products.ZenHub.zodb import onUpdate, onDelete +from Products.ZenHub.zodb import onUpdate from Products.ZenModel.DeviceClass import DeviceClass from Products.ZenModel.Device import Device -from Products.ZenModel.MibBase import MibBase from Products.Zuul.catalog.interfaces import IModelCatalogTool log = logging.getLogger("zen.HubService.SnmpTrapConfig") @@ -140,20 +139,6 @@ def deviceClassUpdated(self, object, event): def deviceUpdated(self, object, event): self._objectUpdated(object) - @onUpdate(MibBase) - def mibsChanged(self, device, event): - for listener in self.listeners: - listener.callRemote("notifyConfigChanged") - - @onUpdate(None) # Matches all - def notifyAffectedDevices(self, object, event): - pass - - @onDelete(MibBase) - def mibsDeleted(self, device, event): - for listener in self.listeners: - listener.callRemote("notifyConfigChanged") - if __name__ == "__main__": from pprint import pprint diff --git a/Products/ZenHub/services/ThresholdMixin.py b/Products/ZenHub/services/ThresholdMixin.py index a9f01adf34..dd16cd6c5e 100644 --- a/Products/ZenHub/services/ThresholdMixin.py +++ b/Products/ZenHub/services/ThresholdMixin.py @@ -16,7 +16,7 @@ log = logging.getLogger("zen.thresholdmixin") -class ThresholdMixin: +class ThresholdMixin(object): _cached_thresholdClasses = [] @translateError diff --git a/Products/ZenStatus/zenping.py b/Products/ZenStatus/zenping.py index 87461c0971..bcb07f11b5 100644 --- a/Products/ZenStatus/zenping.py +++ b/Products/ZenStatus/zenping.py @@ -45,8 +45,8 @@ class PingDaemon(CollectorDaemon): - def runPostConfigTasks(self, result=None): - CollectorDaemon.runPostConfigTasks(self, result=result) + def runPostConfigTasks(self): + super(PingDaemon, self).runPostConfigTasks() self.preferences.runPostConfigTasks() From 991b3c625d57e4ebb1c5ee5f5a6d0fdec627089c Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 19 Jan 2024 13:42:19 -0600 Subject: [PATCH 038/147] Ensure builder service loads the correct logging configuration. ZEN-34643 --- Products/Jobber/log.py | 47 ++++++++++++++++++------------- Products/Jobber/tests/test_log.py | 13 +++++---- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/Products/Jobber/log.py b/Products/Jobber/log.py index 7566b4168c..6605f0346b 100644 --- a/Products/Jobber/log.py +++ b/Products/Jobber/log.py @@ -97,8 +97,7 @@ "propagate": False, }, } -_configcache_loggers = { -} +_configcache_loggers = {} _main_handler = { "formatter": "main", @@ -118,13 +117,19 @@ "mode": "a", } -_main_filename = os.path.join(ZenJobs.get("logpath"), "zenjobs.log") -_beat_filename = os.path.join(ZenJobs.get("logpath"), "zenjobs-scheduler.log") -_configcache_filename = os.path.join( - ZenJobs.get("logpath"), "configcache_builder.log" -) - -_loglevelconf_filepath = zenPath("etc", "zenjobs_log_levels.conf") +_logpath = ZenJobs.get("logpath") +_filenames = { + "zenjobs": os.path.join(_logpath, "zenjobs.log"), + "beat": os.path.join(_logpath, "zenjobs-scheduler.log"), + "configcache_builder": os.path.join(_logpath, "configcache-builder.log"), +} +_loglevel_confs = { + "zenjobs": zenPath("etc", "zenjobs_log_levels.conf"), + "beat": zenPath("etc", "zenjobs_log_levels.conf"), + "configcache_builder": zenPath( + "etc", "configcache_builder_log_levels.conf" + ), +} def _get_logger(name=None): @@ -135,27 +140,25 @@ def _get_logger(name=None): return get_logger(name) -def configure_logging(logfile="main", **kw): +def configure_logging(logfile, **kw): """Configure logging for zenjobs.""" - # NOTE: Cleverly using the `-f` command line argument to specify + # NOTE: Cleverly used the `-f` command line argument to specify # which logging configuration to use. - if logfile in ("main", "configcache"): + if logfile in ("zenjobs", "configcache_builder"): _default_config["loggers"].update(**_main_loggers) _default_config["root"]["handlers"].append("main") - if logfile == "main": - _main_handler["filename"] = _main_filename - else: - _main_handler["filename"] = _configcache_filename + _main_handler["filename"] = _filenames[logfile] _default_config["handlers"]["main"] = _main_handler elif logfile == "beat": _default_config["root"]["handlers"].append("beat") - _beat_handler["filename"] = _beat_filename + _beat_handler["filename"] = _filenames[logfile] _default_config["handlers"]["beat"] = _beat_handler logging.config.dictConfig(_default_config) - if os.path.exists(_loglevelconf_filepath): - levelconfig = load_log_level_config(_loglevelconf_filepath) + loglevelconf_filename = _loglevel_confs[logfile] + if os.path.exists(loglevelconf_filename): + levelconfig = load_log_level_config(loglevelconf_filename) apply_levels(levelconfig) stdout_logger = logging.getLogger("STDOUT") @@ -343,8 +346,12 @@ class LogLevelUpdater(object): @classmethod def start(cls): + logfilename = logging._handlers.get("main").baseFilename + name = ( + logfilename.rsplit("/", 1)[-1].rsplit(".", 1)[0].replace("-", "_") + ) if cls.instance is None: - cls.instance = _LogLevelUpdaterThread(_loglevelconf_filepath) + cls.instance = _LogLevelUpdaterThread(_loglevel_confs[name]) cls.instance.start() elif not cls.instance.is_alive(): cls.instance = None diff --git a/Products/Jobber/tests/test_log.py b/Products/Jobber/tests/test_log.py index 86f17e4391..b326b39816 100644 --- a/Products/Jobber/tests/test_log.py +++ b/Products/Jobber/tests/test_log.py @@ -20,7 +20,7 @@ apply_levels, configure_logging, load_log_level_config, - _loglevelconf_filepath, + _loglevel_confs, ) from .utils import LoggingLayer @@ -73,10 +73,11 @@ def test_nominal( exists.return_value = True - configure_logging() + configure_logging("zenjobs") - exists.assert_called_once_with(_loglevelconf_filepath) - _load_log_level_config.assert_called_once_with(_loglevelconf_filepath) + loglevel_confname = _loglevel_confs["zenjobs"] + exists.assert_called_once_with(loglevel_confname) + _load_log_level_config.assert_called_once_with(loglevel_confname) _apply_levels.assert_called_once_with(levelConfig) getLogger.assert_has_calls(getLogger_calls, any_order=True) @@ -128,9 +129,9 @@ def test_missing_loglevel_file( exists.return_value = False - configure_logging() + configure_logging("zenjobs") - exists.assert_called_once_with(_loglevelconf_filepath) + exists.assert_called_once_with(_loglevel_confs["zenjobs"]) _load_log_level_config.assert_has_calls([]) _apply_levels.assert_has_calls([]) From 8fce583d758cb061c3f6ef8fea9c126ec71f8511 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 22 Jan 2024 15:17:44 -0600 Subject: [PATCH 039/147] Restored some CollectorDaemon APIs that ZenPacks depended on. ZEN-34651 --- Products/ZenCollector/config/task.py | 2 +- Products/ZenCollector/daemon.py | 40 ++++++++++++------- Products/ZenCollector/services/config.py | 2 +- Products/ZenCollector/services/push.py | 2 +- Products/ZenHub/PBDaemon.py | 18 ++++++--- Products/ZenHub/services/ConfigCache.py | 2 +- Products/ZenHub/services/DiscoverService.py | 2 +- Products/ZenHub/services/EventService.py | 2 +- Products/ZenHub/services/ModelerService.py | 2 +- Products/ZenHub/services/PerformanceConfig.py | 6 ++- Products/ZenHub/services/ThresholdMixin.py | 2 +- 11 files changed, 50 insertions(+), 30 deletions(-) diff --git a/Products/ZenCollector/config/task.py b/Products/ZenCollector/config/task.py index e1ff0b341b..ee320b7a47 100644 --- a/Products/ZenCollector/config/task.py +++ b/Products/ZenCollector/config/task.py @@ -131,7 +131,7 @@ def _processPropertyItems(self, propertyItems): def _processThresholdClasses(self, thresholdClasses): log.debug("processing received threshold classes") if thresholdClasses: - self._collector._loadThresholdClasses(thresholdClasses) + self._collector.loadThresholdClasses(thresholdClasses) def _processThresholds(self, thresholds): log.debug("processing received thresholds") diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index ece0d1e6a7..c4240e3e86 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -32,7 +32,7 @@ from Products.ZenUtils import metrics from Products.ZenUtils.deprecated import deprecated from Products.ZenUtils.observable import ObservableProxy -from Products.ZenUtils.Utils import importClass, load_config +from Products.ZenUtils.Utils import load_config from .config import DeviceConfigLoader from .interfaces import ( @@ -162,6 +162,7 @@ def __init__( self._device_config_update_interval = 300 self._deviceGuids = {} + self._devices = set() # deprecated; kept for vSphere ZP compatibility self._unresponsiveDevices = set() self._rrd = None self.reconfigureTimeout = None @@ -266,6 +267,16 @@ def parseOptions(self): super(CollectorDaemon, self).parseOptions() self.preferences.options = self.options + # @deprecated + def getInitialServices(self): + # Retained for compatibility with ZenPacks fixing CollectorDaemon's old + # behavior regarding the `initialServices` attribute. This new + # CollectorDaemon respects changes made to the `initialServices` + # attribute by subclasses, so the reason for overriding this method + # is no longer valid. However, for this method must continue to exist + # to avoid AttributeError exceptions. + return self.initialServices + def watchdogCycleTime(self): """ Return our cycle time (in minutes) @@ -384,11 +395,9 @@ def getRemoteConfigCacheProxy(self): proxy = yield self.getService("ConfigCache") defer.returnValue(proxy) - @defer.inlineCallbacks def getRemoteConfigServiceProxy(self): """Return the remote configuration service proxy object.""" - proxy = yield self.getService(self.preferences.configurationService) - defer.returnValue(proxy) + return self.getServiceNow(self.preferences.configurationService) def generateEvent(self, event, **kw): eventCopy = super(CollectorDaemon, self).generateEvent(event, **kw) @@ -637,12 +646,18 @@ def _deleteDevice(self, deviceId): self._configListener.deleted(deviceId) self._scheduler.removeTasksForConfig(deviceId) self._deviceGuids.pop(deviceId, None) + self._devices.discard(deviceId) def _updateConfig(self, cfg): - """Update device configuration.""" + """ + Update device configuration. + + Returns True if the configuration was processed, otherwise, + False is returned. + """ # guard against parsing updates during a disconnect if cfg is None: - return + return False configFilter = getattr(self.preferences, "configFilter", _always_ok) if not ( @@ -652,7 +667,7 @@ def _updateConfig(self, cfg): self.log.info( "filtered out device config config-id=%s", cfg.configId ) - return + return False configId = cfg.configId self.log.info("processing device config config-id=%s", configId) @@ -673,6 +688,7 @@ def _updateConfig(self, cfg): self._scheduler.removeTasks(task.name for task in tasksToRemove) self._configListener.updated(cfg) else: + self._devices.add(configId) self._configListener.added(cfg) newTasks = self._taskSplitter.splitConfiguration([cfg]) @@ -715,6 +731,8 @@ def _updateConfig(self, cfg): self.log.debug("pausing tasks for device %s", configId) self._scheduler.pauseTasksForConfig(configId) + return True + def setPropertyItems(self, items): """Override so that preferences are updated.""" super(CollectorDaemon, self).setPropertyItems(items) @@ -728,14 +746,6 @@ def _setCollectorPreferences(self, preferenceItems): self.log.debug("updated %s preference to %s", name, value) setattr(self.preferences, name, value) - def _loadThresholdClasses(self, thresholdClasses): - for c in thresholdClasses: - try: - importClass(c) - self.log.info("imported threshold class class=%r", c) - except ImportError: - self.log.exception("unable to import class %s", c) - def _configureThresholds(self, thresholds): self.getThresholds().updateList(thresholds) diff --git a/Products/ZenCollector/services/config.py b/Products/ZenCollector/services/config.py index d37a3298dd..10b9d800be 100644 --- a/Products/ZenCollector/services/config.py +++ b/Products/ZenCollector/services/config.py @@ -15,8 +15,8 @@ from twisted.spread import pb from ZODB.transact import transact +from Products.ZenHub.errors import translateError from Products.ZenHub.HubService import HubService -from Products.ZenHub.PBDaemon import translateError from Products.ZenHub.services.ThresholdMixin import ThresholdMixin from Products.ZenModel.Device import Device from Products.ZenUtils.guid.interfaces import IGlobalIdentifier diff --git a/Products/ZenCollector/services/push.py b/Products/ZenCollector/services/push.py index 413f309e2d..db03912c63 100644 --- a/Products/ZenCollector/services/push.py +++ b/Products/ZenCollector/services/push.py @@ -36,5 +36,5 @@ def zenPackUpdated(self, zenpack, event): ) except Exception: self.log.warning( - "Error notifying a listener of new classes" + "Error notifying a listener of new threshold classes" ) diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index c48439153c..b643b41b34 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -36,6 +36,7 @@ MetricWriter, ThresholdNotifier, ) +from Products.ZenUtils.Utils import importClass, lookupClass from Products.ZenUtils.ZenDaemon import ZenDaemon from .errors import HubDown, translateError @@ -431,14 +432,21 @@ def remote_setPropertyItems(self, items): @translateError def remote_updateThresholdClasses(self, classes): - from Products.ZenUtils.Utils import importClass + self.loadThresholdClasses(classes) - for c in classes: + def loadThresholdClasses(self, classnames): + for name in classnames: try: - importClass(c) - self.log.info("imported threshold class class=%r", c) + cls = lookupClass(name) + if cls: + self.log.debug( + "already imported threshold class class=%s", name + ) + continue + importClass(name) + self.log.info("imported threshold class class=%s", name) except ImportError: - self.log.error("unable to import class %s", c) + self.log.exception("unable to import threshold class %s", name) def buildOptions(self): super(PBDaemon, self).buildOptions() diff --git a/Products/ZenHub/services/ConfigCache.py b/Products/ZenHub/services/ConfigCache.py index 98cf8df38b..924a67592e 100644 --- a/Products/ZenHub/services/ConfigCache.py +++ b/Products/ZenHub/services/ConfigCache.py @@ -13,8 +13,8 @@ from Products.ZenCollector.configcache.cache import ConfigQuery from Products.ZenCollector.interfaces import IConfigurationDispatchingFilter +from Products.ZenHub.errors import translateError from Products.ZenHub.HubService import HubService -from Products.ZenHub.PBDaemon import translateError from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl diff --git a/Products/ZenHub/services/DiscoverService.py b/Products/ZenHub/services/DiscoverService.py index f91c87cb77..06d12e4ec4 100644 --- a/Products/ZenHub/services/DiscoverService.py +++ b/Products/ZenHub/services/DiscoverService.py @@ -18,13 +18,13 @@ from Products.Jobber.exceptions import NoSuchJobException from Products.ZenEvents.ZenEventClasses import Status_Ping -from Products.ZenHub.PBDaemon import translateError from Products.ZenModel.Device import manage_createDevice from Products.ZenModel.Exceptions import DeviceExistsError from Products.ZenRelations.ZenPropertyManager import iszprop from Products.ZenRelations.zPropertyCategory import getzPropertyCategory from Products.ZenUtils.IpUtil import strip, ipunwrap, isip +from ..errors import translateError from .ModelerService import ModelerService DEFAULT_PING_THRESH = 168 diff --git a/Products/ZenHub/services/EventService.py b/Products/ZenHub/services/EventService.py index af1b92ee14..73b37632fa 100644 --- a/Products/ZenHub/services/EventService.py +++ b/Products/ZenHub/services/EventService.py @@ -13,8 +13,8 @@ from zenoss.protocols.services import ServiceConnectionError from Products.ZenEvents.Event import Event +from Products.ZenHub.errors import translateError from Products.ZenHub.HubService import HubService -from Products.ZenHub.PBDaemon import translateError from Products.Zuul import getFacade from .ThresholdMixin import ThresholdMixin diff --git a/Products/ZenHub/services/ModelerService.py b/Products/ZenHub/services/ModelerService.py index 5d2accf024..c184c8da3e 100644 --- a/Products/ZenHub/services/ModelerService.py +++ b/Products/ZenHub/services/ModelerService.py @@ -22,7 +22,7 @@ from Products.DataCollector.Plugins import loadPlugins from Products.ZenCollector.interfaces import IConfigurationDispatchingFilter from Products.ZenEvents import Event -from Products.ZenHub.PBDaemon import translateError +from Products.ZenHub.errors import translateError from Products.ZenHub.services.PerformanceConfig import PerformanceConfig diff --git a/Products/ZenHub/services/PerformanceConfig.py b/Products/ZenHub/services/PerformanceConfig.py index abe3594e83..edbe1303e2 100644 --- a/Products/ZenHub/services/PerformanceConfig.py +++ b/Products/ZenHub/services/PerformanceConfig.py @@ -10,9 +10,9 @@ from twisted.spread import pb from zope import component +from Products.ZenHub.errors import translateError from Products.ZenHub.HubService import HubService from Products.ZenHub.interfaces import IBatchNotifier -from Products.ZenHub.PBDaemon import translateError from Products.ZenHub.zodb import onUpdate from Products.ZenModel.PerformanceConf import PerformanceConf from Products.ZenModel.ZenPack import ZenPack @@ -151,4 +151,6 @@ def zenPackUpdated(self, object, event): "updateThresholdClasses", self.remote_getThresholdClasses() ) except Exception: - self.log.warning("Error notifying a listener of new classes") + self.log.warning( + "Error notifying a listener of new threshold classes" + ) diff --git a/Products/ZenHub/services/ThresholdMixin.py b/Products/ZenHub/services/ThresholdMixin.py index dd16cd6c5e..9f42e0e687 100644 --- a/Products/ZenHub/services/ThresholdMixin.py +++ b/Products/ZenHub/services/ThresholdMixin.py @@ -9,7 +9,7 @@ import logging -from Products.ZenHub.PBDaemon import translateError +from Products.ZenHub.errors import translateError from Products.ZenModel.MinMaxThreshold import MinMaxThreshold from Products.ZenModel.ValueChangeThreshold import ValueChangeThreshold From 469dbb2f2e89229ef4b2c773dbd72ef8ae15d0ab Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 23 Jan 2024 13:25:26 -0600 Subject: [PATCH 040/147] Correctly locates configuration services in ZenPack egg installs. Restored DeviceOidTransform class to support vSphere ZenPack. ZEN-34658 --- .../ZenCollector/configcache/invalidator.py | 6 ++- .../ZenCollector/configcache/utils/pollers.py | 15 ++---- .../configcache/utils/services.py | 18 ++++--- Products/ZenHub/hub.zcml | 49 ++++++++++--------- Products/ZenHub/invalidationoid.py | 18 +++++++ 5 files changed, 64 insertions(+), 42 deletions(-) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 41407821d0..1bdeb67118 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -79,16 +79,20 @@ def add_arguments(parser, subparsers): Application.add_genconf_command(subsubparsers, (subp_run, subp_debug)) def __init__(self, config, context): + self.log = logging.getLogger("zen.configcache.invalidator") self.ctx = context configClasses = getConfigServices() + for cls in configClasses: + self.log.info( + "using service class %s.%s", cls.__module__, cls.__name__ + ) self.dispatcher = BuildConfigTaskDispatcher(configClasses) client = getRedisClient(url=getRedisUrl()) self.store = createObject("configcache-store", client) self.interval = config["poll-interval"] - self.log = logging.getLogger("zen.configcache.invalidator") def run(self): self._synchronize() diff --git a/Products/ZenCollector/configcache/utils/pollers.py b/Products/ZenCollector/configcache/utils/pollers.py index fad585b642..3345b57499 100644 --- a/Products/ZenCollector/configcache/utils/pollers.py +++ b/Products/ZenCollector/configcache/utils/pollers.py @@ -72,16 +72,11 @@ def initialize_invalidation_filters(ctx): for fltr in sorted(filters, key=lambda f: getattr(f, "weight", 100)): fltr.initialize(ctx) invalidation_filters.append(fltr) - log.info( - "registered %s invalidation filters.", len(invalidation_filters) - ) - if log.isEnabledFor(logging.DEBUG): - log.debug( - "invalidation filters: %s", - ", ".join( - "{0.__module__}.{0.__class__.__name__}".format(flt) - for flt in invalidation_filters - ) + for fltr in invalidation_filters: + log.info( + "using invalidation filter %s.%s", + fltr.__module__, + fltr.__class__.__name__ ) return invalidation_filters except Exception: diff --git a/Products/ZenCollector/configcache/utils/services.py b/Products/ZenCollector/configcache/utils/services.py index df8bbe8e1a..99870302e5 100644 --- a/Products/ZenCollector/configcache/utils/services.py +++ b/Products/ZenCollector/configcache/utils/services.py @@ -37,11 +37,12 @@ def mod_from_path(path): :returns: The package path :rtype: pathlib.Path """ - if "Products" in path.parts: - offset = path.parts.index("Products") - elif "ZenPacks" in path.parts: - offset = path.parts.index("ZenPacks") - return ".".join(itertools.chain(path.parts[offset:-1], [path.stem])) + rpath = path.parts[::-1] # reverse the path + if "Products" in rpath: + offset = rpath.index("Products") + elif "ZenPacks" in rpath: + offset = rpath.index("ZenPacks") + return ".".join(itertools.chain(rpath[1:offset + 1][::-1], [path.stem])) def getConfigServicesFromModule(name): @@ -86,11 +87,14 @@ def getConfigServices(): :returns: Tuple of configuration service classes :rtype: tuple[CollectorConfigService] """ - search_paths = itertools.chain(Products.__path__, ZenPacks.__path__) + search_paths = ( + pathlib.Path(p) + for p in itertools.chain(Products.__path__, ZenPacks.__path__) + ) service_paths = ( svcpath for path in search_paths - for svcpath in pathlib.Path(path).rglob("**/services") + for svcpath in path.rglob("**/services") ) module_names = ( mod_from_path(codepath) diff --git a/Products/ZenHub/hub.zcml b/Products/ZenHub/hub.zcml index 414c9f4609..843cc45884 100644 --- a/Products/ZenHub/hub.zcml +++ b/Products/ZenHub/hub.zcml @@ -11,34 +11,35 @@ --> - - + + - + - + - + - + diff --git a/Products/ZenHub/invalidationoid.py b/Products/ZenHub/invalidationoid.py index 9c786a7357..8a55abdcca 100644 --- a/Products/ZenHub/invalidationoid.py +++ b/Products/ZenHub/invalidationoid.py @@ -30,3 +30,21 @@ def __init__(self, obj): def transformOid(self, oid): return oid + + +# DeviceOidTransform kept for backward compability with vSphere ZenPack. +class DeviceOidTransform(object): + + def __init__(self, obj): + self._obj = obj + + def transformOid(self, oid): + # get device oid + result = oid + device = getattr(self._obj, "device", lambda: None)() + if device: + result = device._p_oid + log.debug( + "oid for %s changed to device oid for %s", self._obj, device + ) + return result From 0c04fce6ef376558af4f50519727aa6c121dbc63 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 23 Jan 2024 16:46:44 -0600 Subject: [PATCH 041/147] Implemented configcache's 'show' and 'expire' commands. ZEN-34650 --- Products/ZenCollector/configcache/cli.py | 115 +++++++++++++++++++++-- 1 file changed, 107 insertions(+), 8 deletions(-) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 3297ce56d9..a405446bd9 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -11,6 +11,8 @@ import argparse +import pprint + from datetime import datetime from zope.component import createObject @@ -18,7 +20,7 @@ from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl from .app import initialize_environment -from .cache import ConfigQuery, ConfigStatus +from .cache import ConfigKey, ConfigQuery, ConfigStatus from .misc.args import get_subparser @@ -147,11 +149,11 @@ def _format_date(ts): class Show(object): - description = "Show a configuration (JSON)" + description = "Show a configuration" @staticmethod def add_arguments(parser, subparsers): - subp = get_subparser(subparsers, "show", "Show a configuration (JSON)") + subp = get_subparser(subparsers, "show", "Show a configuration") subp.add_argument( "service", nargs=1, help="name of the configuration service" ) @@ -162,15 +164,24 @@ def add_arguments(parser, subparsers): subp.set_defaults(factory=Show) def __init__(self, args): - pass + self._monitor = args.monitor[0] + self._service = args.service[0] + self._device = args.device[0] def run(self): - pass + initialize_environment() + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + key = ConfigKey( + service=self._service, monitor=self._monitor, device=self._device + ) + results = store.get(key) + pprint.pprint(results.config.__dict__) class Expire(object): - description = "" + description = "Mark configurations as expired" @staticmethod def add_arguments(parser, subparsers): @@ -183,10 +194,98 @@ def add_arguments(parser, subparsers): subp.set_defaults(factory=Expire) def __init__(self, args): - pass + self._monitor = args.monitor + self._service = args.service + self._devices = getattr(args, "device", []) def run(self): - pass + if not self._confirm_inputs(): + print("exit") + return + initialize_environment() + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + query = ConfigQuery(service=self._service, monitor=self._monitor) + results = store.get_status(*store.search(query)) + method = self._no_devices if not self._devices else self._with_devices + keys = method(results) + store.set_expired(*keys) + count = len(keys) + print( + "expired %d device configuration%s" + % (count, "" if count == 1 else "s") + ) + + def _no_devices(self, results): + return tuple(key for key, state in results) + + def _with_devices(self, results): + return tuple( + key for key, state in results if key.device in self._devices + ) + + def _confirm_inputs(self): + if self._devices: + return True + if (self._monitor, self._service) == ("*", "*"): + mesg = "Recreate all device configurations" + elif "*" not in self._monitor and self._service == "*": + mesg = ( + "Recreate all device configurations monitored by the " + "'%s' collector" % (self._monitor,) + ) + elif "*" in self._monitor and self._service == "*": + mesg = ( + "Recreate all device configurations monitored by all " + "collectors matching '%s'" % (self._monitor,) + ) + elif self._monitor == "*" and "*" not in self._service: + mesg = ( + "Recreate all device configurations created by the '%s' " + "configuration service" % (self._service.split(".")[-1],) + ) + elif self._monitor == "*" and "*" in self._service: + mesg = ( + "Recreate all device configurations created by all " + "configuration services matching '%s'" % (self._service,) + ) + elif "*" in self._monitor and "*" not in self._service: + mesg = ( + "Recreate all device configurations created by the " + "'%s' configuration service and monitored by all " + "collectors matching '%s'" % (self._service, self._monitor) + ) + elif "*" not in self._monitor and "*" in self._service: + mesg = ( + "Recreate all device configurations monitored by the '%s' " + "collector and created by all configuration services " + "matching '%s'" % (self._monitor, self._service) + ) + elif "*" not in self._monitor and "*" not in self._service: + mesg = ( + "Recreate all device configurations monitored by the '%s' " + "collector and created by the '%s' configuration service" + % (self._monitor, self._service) + ) + elif "*" in self._monitor and "*" in self._service: + mesg = ( + "Recreate all device configurations monitored by all " + "collectors matching '%s' and created by all configuration " + "services matching '%s'" % (self._monitor, self._service) + ) + else: + mesg = "monitor '%s' service '%s'" % ( + self._monitor, + self._service, + ) + return _confirm(mesg) + + +def _confirm(mesg): + response = None + while response not in ["y", "n", ""]: + response = raw_input("%s. Are you sure (y/N)? " % (mesg,)).lower() + return response == "y" # list - list configs; From ecb07f277e87d7a992cf9f4c2c8e134561504f55 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 24 Jan 2024 14:12:03 -0600 Subject: [PATCH 042/147] Handle showing a config that doesn't exist. ZEN-34650 --- Products/ZenCollector/configcache/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index a405446bd9..4dba9295c2 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -10,8 +10,8 @@ from __future__ import absolute_import, print_function import argparse - import pprint +import sys from datetime import datetime @@ -176,7 +176,10 @@ def run(self): service=self._service, monitor=self._monitor, device=self._device ) results = store.get(key) - pprint.pprint(results.config.__dict__) + if results: + pprint.pprint(results.config.__dict__) + else: + print("configuration not found", file=sys.stderr) class Expire(object): From 9ddb3dc15b6d0d75a7252f1e3464e2aa7c229dca Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 25 Jan 2024 14:23:16 -0600 Subject: [PATCH 043/147] Fix call to dispatching strategy class. Moved ConfigCache service to ZenCollector.services package. Removed redundant getOptionsFilter implementation. ZEN-34652 --- Products/ZenCollector/daemon.py | 6 +-- .../services/ConfigCache.py | 40 +++++++------------ 2 files changed, 18 insertions(+), 28 deletions(-) rename Products/{ZenHub => ZenCollector}/services/ConfigCache.py (84%) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index c4240e3e86..de0999eb6e 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -59,8 +59,8 @@ class CollectorDaemon(RRDDaemon): _frameworkFactoryName = "default" # type: str """Identifies the IFrameworkFactory implementation to use.""" - # CollectorDaemon has an additional service: ConfigCache - initialServices = RRDDaemon.initialServices + ["ConfigCache"] + _cacheServiceName = "Products.ZenCollector.services.ConfigCache" + initialServices = RRDDaemon.initialServices + [_cacheServiceName] @property def preferences(self): # type: () -> ICollectorPreferences @@ -392,7 +392,7 @@ def _startDeviceConfigLoader(self): @defer.inlineCallbacks def getRemoteConfigCacheProxy(self): """Return the remote configuration cache proxy.""" - proxy = yield self.getService("ConfigCache") + proxy = yield self.getService(self._cacheServiceName) defer.returnValue(proxy) def getRemoteConfigServiceProxy(self): diff --git a/Products/ZenHub/services/ConfigCache.py b/Products/ZenCollector/services/ConfigCache.py similarity index 84% rename from Products/ZenHub/services/ConfigCache.py rename to Products/ZenCollector/services/ConfigCache.py index 924a67592e..c7cea8b200 100644 --- a/Products/ZenHub/services/ConfigCache.py +++ b/Products/ZenCollector/services/ConfigCache.py @@ -9,14 +9,15 @@ import logging -from zope.component import createObject, getUtilitiesFor +from zope.component import createObject from Products.ZenCollector.configcache.cache import ConfigQuery -from Products.ZenCollector.interfaces import IConfigurationDispatchingFilter from Products.ZenHub.errors import translateError from Products.ZenHub.HubService import HubService from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl +from .optionsfilter import getOptionsFilter + class ConfigCache(HubService): """ZenHub service for retrieving device configs from Redis.""" @@ -96,7 +97,8 @@ def remote_getDeviceConfigs( options, ) previous = set(deviceids) - current_keys = tuple(self._filter(self._keys(servicename), options)) + predicate = getOptionsFilter(options) + current_keys = tuple(self._filter(self._keys(servicename), predicate)) # 'newest_keys' references devices not found in 'previous' newest_keys = ( @@ -138,22 +140,23 @@ def _keys(self, servicename): self.log.info("[ConfigCache] using query %s", query) return self._store.search(query) - def _filter(self, keys, options): + def _filter(self, keys, predicate): """ Returns a subset of device IDs in `names` based on the contents of the `options` parameter. @param keys: Cache config keys @type keys: Iterable[ConfigKey] - @param options: Arguments into filters - @type options: Mapping[str, Any] + @param predicate: Function that determines whether to keep the device + @type options: Function(Device) -> Boolean @rtype: Iterator[str] """ # _filter is a generator function returning Device objects - predicate = self._getOptionsFilter(options) + proxy = _DeviceProxy() for key in keys: try: - if predicate(key.device): + proxy.id = key.device + if predicate(proxy): yield key except Exception: if self.log.isEnabledFor(logging.DEBUG): @@ -162,21 +165,8 @@ def _filter(self, keys, options): method = self.log.warn method("error filtering device ID %s", key.device) - def _getOptionsFilter(self, options): - def _alwaysTrue(x): - return True - deviceFilter = _alwaysTrue - if options: - dispatchFilterName = ( - options.get("configDispatch", "") if options else "" - ) - filterFactories = dict( - getUtilitiesFor(IConfigurationDispatchingFilter) - ) - filterFactory = filterFactories.get( - dispatchFilterName, None - ) or filterFactories.get("", None) - if filterFactory: - deviceFilter = filterFactory.getFilter(options) or deviceFilter - return deviceFilter +class _DeviceProxy(object): + # The predicate returned by getOptionsFilter expects an object + # with an `id` attribute. So make a simple class with one attribute. + id = None From df5c9a95a3f97ed80bb93185eadddd8d3924ebde Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 26 Jan 2024 12:30:45 -0600 Subject: [PATCH 044/147] Allow runtime initialization without Zope. ZEN-34663 --- Products/ZenCollector/configcache/app/base.py | 2 - Products/ZenCollector/configcache/app/init.py | 43 ++++++++++++++----- Products/ZenCollector/configcache/cli.py | 37 ++++++---------- .../ZenCollector/configcache/configure.zcml | 6 +-- Products/ZenCollector/configcache/expire.zcml | 13 ++++++ Products/ZenCollector/configcache/list.zcml | 13 ++++++ Products/ZenCollector/configcache/show.zcml | 13 ++++++ Products/ZenCollector/configcache/store.zcml | 15 +++++++ 8 files changed, 100 insertions(+), 42 deletions(-) create mode 100644 Products/ZenCollector/configcache/expire.zcml create mode 100644 Products/ZenCollector/configcache/list.zcml create mode 100644 Products/ZenCollector/configcache/show.zcml create mode 100644 Products/ZenCollector/configcache/store.zcml diff --git a/Products/ZenCollector/configcache/app/base.py b/Products/ZenCollector/configcache/app/base.py index 56f6612f44..c1854befc0 100644 --- a/Products/ZenCollector/configcache/app/base.py +++ b/Products/ZenCollector/configcache/app/base.py @@ -18,8 +18,6 @@ from MySQLdb import OperationalError -# from ..misc import app_name - from .config import add_config_arguments, getConfigFromArguments from .init import initialize_environment from .genconf import GenerateConfig diff --git a/Products/ZenCollector/configcache/app/init.py b/Products/ZenCollector/configcache/app/init.py index c5de57e180..d1f2f14ed9 100644 --- a/Products/ZenCollector/configcache/app/init.py +++ b/Products/ZenCollector/configcache/app/init.py @@ -7,21 +7,44 @@ # ############################################################################## -from OFS.Application import import_products -from Zope2.App import zcml +from zope.configuration import xmlconfig -import Products.ZenWidgets -from Products.ZenUtils.Utils import load_config, load_config_override -from Products.ZenUtils.zenpackload import load_zenpacks +def initialize_environment(configs=(), overrides=(), useZope=True): + if useZope: + _use_zope(configs=configs, overrides=overrides) + else: + _no_zope(configs=configs, overrides=overrides) -def initialize_environment(configs=(), overrides=()): +def _use_zope(configs, overrides): + from Zope2.App import zcml + from OFS.Application import import_products + from Products.ZenUtils.zenpackload import load_zenpacks + import Products.ZenWidgets + import_products() load_zenpacks() zcml.load_site() - load_config_override('scriptmessaging.zcml', Products.ZenWidgets) - for filepath, module in configs: - load_config(filepath, module) + _load_overrides( + zcml._context, [("scriptmessaging.zcml", Products.ZenWidgets)] + ) + _load_configs(zcml._context, configs) + _load_overrides(zcml._context, overrides) + + +def _no_zope(configs, overrides): + ctx = xmlconfig._getContext() + _load_configs(ctx, configs) + _load_overrides(ctx, overrides) + + +def _load_configs(ctx, configs): + for filename, module in configs: + xmlconfig.file(filename, package=module, context=ctx) + + +def _load_overrides(ctx, overrides): for filepath, module in overrides: - load_config_override(filepath, module) + xmlconfig.includeOverrides(ctx, filepath, package=module) + ctx.execute_actions() diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 4dba9295c2..f5dc080cbf 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -17,6 +17,8 @@ from zope.component import createObject +import Products.ZenCollector.configcache as CONFIGCACHE_MODULE + from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl from .app import initialize_environment @@ -28,6 +30,8 @@ class List_(object): description = "List configurations" + configs = (("list.zcml", CONFIGCACHE_MODULE),) + @staticmethod def add_arguments(parser, subparsers): subp = get_subparser( @@ -68,7 +72,7 @@ def __init__(self, args): self._states = () def run(self): - initialize_environment() + initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) query = ConfigQuery(service=self._service, monitor=self._monitor) @@ -137,7 +141,7 @@ def _format_status(status): _format_date(status.submitted) ) elif isinstance(status, ConfigStatus.Building): - return "building started {}".format(_format_date(status.started)) + return "build started {}".format(_format_date(status.started)) else: return "????" @@ -151,6 +155,8 @@ class Show(object): description = "Show a configuration" + configs = (("show.zcml", CONFIGCACHE_MODULE),) + @staticmethod def add_arguments(parser, subparsers): subp = get_subparser(subparsers, "show", "Show a configuration") @@ -169,7 +175,7 @@ def __init__(self, args): self._device = args.device[0] def run(self): - initialize_environment() + initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) key = ConfigKey( @@ -186,6 +192,8 @@ class Expire(object): description = "Mark configurations as expired" + configs = (("expire.zcml", CONFIGCACHE_MODULE),) + @staticmethod def add_arguments(parser, subparsers): subp = get_subparser( @@ -205,7 +213,7 @@ def run(self): if not self._confirm_inputs(): print("exit") return - initialize_environment() + initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) query = ConfigQuery(service=self._service, monitor=self._monitor) @@ -291,27 +299,6 @@ def _confirm(mesg): return response == "y" -# list - list configs; -# ls [-m monitor] [-s service] [-u] [-f state] [device] -# where 'monitor', 'service' and 'device' can be globs. -# Output should look like: -# [device] [state] [monitor] [service] -# if '-u' is given, then -# [device-path] [state] [monitor] [service] -# where 'state' is: -# Current HH:MM:SS - current with time remaining -# Expired - expired configuration -# Pending HH:MM:SS - pending with time remaining -# and 'device-path' is the dmd path (UID) -# -# show - show config in JSON format; -# cat [service] [monitor] [device] -# No wildcard support. -# -# expire - Mark one or more configurations expired; -# expire [-m monitor] [-s service] [device] - - class MultiChoice(argparse.Action): """Allow multiple values for a choice option.""" diff --git a/Products/ZenCollector/configcache/configure.zcml b/Products/ZenCollector/configcache/configure.zcml index 1f249e5ad0..394852439e 100644 --- a/Products/ZenCollector/configcache/configure.zcml +++ b/Products/ZenCollector/configcache/configure.zcml @@ -8,10 +8,6 @@ License.zenoss under the directory where your Zenoss product is installed. - - + diff --git a/Products/ZenCollector/configcache/expire.zcml b/Products/ZenCollector/configcache/expire.zcml new file mode 100644 index 0000000000..8ec2993701 --- /dev/null +++ b/Products/ZenCollector/configcache/expire.zcml @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/Products/ZenCollector/configcache/list.zcml b/Products/ZenCollector/configcache/list.zcml new file mode 100644 index 0000000000..8ec2993701 --- /dev/null +++ b/Products/ZenCollector/configcache/list.zcml @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/Products/ZenCollector/configcache/show.zcml b/Products/ZenCollector/configcache/show.zcml new file mode 100644 index 0000000000..8ec2993701 --- /dev/null +++ b/Products/ZenCollector/configcache/show.zcml @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/Products/ZenCollector/configcache/store.zcml b/Products/ZenCollector/configcache/store.zcml new file mode 100644 index 0000000000..8226384e76 --- /dev/null +++ b/Products/ZenCollector/configcache/store.zcml @@ -0,0 +1,15 @@ + + + + + + + From 8ff266d5b367f016aae00b1321633dbebb6d8bd0 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 30 Jan 2024 15:23:22 -0600 Subject: [PATCH 045/147] Use defaults for missing ConfigCache z-properties. ZEN-34653 --- .../ZenCollector/configcache/invalidator.py | 8 +++++-- Products/ZenCollector/configcache/manager.py | 12 +++++++--- .../configcache/utils/__init__.py | 5 ++++ .../configcache/utils/propertymap.py | 24 ++++++++++++++----- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 1bdeb67118..a8f0ccfbda 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -127,7 +127,9 @@ def _synchronize(self): if count == 0: self.log.info("no dangling configurations found") timelimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, Constants.build_timeout_id + self.ctx.dmd.Devices, + Constants.build_timeout_id, + Constants.build_timeout_value, ) new_devices = _addNew( self.log, tool, timelimitmap, self.store, self.dispatcher @@ -144,7 +146,9 @@ def _process(self, invalidation): ) if not keys: timelimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, Constants.build_timeout_id + self.ctx.dmd.Devices, + Constants.build_timeout_id, + Constants.build_timeout_value, ) uid = device.getPrimaryId() timeout = timelimitmap.get(uid) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index b6261c76ac..feb4246aa2 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -96,7 +96,9 @@ def run(self): def _retry_pending_builds(self): pendinglimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, Constants.pending_timeout_id + self.ctx.dmd.Devices, + Constants.pending_timeout_id, + Constants.pending_timeout_value, ) now = time() count = 0 @@ -123,10 +125,14 @@ def _retry_pending_builds(self): def _rebuild_older_configs(self): buildlimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, Constants.build_timeout_id + self.ctx.dmd.Devices, + Constants.build_timeout_id, + Constants.build_timeout_value, ) agelimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, Constants.time_to_live_id + self.ctx.dmd.Devices, + Constants.time_to_live_id, + Constants.time_to_live_value, ) min_limit = agelimitmap.smallest_value() self.log.debug( diff --git a/Products/ZenCollector/configcache/utils/__init__.py b/Products/ZenCollector/configcache/utils/__init__.py index dfac1040ef..b15cd0ae42 100644 --- a/Products/ZenCollector/configcache/utils/__init__.py +++ b/Products/ZenCollector/configcache/utils/__init__.py @@ -21,6 +21,11 @@ class Constants(object): pending_timeout_id = "zDeviceConfigPendingTimeout" time_to_live_id = "zDeviceConfigTTL" + # Default values + build_timeout_value = 7200 + pending_timeout_value = 7200 + time_to_live_value = 43200 + __all__ = ( "BuildConfigTaskDispatcher", diff --git a/Products/ZenCollector/configcache/utils/propertymap.py b/Products/ZenCollector/configcache/utils/propertymap.py index 3ebfc4edee..7b65060651 100644 --- a/Products/ZenCollector/configcache/utils/propertymap.py +++ b/Products/ZenCollector/configcache/utils/propertymap.py @@ -7,6 +7,10 @@ # ############################################################################## +import logging + +log = logging.getLogger("zen.configcache.propertymap") + class DevicePropertyMap(object): """ @@ -20,8 +24,8 @@ class DevicePropertyMap(object): """ @classmethod - def from_organizer(cls, obj, propname, relName="devices"): - return cls(getPropertyValues(obj, propname, relName=relName)) + def from_organizer(cls, obj, propname, default, relName="devices"): + return cls(getPropertyValues(obj, propname, default, relName=relName)) def __init__(self, values): self.__values = tuple((p.split("/")[1:], v) for p, v in values.items()) @@ -46,22 +50,23 @@ def get(self, request_uid): # the longest match with the request. return max(matches, key=lambda item: item[0])[1] except ValueError: + log.exception("failed looking for value") # No path parts matched the request. return None -def getPropertyValues(obj, propname, relName="devices"): +def getPropertyValues(obj, propname, default, relName="devices"): """ Returns a mapping of UID -> property-value for the given z-property. """ - values = {obj.getPrimaryId(): obj.getZ(propname)} + values = {obj.getPrimaryId(): _getValue(obj, propname, default)} values.update( - (inst.getPrimaryId(), inst.getZ(propname)) + (inst.getPrimaryId(), _getValue(inst, propname, default)) for inst in obj.getSubInstances(relName) if inst.isLocal(propname) ) values.update( - (inst.getPrimaryId(), inst.getZ(propname)) + (inst.getPrimaryId(), _getValue(inst, propname, default)) for inst in obj.getOverriddenObjects(propname) ) if not values or any(v is None for v in values.values()): @@ -70,3 +75,10 @@ def getPropertyValues(obj, propname, relName="devices"): "z-property=%s" % (propname,) ) return values + + +def _getValue(obj, propname, default): + value = obj.getZ(propname) + if value is None: + return default + return value From f3189e354f2f143e75d5a4cb193d3e43e0eee36d Mon Sep 17 00:00:00 2001 From: Dmitri Budko Date: Mon, 3 Dec 2018 12:37:10 +0200 Subject: [PATCH 046/147] ZING-1631 [RCE] Remote Code Execution with weak SUDO configuration (Device Administration ) (#3485) --- bin/zenrun.d/zenpack-manager.sh | 3 --- etc/sudoers.d/zenoss_nmap | 2 +- etc/sudoers.d/zenoss_var_chown | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 etc/sudoers.d/zenoss_var_chown diff --git a/bin/zenrun.d/zenpack-manager.sh b/bin/zenrun.d/zenpack-manager.sh index 6ecbcc4d46..b0340bf9f9 100644 --- a/bin/zenrun.d/zenpack-manager.sh +++ b/bin/zenrun.d/zenpack-manager.sh @@ -85,9 +85,6 @@ link() { fi rsync -a "$zenpackPath"/ "$TARGET"/ - # Get var path (/var/zenoss) to strip off of TARGET - VARPATH=$(unset TERM; echo "from Products.ZenUtils.Utils import varPath;print varPath()" | zendmd --script /dev/stdin) - sudo /opt/zenoss/bin/var_chown "${TARGET#$VARPATH}" shift zenpack --link --install "$TARGET" "$@" diff --git a/etc/sudoers.d/zenoss_nmap b/etc/sudoers.d/zenoss_nmap index 4543296c66..a5c3c684f1 100644 --- a/etc/sudoers.d/zenoss_nmap +++ b/etc/sudoers.d/zenoss_nmap @@ -1,4 +1,4 @@ # Allows Zenoss to use nmap for pinging -%zenoss ALL=(ALL) NOPASSWD: /usr/bin/nmap +%zenoss ALL=(ALL) NOPASSWD: /usr/bin/nmap !(*—script*) !(*-sC*) Defaults:zenoss !requiretty diff --git a/etc/sudoers.d/zenoss_var_chown b/etc/sudoers.d/zenoss_var_chown deleted file mode 100644 index 486cb0e4e8..0000000000 --- a/etc/sudoers.d/zenoss_var_chown +++ /dev/null @@ -1,4 +0,0 @@ -# Allows Zenoss to run a chown command in its var directory -%zenoss ALL=(ALL) NOPASSWD: /opt/zenoss/bin/var_chown -Defaults:zenoss !requiretty - From 3fde29ab46f64d3e10ac6215fc554e6d67aca189 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 31 Jan 2024 06:57:42 -0600 Subject: [PATCH 047/147] Removed obsolete sudo config for dmidecode --- etc/sudoers.d/zenoss_dmidecode | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 etc/sudoers.d/zenoss_dmidecode diff --git a/etc/sudoers.d/zenoss_dmidecode b/etc/sudoers.d/zenoss_dmidecode deleted file mode 100644 index b8152fc353..0000000000 --- a/etc/sudoers.d/zenoss_dmidecode +++ /dev/null @@ -1,4 +0,0 @@ -# Allows Zenoss to examine hardware information -%zenoss ALL=(ALL) NOPASSWD: /usr/sbin/dmidecode -Defaults:zenoss !requiretty - From 5218849fc2e7a604f2864a90e3ff6d69769a7eeb Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 31 Jan 2024 10:21:11 -0600 Subject: [PATCH 048/147] Pass correct options to curl for HEAD requests. ZEN-34674 --- bin/healthchecks/MetricShipper/store_answering | 2 +- bin/healthchecks/metrics_answering | 2 +- bin/healthchecks/query_answering | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/healthchecks/MetricShipper/store_answering b/bin/healthchecks/MetricShipper/store_answering index 904f4e6257..38f6f8f878 100755 --- a/bin/healthchecks/MetricShipper/store_answering +++ b/bin/healthchecks/MetricShipper/store_answering @@ -13,7 +13,7 @@ is_ready() { status_url=$1 - timeout 3 curl -A 'Metric_Shipper Store_answering Healthcheck' -w %{http_code} -s -XHEAD ${status_url} + timeout 3 curl -A 'Metric_Shipper Store_answering Healthcheck' -o /dev/null -w %{http_code} -s --head ${status_url} } http_code=$(is_ready http://localhost:8080/ping/status/metrics) diff --git a/bin/healthchecks/metrics_answering b/bin/healthchecks/metrics_answering index 20c3fb8057..b70c27f555 100755 --- a/bin/healthchecks/metrics_answering +++ b/bin/healthchecks/metrics_answering @@ -13,7 +13,7 @@ is_ready() { status_url=$1 - timeout 3 curl -A 'Metrics_answering Healthcheck' -w %{http_code} -s -XHEAD ${status_url} + timeout 3 curl -A 'Metrics_answering Healthcheck' -o /dev/null -w %{http_code} -s --head ${status_url} } http_code=$(is_ready http://localhost:8080/ping/status/metrics) diff --git a/bin/healthchecks/query_answering b/bin/healthchecks/query_answering index 4d8f9e36fa..9ebc341ad8 100755 --- a/bin/healthchecks/query_answering +++ b/bin/healthchecks/query_answering @@ -13,7 +13,7 @@ is_ready() { status_url=$1 - timeout 3 curl -A 'Query_answering is_ready' -w %{http_code} -s -XHEAD ${status_url} + timeout 3 curl -A 'Query_answering is_ready' -o /dev/null -w %{http_code} -s --head ${status_url} } http_code=$(is_ready http://localhost:8080/ping/status/performance) From 31daf12f0dc0dce226cfe445f4df6d58fcb23f6f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 1 Feb 2024 13:25:19 -0600 Subject: [PATCH 049/147] Add nmap wrapper script and update zenoss nmap sudoers config. ZEN-34616 --- bin/nmap | 22 ++++++++++++++++++++++ etc/sudoers.d/zenoss_nmap | 5 ++--- etc/sudoers.d/zenoss_ping | 6 ++---- 3 files changed, 26 insertions(+), 7 deletions(-) create mode 100755 bin/nmap diff --git a/bin/nmap b/bin/nmap new file mode 100755 index 0000000000..b8a59641de --- /dev/null +++ b/bin/nmap @@ -0,0 +1,22 @@ +#!/bin/bash + +NMAP=/usr/bin/nmap + +CLEANED=() + +while [[ $# -gt 0 ]]; do + case $1 in + --script|-sC) + echo argument $1 not allowed + exit 1 + ;; + *) + CLEANED+=("$1") + shift + ;; + esac +done + +set -- "${CLEANED[@]}" + +exec ${NMAP} $@ diff --git a/etc/sudoers.d/zenoss_nmap b/etc/sudoers.d/zenoss_nmap index a5c3c684f1..0fa92555b0 100644 --- a/etc/sudoers.d/zenoss_nmap +++ b/etc/sudoers.d/zenoss_nmap @@ -1,4 +1,3 @@ -# Allows Zenoss to use nmap for pinging -%zenoss ALL=(ALL) NOPASSWD: /usr/bin/nmap !(*—script*) !(*-sC*) +# Allow privileged execution of nmap wrapper script +%zenoss ALL = NOPASSWD: /opt/zenoss/bin/nmap Defaults:zenoss !requiretty - diff --git a/etc/sudoers.d/zenoss_ping b/etc/sudoers.d/zenoss_ping index ccb1eb9ac5..b5ccf5b671 100644 --- a/etc/sudoers.d/zenoss_ping +++ b/etc/sudoers.d/zenoss_ping @@ -1,5 +1,3 @@ -# Allows Zenoss to use ping for pinging -%zenoss ALL=(ALL) NOPASSWD: /usr/bin/ping -%zenoss ALL=(ALL) NOPASSWD: /usr/bin/ping6 +# Allow privileged execution of ping and ping6 +%zenoss ALL = NOPASSWD: /usr/bin/ping,/usr/bin/ping6 Defaults:zenoss !requiretty - From b20ce69fe114592687afa4e46b40d0303b3ea8d6 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 1 Feb 2024 15:05:26 -0600 Subject: [PATCH 050/147] Add error handling for updating thresholds from a device config. --- Products/ZenCollector/daemon.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index de0999eb6e..273a89320d 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -695,7 +695,7 @@ def _updateConfig(self, cfg): self.log.debug("tasks for config %s: %s", configId, newTasks) nowTime = time.time() - for (taskName, task_) in newTasks.iteritems(): + for taskName, task_ in newTasks.iteritems(): # if not cycling run the task immediately, # otherwise let the scheduler decide when to run the task now = not self.options.cycle @@ -717,7 +717,16 @@ def _updateConfig(self, cfg): # TODO: another hack? if hasattr(cfg, "thresholds"): - self.getThresholds().updateForDevice(configId, cfg.thresholds) + try: + self.getThresholds().updateForDevice( + configId, cfg.thresholds + ) + except Exception: + self.log.exception( + "failed to update thresholds " + "config-id=%s thresholds=%r", + configId, cfg.thresholds, + ) # if we're not running a normal daemon cycle then keep track of the # tasks we just added for this device so that we can shutdown once From b4570e3a300ce75f3b1d03b64140779621f5b88f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 1 Feb 2024 16:13:19 -0600 Subject: [PATCH 051/147] minor code refactor --- Products/ZenHub/PBDaemon.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index b643b41b34..8edd59d2c1 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -14,6 +14,8 @@ from itertools import chain from urlparse import urlparse +import six + from twisted.cred.credentials import UsernamePassword from twisted.internet.endpoints import clientFromString from twisted.internet import defer, reactor, task @@ -150,7 +152,7 @@ def generateEvent(self, event, **kw): """ eventCopy = {} for k, v in chain(event.items(), kw.items()): - if isinstance(v, basestring): + if isinstance(v, six.string_types): # default max size is 512k size = LIMITS.get(k, DEFAULT_LIMIT) eventCopy[k] = v[0:size] if len(v) > size else v @@ -204,12 +206,12 @@ def metricWriter(self): if os.environ.get("CONTROLPLANE", "0") == "1": internal_publisher = self.internalPublisher() if internal_publisher: - internal_metric_filter = ( - lambda metric, value, timestamp, tags: tags - and tags.get("internal", False) - ) + + def _check_internal(metric, value, timestamp, tags): + return tags and tags.get("internal", False) + internal_metric_writer = FilteredMetricWriter( - internal_publisher, internal_metric_filter + internal_publisher, _check_internal ) self.__metric_writer = AggregateMetricWriter( [metric_writer, internal_metric_writer] @@ -323,7 +325,7 @@ def _load_initial_services(self): for svcname in self.initialServices: try: yield self.getService(svcname) - except Exception as ex: + except Exception: if self.options.cycle: self.log.exception(msg) else: From 72ab73d6bf8cb8b1fb93f7f5f8aadb48496a6fa7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 2 Feb 2024 06:51:00 -0600 Subject: [PATCH 052/147] Actually use /opt/zenoss/bin/nmap ZEN-34616 --- Products/ZenStatus/nmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenStatus/nmap/util.py b/Products/ZenStatus/nmap/util.py index a93e40b0ff..5ac40f92dd 100644 --- a/Products/ZenStatus/nmap/util.py +++ b/Products/ZenStatus/nmap/util.py @@ -24,7 +24,7 @@ MAX_NMAP_OVERHEAD = 0.5 # in seconds MIN_PING_TIMEOUT = 0.1 # in seconds -_NMAP_BINARY = "/usr/bin/nmap" +_NMAP_BINARY = "/opt/zenoss/bin/nmap" @defer.inlineCallbacks From 5b3e1ecd909437d0bda3922c9ed1b222f1720ed7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 2 Feb 2024 10:14:04 -0600 Subject: [PATCH 053/147] restore classic zenhub_answering healthcheck --- Products/ZenHub/zenhubclient.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Products/ZenHub/zenhubclient.py b/Products/ZenHub/zenhubclient.py index 2e1a56327a..74bfcd103b 100644 --- a/Products/ZenHub/zenhubclient.py +++ b/Products/ZenHub/zenhubclient.py @@ -10,6 +10,7 @@ import collections import logging import importlib +import os import sys import six @@ -19,6 +20,7 @@ from twisted.spread import pb from Products.ZenUtils.PBUtil import setKeepAlive +from Products.ZenUtils.Utils import zenPath, atomicWrite from .errors import HubDown from .server import ZenPBClientFactory @@ -78,6 +80,7 @@ def __init__( self.__pinger = None self.__zenhub = None self.__instanceId = None + self.__signalFile = ConnectedToZenHubSignalFile() @property def instance_id(self): @@ -212,10 +215,12 @@ def _new_connection(self, broker): # defer.returnValue(None) except Exception: log.exception("unexpected error while logging into ZenHub") + self.__signalFile.remove() self.__reactor.stop() else: log.debug("logged into ZenHub instance-id=%s", self.__instanceId) try: + self.__signalFile.touch() # Connection complete; install a listener to be notified if # the connection is lost. broker.notifyOnDisconnect(self._disconnected) @@ -271,6 +276,7 @@ def _reset(self): self.__pinger.stop() self.__pinger = None log.debug("stopped and removed ZenHub pinger") + self.__signalFile.remove() def _pingFail(self, ex): log.error("pinger failed: %s", ex) @@ -340,3 +346,26 @@ def _remoteErrorType(ex): def _fromRemoteError(ex): return _remoteErrorType(ex)(*ex.args) + + +class ConnectedToZenHubSignalFile(object): + """Manages a file that indicates successful connection to ZenHub.""" + + def __init__(self): + """Initialize a ConnectedToZenHubSignalFile instance.""" + filename = "zenhub_connected" + self.__signalFilePath = zenPath("var", filename) + self.__log = log.getChild("signalfile") + + def touch(self): + """Create the file.""" + atomicWrite(self.__signalFilePath, "") + self.__log.debug("Created file '%s'", self.__signalFilePath) + + def remove(self): + """Delete the file.""" + try: + os.remove(self.__signalFilePath) + except Exception: + pass + self.__log.debug("Removed file '%s'", self.__signalFilePath) From b75d0b2ded8c84faa90a33ee9dd6766ba8a300de Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 6 Feb 2024 15:36:14 -0600 Subject: [PATCH 054/147] fix: don't cache celery Signature objects. Signature objects create a task ID by default, so re-using the same signature to send tasks results in the same task ID being used. ZEN-34687 --- .../configcache/tests/test_dispatcher.py | 83 +++++++++++++++++++ .../configcache/utils/dispatcher.py | 29 ++++--- 2 files changed, 97 insertions(+), 15 deletions(-) create mode 100644 Products/ZenCollector/configcache/tests/test_dispatcher.py diff --git a/Products/ZenCollector/configcache/tests/test_dispatcher.py b/Products/ZenCollector/configcache/tests/test_dispatcher.py new file mode 100644 index 0000000000..13ac95abd1 --- /dev/null +++ b/Products/ZenCollector/configcache/tests/test_dispatcher.py @@ -0,0 +1,83 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2019, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +from unittest import TestCase + +from mock import call, patch + +from ..utils.dispatcher import BuildConfigTaskDispatcher, build_device_config + + +PATH = {"src": "Products.ZenCollector.configcache.utils.dispatcher"} + + +class BuildConfigTaskDispatcherTest(TestCase): + """Test the BuildConfigTaskDispatcher object.""" + + def setUp(t): + t.class_a = type( + "a", (object,), {"__module__": "some.path.one", "__name__": "a"} + ) + t.class_a_name = ".".join((t.class_a.__module__, t.class_a.__name__)) + t.class_b = type( + "b", (object,), {"__module__": "some.path.two", "__name__": "b"} + ) + t.class_b_name = ".".join((t.class_b.__module__, t.class_b.__name__)) + + t.bctd = BuildConfigTaskDispatcher((t.class_a, t.class_b)) + + @patch.object(build_device_config, "apply_async") + def test_dispatch_all(t, _apply_async): + timeout = 100.0 + soft = 100.0 + hard = 110.0 + monitor = "local" + device = "linux" + t.bctd.dispatch_all(monitor, device, timeout) + + _apply_async.assert_has_calls( + ( + call( + args=(monitor, device, t.class_a_name), + soft_time_limit=soft, + time_limit=hard, + ), + call( + args=(monitor, device, t.class_b_name), + soft_time_limit=soft, + time_limit=hard, + ), + ) + ) + + @patch.object(build_device_config, "apply_async") + def test_dispatch(t, _apply_async): + timeout = 100.0 + soft = 100.0 + hard = 110.0 + monitor = "local" + device = "linux" + svcname = t.class_a.__module__ + t.bctd.dispatch(svcname, monitor, device, timeout) + + _apply_async.assert_called_once_with( + args=(monitor, device, t.class_a_name), + soft_time_limit=soft, + time_limit=hard, + ) + + def test_dispatch_unknown_service(t): + timeout = 100.0 + monitor = "local" + device = "linux" + + with t.assertRaises(ValueError): + t.bctd.dispatch("unknown", monitor, device, timeout) diff --git a/Products/ZenCollector/configcache/utils/dispatcher.py b/Products/ZenCollector/configcache/utils/dispatcher.py index 1fafe55312..1b864e2016 100644 --- a/Products/ZenCollector/configcache/utils/dispatcher.py +++ b/Products/ZenCollector/configcache/utils/dispatcher.py @@ -24,10 +24,8 @@ def __init__(self, configClasses): @type configClasses: Sequence[Class] """ - self._sigs = { - cls.__module__: build_device_config.s( - ".".join((cls.__module__, cls.__name__)) - ) + self._classnames = { + cls.__module__: ".".join((cls.__module__, cls.__name__)) for cls in configClasses } @@ -37,9 +35,9 @@ def dispatch_all(self, monitorid, deviceid, timeout): configuration service. """ soft_limit, hard_limit = _get_limits(timeout) - for sig in self._sigs.values(): - sig.apply_async( - (monitorid, deviceid), + for name in self._classnames.values(): + build_device_config.apply_async( + args=(monitorid, deviceid, name), soft_time_limit=soft_limit, time_limit=hard_limit, ) @@ -52,14 +50,15 @@ def dispatch(self, servicename, monitorid, deviceid, timeout): @type monitorid: str @type deviceId: str """ - sig = self._sigs[servicename] - if sig: - soft_limit, hard_limit = _get_limits(timeout) - sig.apply_async( - (monitorid, deviceid), - soft_time_limit=soft_limit, - time_limit=hard_limit, - ) + name = self._classnames.get(servicename) + if name is None: + raise ValueError("service name '%s' not found" % servicename) + soft_limit, hard_limit = _get_limits(timeout) + build_device_config.apply_async( + args=(monitorid, deviceid, name), + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) def _get_limits(timeout): From 98045b9110695971c4f1d9bb1c5512420eb6e413 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 5 Feb 2024 12:01:51 -0600 Subject: [PATCH 055/147] fix: add default value to DevicePropertyMap The DevicePropertyMap is initialized with a default value that is returned when no match is found in the map. ZEN-34653 --- ...st_bestmatchmap.py => test_propertymap.py} | 8 +++-- .../configcache/utils/propertymap.py | 31 ++++++++++++++----- 2 files changed, 28 insertions(+), 11 deletions(-) rename Products/ZenCollector/configcache/tests/{test_bestmatchmap.py => test_propertymap.py} (91%) diff --git a/Products/ZenCollector/configcache/tests/test_bestmatchmap.py b/Products/ZenCollector/configcache/tests/test_propertymap.py similarity index 91% rename from Products/ZenCollector/configcache/tests/test_bestmatchmap.py rename to Products/ZenCollector/configcache/tests/test_propertymap.py index 53f3f5f80e..a0c63c91c7 100644 --- a/Products/ZenCollector/configcache/tests/test_bestmatchmap.py +++ b/Products/ZenCollector/configcache/tests/test_propertymap.py @@ -18,7 +18,7 @@ class EmptyBestMatchMapTest(TestCase): """Test an empty BestMatchMap object.""" def setUp(t): - t.bmm = DevicePropertyMap({}) + t.bmm = DevicePropertyMap({}, None) def tearDown(t): del t.bmm @@ -41,8 +41,10 @@ class BestMatchMapTest(TestCase): "/zport/dmd/Devices/Network": 14, } + _default = 15 + def setUp(t): - t.bmm = DevicePropertyMap(t.mapping) + t.bmm = DevicePropertyMap(t.mapping, t._default) def tearDown(t): del t.bmm @@ -63,7 +65,7 @@ def test_get_best_match(t): def test_get_too_short_request(t): value = t.bmm.get("/Devices") - t.assertIsNone(value) + t.assertEqual(t._default, value) def test_smallest_value(t): value = t.bmm.smallest_value() diff --git a/Products/ZenCollector/configcache/utils/propertymap.py b/Products/ZenCollector/configcache/utils/propertymap.py index 7b65060651..5a90c03b20 100644 --- a/Products/ZenCollector/configcache/utils/propertymap.py +++ b/Products/ZenCollector/configcache/utils/propertymap.py @@ -25,16 +25,28 @@ class DevicePropertyMap(object): @classmethod def from_organizer(cls, obj, propname, default, relName="devices"): - return cls(getPropertyValues(obj, propname, default, relName=relName)) + return cls( + getPropertyValues(obj, propname, default, relName=relName), + default + ) - def __init__(self, values): - self.__values = tuple((p.split("/")[1:], v) for p, v in values.items()) + def __init__(self, values, default): + self.__values = tuple( + (p.split("/")[1:], v) + for p, v in values.items() + if v is not None + ) + self.__default = default def smallest_value(self): try: return min(self.__values, key=lambda item: item[1])[1] - except ValueError: - return None + except ValueError as ex: + # Check whether the ValueError is about an empty sequence. + # If it's not, re-raise the exception. + if "arg is an empty sequence" not in str(ex): + raise + return self.__default def get(self, request_uid): # Split the request into its parts @@ -49,10 +61,13 @@ def get(self, request_uid): # Return the value associated with the path parts having # the longest match with the request. return max(matches, key=lambda item: item[0])[1] - except ValueError: - log.exception("failed looking for value") + except ValueError as ex: + # Check whether the ValueError is about an empty sequence. + # If it's not, re-raise the exception. + if "arg is an empty sequence" not in str(ex): + raise # No path parts matched the request. - return None + return self.__default def getPropertyValues(obj, propname, default, relName="devices"): From b7b18be01ebb8e1b9990ca43cdade1cfc7857eac Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 6 Feb 2024 16:27:10 -0600 Subject: [PATCH 056/147] fix: fix timestamp format in manager.py ZEN-34688 --- Products/ZenCollector/configcache/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index feb4246aa2..44a332d422 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -111,7 +111,7 @@ def _retry_pending_builds(self): "pending configuration build has timed out " "submitted=%s service=%s monitor=%s device=%s", datetime.fromtimestamp(status.submitted).strftime( - "%Y-%M-%d %H:%m:%S" + "%Y-%m-%d %H:%M:%S" ), Constants.build_timeout_id, duration, @@ -170,7 +170,7 @@ def _rebuild_older_configs(self): "submitted job to rebuild old config " "updated=%s %s=%s service=%s monitor=%s device=%s", datetime.fromtimestamp(status.updated).strftime( - "%Y-%M-%d %H:%m:%S" + "%Y-%m-%d %H:%M:%S" ), Constants.time_to_live_id, ttl_limit, From bf1c079ee4bc894ebe4232aa4855f29222148b2e Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 23 Oct 2023 12:44:25 -0500 Subject: [PATCH 057/147] Replace zenhub answering file with HTTP endpoint. ZEN-34622 --- Products/ZenHub/PBDaemon.py | 44 +++++- Products/ZenHub/localserver/__init__.py | 12 ++ Products/ZenHub/localserver/errors.py | 21 +++ Products/ZenHub/localserver/options.py | 10 ++ Products/ZenHub/localserver/resource.py | 27 ++++ Products/ZenHub/localserver/server.py | 48 ++++++ Products/ZenHub/localserver/zhstatus.py | 22 +++ Products/ZenHub/tests/test_PBDaemon.py | 11 +- Products/ZenHub/tests/test_zenhubworker.py | 63 +++++--- Products/ZenHub/zenhubworker.py | 161 ++++++++++++++------- bin/dumpstats | 18 +++ bin/healthchecks/zenhub_answering | 19 ++- 12 files changed, 375 insertions(+), 81 deletions(-) create mode 100644 Products/ZenHub/localserver/__init__.py create mode 100644 Products/ZenHub/localserver/errors.py create mode 100644 Products/ZenHub/localserver/options.py create mode 100644 Products/ZenHub/localserver/resource.py create mode 100644 Products/ZenHub/localserver/server.py create mode 100644 Products/ZenHub/localserver/zhstatus.py create mode 100755 bin/dumpstats diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index 8edd59d2c1..2b7ceb745f 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -17,7 +17,7 @@ import six from twisted.cred.credentials import UsernamePassword -from twisted.internet.endpoints import clientFromString +from twisted.internet.endpoints import clientFromString, serverFromString from twisted.internet import defer, reactor, task from twisted.internet.error import ReactorNotRunning from twisted.spread import pb @@ -43,6 +43,7 @@ from .errors import HubDown, translateError from .events import EventClient, EventQueueManager +from .localserver import LocalServer, ZenHubStatus from .metricpublisher import publisher from .zenhubclient import ZenHubClient @@ -138,6 +139,33 @@ def __init__( self.__record_queued_events_count ) + self.__server = _getLocalServer(self.options) + self.__server.add_resource( + "zenhub", + ZenHubStatus( + lambda: "connected" + if self.__zenhub_connected + else "disconnected" + ), + ) + self.__zhclient.notifyOnConnect( + lambda: self._set_zenhub_connected(True) + ) + self.__zenhub_connected = False + + def _set_zenhub_connected(self, state): + self.__zenhub_connected = state + if state: + # Re-add the disconnect callback because the ZenHub client + # removes all disconnect callbacks after a disconnect. + self.__zhclient.notifyOnDisconnect( + lambda: self._set_zenhub_connected(False) + ) + + @property + def local_server(self): + return self.__server + @property def services(self): return self.__zhclient.services @@ -288,6 +316,9 @@ def run(self): self.derivativeTracker(), ) + self.__server.start() + reactor.addSystemEventTrigger("before", "shutdown", self.__server.stop) + reactor.addSystemEventTrigger( "after", "shutdown", @@ -452,6 +483,7 @@ def loadThresholdClasses(self, classnames): def buildOptions(self): super(PBDaemon, self).buildOptions() + LocalServer.buildOptions(self.parser) self.parser.add_option( "--hubhost", dest="hubhost", @@ -613,3 +645,13 @@ def _getZenHubClient(app, options): options.hubtimeout, options.zhPingInterval, ) + + +def _getLocalServer(options): + # bind the server to the localhost interface so only local + # connections can be established. + server_endpoint_descriptor = "tcp:{port}:interface=127.0.0.1".format( + port=options.localport + ) + server_endpoint = serverFromString(reactor, server_endpoint_descriptor) + return LocalServer(reactor, server_endpoint) diff --git a/Products/ZenHub/localserver/__init__.py b/Products/ZenHub/localserver/__init__.py new file mode 100644 index 0000000000..e092f23b1f --- /dev/null +++ b/Products/ZenHub/localserver/__init__.py @@ -0,0 +1,12 @@ +from .errors import ErrorResponse, NotFound +from .resource import ZenResource +from .server import LocalServer +from .zhstatus import ZenHubStatus + +__all__ = ( + "ErrorResponse", + "LocalServer", + "NotFound", + "ZenHubStatus", + "ZenResource", +) diff --git a/Products/ZenHub/localserver/errors.py b/Products/ZenHub/localserver/errors.py new file mode 100644 index 0000000000..2af3a4ed29 --- /dev/null +++ b/Products/ZenHub/localserver/errors.py @@ -0,0 +1,21 @@ +import json + +from twisted.web.resource import Resource +from twisted.web._responses import NOT_FOUND + + +class ErrorResponse(Resource): + def __init__(self, code, detail): + Resource.__init__(self) + self.code = code + self.detail = detail + + def render(self, request): + request.setResponseCode(self.code) + request.setHeader(b"content-type", b"application/json; charset=utf-8") + return json.dumps({"error": self.code, "message": self.detail}) + + +class NotFound(ErrorResponse): + def __init__(self): + ErrorResponse.__init__(self, NOT_FOUND, "resource not found") diff --git a/Products/ZenHub/localserver/options.py b/Products/ZenHub/localserver/options.py new file mode 100644 index 0000000000..d7db14185a --- /dev/null +++ b/Products/ZenHub/localserver/options.py @@ -0,0 +1,10 @@ + + +def add_options(parser): + parser.add_option( + "--localport", + dest="localport", + type="int", + default=14682, + help="The app responds to localhost HTTP connections on this port", + ) diff --git a/Products/ZenHub/localserver/resource.py b/Products/ZenHub/localserver/resource.py new file mode 100644 index 0000000000..c40ab06164 --- /dev/null +++ b/Products/ZenHub/localserver/resource.py @@ -0,0 +1,27 @@ +import logging + +from twisted.web.resource import Resource +from twisted.web._responses import INTERNAL_SERVER_ERROR + +from .errors import ErrorResponse, NotFound + + +class ZenResource(Resource): + def __init__(self): + Resource.__init__(self) + name = self.__class__.__name__.lower() + self.log = logging.getLogger("zen.localserver.%s" % (name,)) + + def getChild(self, path, request): + return NotFound() + + def render(self, request): + try: + response = Resource.render(self, request) + if isinstance(response, Resource): + return response.render(request) + return response + except Exception: + return ErrorResponse( + INTERNAL_SERVER_ERROR, "unexpected problem" + ).render(request) diff --git a/Products/ZenHub/localserver/server.py b/Products/ZenHub/localserver/server.py new file mode 100644 index 0000000000..9d223d9702 --- /dev/null +++ b/Products/ZenHub/localserver/server.py @@ -0,0 +1,48 @@ +import logging + +from twisted.web.server import Site + +from .resource import ZenResource +from .options import add_options + + +class LocalServer(object): + """ + Server class to listen to local connections. + """ + + buildOptions = staticmethod(add_options) + + def __init__(self, reactor, endpoint): + self.__reactor = reactor + self.__endpoint = endpoint + + root = ZenResource() + self.__site = Site(root) + + self.__listener = None + self.__log = logging.getLogger("zen.localserver") + + def add_resource(self, name, resource): + self.__site.resource.putChild(name, resource) + + def start(self): + """Start listening.""" + d = self.__endpoint.listen(self.__site) + d.addCallbacks(self._success, self._failure) + + def stop(self): + if self._listener: + self._listener.stopListening() + + def _success(self, listener): + self.__log.info("opened localhost port %d", self.__endpoint._port) + self._listener = listener + + def _failure(self, error): + self.__log.error( + "failed to open local port port=%s error=%r", + self.__endpoint._port, + error, + ) + self.__reactor.stop() diff --git a/Products/ZenHub/localserver/zhstatus.py b/Products/ZenHub/localserver/zhstatus.py new file mode 100644 index 0000000000..7fe15c2472 --- /dev/null +++ b/Products/ZenHub/localserver/zhstatus.py @@ -0,0 +1,22 @@ +from twisted.web._responses import INTERNAL_SERVER_ERROR + +from .errors import ErrorResponse +from .resource import ZenResource + + +class ZenHubStatus(ZenResource): + def __init__(self, statusgetter): + ZenResource.__init__(self) + self._getstatus = statusgetter + + def render_GET(self, request): + try: + request.responseHeaders.addRawHeader( + b"content-type", b"text/plain; charset=utf-8" + ) + return self._getstatus() + except Exception: + self.log.exception("failed to get ZenHub connection status") + return ErrorResponse( + INTERNAL_SERVER_ERROR, "zenhub status unavailable" + ) diff --git a/Products/ZenHub/tests/test_PBDaemon.py b/Products/ZenHub/tests/test_PBDaemon.py index 809237d569..3ec77745e8 100644 --- a/Products/ZenHub/tests/test_PBDaemon.py +++ b/Products/ZenHub/tests/test_PBDaemon.py @@ -2,7 +2,7 @@ import sys from unittest import TestCase -from mock import Mock, patch, create_autospec, call +from mock import ANY, Mock, patch, create_autospec, call # Breaks Test Isolation. Products/ZenHub/metricpublisher/utils.py:15 # ImportError: No module named eventlet @@ -34,6 +34,7 @@ class PBDaemonInitTest(TestCase): @patch("{src}.DaemonStats".format(**PATH), autospec=True) @patch("{src}.EventQueueManager".format(**PATH), autospec=True) @patch("{src}.ZenDaemon.__init__".format(**PATH), autospec=True) + @patch("{src}._getLocalServer".format(**PATH), autospec=True) @patch("{src}._getZenHubClient".format(**PATH), autospec=True) @patch("{src}.Thresholds".format(**PATH), autospec=True) @patch("{src}.ThresholdNotifier".format(**PATH), autospec=True) @@ -42,6 +43,7 @@ def test___init__( ThresholdNotifier, Thresholds, _getZenHubClient, + _getLocalServer, ZenDaemon_init, EventQueueManager, DaemonStats, @@ -74,7 +76,12 @@ def test___init__( t.assertEqual(pbd.mname, name) zhc = _getZenHubClient.return_value - zhc.notifyOnConnect.assert_called_with(pbd._load_initial_services) + zhc.notifyOnConnect.assert_has_calls( + [call(pbd._load_initial_services), call(ANY)] + ) + + ls = _getLocalServer.return_value + ls.add_resource.assert_called_once_with("zenhub", ANY) EventQueueManager.assert_called_with(PBDaemon.options, PBDaemon.log) diff --git a/Products/ZenHub/tests/test_zenhubworker.py b/Products/ZenHub/tests/test_zenhubworker.py index 4eedceb60a..3091da5fff 100644 --- a/Products/ZenHub/tests/test_zenhubworker.py +++ b/Products/ZenHub/tests/test_zenhubworker.py @@ -71,6 +71,7 @@ def setUp(t): t.options.hubpassword = sentinel.hubpassword t.options.workerid = sentinel.workerid t.options.monitor = sentinel.monitor + t.options.localport = 12345 # Patch external dependencies needs_patching = [ @@ -88,6 +89,8 @@ def setUp(t): "ServiceRegistry", "UsernamePassword", "ZenHubClient", + "serverFromString", + "LocalServer", ] t.patchers = {} for target in needs_patching: @@ -160,6 +163,18 @@ def test___init__(t): ) t.assertEqual(t.ZenHubClient.return_value, t.zhw._ZenHubWorker__client) + t.serverFromString.assert_called_once_with( + t.reactor, + "tcp:{}:interface=127.0.0.1".format(t.zhw.options.localport), + ) + t.LocalServer.assert_called_once_with( + t.reactor, t.serverFromString.return_value + ) + server = t.LocalServer.return_value + server.add_resource.assert_has_calls( + [call("zenhub", ANY), call("stats", ANY)] + ) + t.MetricManager.assert_called_with( daemon_tags={ "zenoss_daemon": "zenhub_worker_%s_%s" @@ -180,6 +195,14 @@ def test___init__(t): name="zenhub_worker_metricmanager", ) + def test_getZenHubStatus_disconnected(t): + t.zhw._ZenHubWorker__client.is_connected = False + t.assertEqual(t.zhw.getZenHubStatus(), "disconnected") + + def test_getZenHubStatus_connected(t): + t.zhw._ZenHubWorker__client.is_connected = True + t.assertEqual(t.zhw.getZenHubStatus(), "connected") + @patch("{src}.signal".format(**PATH), autospec=True) def test_start(t, signal): signal.SIGUSR1 = sentinel.SIGUSR1 @@ -195,11 +218,13 @@ def test_start(t, signal): ) t.ZenHubClient.return_value.start.assert_called_once_with() + t.LocalServer.return_value.start.assert_called_once_with() t.MetricManager.return_value.start.assert_called_once_with() t.reactor.addSystemEventTrigger.assert_has_calls( [ call("before", "shutdown", t.ZenHubClient.return_value.stop), + call("before", "shutdown", t.LocalServer.return_value.stop), call("before", "shutdown", t.MetricManager.return_value.stop), ] ) @@ -425,7 +450,6 @@ def setUp(t): "ZenPBClientFactory", "clientFromString", "ClientService", - "ConnectedToZenHubSignalFile", "PingZenHub", "backoffPolicy", "getLogger", @@ -464,10 +488,15 @@ def test___init__(t): t.assertIsNone(t.zhc._ZenHubClient__pinger) t.assertIsNone(t.zhc._ZenHubClient__service) t.assertEqual(t.zhc._ZenHubClient__log, t.getLogger.return_value) - t.assertEqual( - t.zhc._ZenHubClient__signalFile, - t.ConnectedToZenHubSignalFile.return_value, - ) + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) + + def test_is_connected_false(t): + t.zhc._ZenHubClient__zenhub_connected = False + t.assertFalse(t.zhc.is_connected) + + def test_is_connected_true(t): + t.zhc._ZenHubClient__zenhub_connected = True + t.assertTrue(t.zhc.is_connected) @patch.object(ZenHubClient, "_ZenHubClient__prepForConnection") def test_start(t, prepForConnection): @@ -500,30 +529,28 @@ def test_restart(t, start, reset): start.assert_called_once_with() def test___reset_not_started(t): - signalFile = t.ConnectedToZenHubSignalFile.return_value service = t.ClientService.return_value pinger = t.LoopingCall.return_value t.zhc._ZenHubClient__reset() - signalFile.remove.assert_called_once_with() service.stopService.assert_not_called() pinger.stop.assert_not_called() def test___reset_after_start(t): - signalFile = t.ConnectedToZenHubSignalFile.return_value service = t.ClientService.return_value t.zhc._ZenHubClient__service = service pinger = t.LoopingCall.return_value t.zhc._ZenHubClient__pinger = pinger + t.zhc._ZenHubClient__zenhub_connected = True t.zhc._ZenHubClient__reset() - signalFile.remove.assert_called_once_with() service.stopService.assert_called_once_with() pinger.stop.assert_called_once_with() t.assertIsNone(t.zhc._ZenHubClient__pinger) t.assertIsNone(t.zhc._ZenHubClient__service) + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) @patch.object(ZenHubClient, "_ZenHubClient__connected") @patch.object(ZenHubClient, "_ZenHubClient__notConnected") @@ -548,25 +575,24 @@ def test___prepForConnection_after_stopping(t): @patch.object(ZenHubClient, "_ZenHubClient__prepForConnection") def test___disconnected_not_connected(t, prepForConnection): - signalFile = t.ConnectedToZenHubSignalFile.return_value t.zhc._ZenHubClient__disconnected() + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) prepForConnection.assert_called_once_with() - signalFile.remove.assert_called_once_with() @patch.object(ZenHubClient, "_ZenHubClient__prepForConnection") def test___disconnected_after_connection(t, prepForConnection): - signalFile = t.ConnectedToZenHubSignalFile.return_value pinger = t.LoopingCall.return_value t.zhc._ZenHubClient__pinger = pinger + t.zhc._ZenHubClient__zenhub_connected = True t.zhc._ZenHubClient__disconnected() prepForConnection.assert_called_once_with() - signalFile.remove.assert_called_once_with() pinger.stop.assert_called_once_with() t.assertIsNone(t.zhc._ZenHubClient__pinger) + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) @patch.object(ZenHubClient, "restart") @patch.object(ZenHubClient, "_ZenHubClient__login") @@ -577,6 +603,7 @@ def test___connected_no_connection(t, login, restart): t.zhc._ZenHubClient__connected(broker) restart.assert_called_once_with() login.assert_not_called() + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) @patch.object(ZenHubClient, "_ZenHubClient__login") @patch.object(ZenHubClient, "_ZenHubClient__pingFail") @@ -605,12 +632,11 @@ def test___connected(t, setKeepAlive, restart, pingFail, login): now=False, ) pinger_deferred.addErrback.assert_called_once_with(pingFail) - t.zhc._ZenHubClient__signalFile.touch.assert_called_once_with() broker.notifyOnDisconnect.assert_called_once_with( t.zhc._ZenHubClient__disconnected, ) - t.zhc._ZenHubClient__signalFile.remove.assert_not_called() + t.assertTrue(t.zhc._ZenHubClient__zenhub_connected) restart.assert_not_called() t.reactor.stop.assert_not_called() @@ -629,7 +655,7 @@ def test___connected_login_failure(t, login): type(ex).__name__, ex, ) - t.zhc._ZenHubClient__signalFile.remove.assert_called_once_with() + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) t.reactor.stop.assert_called_once_with() @patch.object(ZenHubClient, "_ZenHubClient__login") @@ -646,8 +672,7 @@ def test___connected_login_timeout(t, restart, login): t.zhc._ZenHubClient__log.error.assert_called_once_with(ANY) restart.assert_called_once_with() - t.zhc._ZenHubClient__signalFile.remove.assert_not_called() - t.zhc._ZenHubClient__signalFile.touch.assert_not_called() + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) t.reactor.stop.assert_not_called() @patch.object(ZenHubClient, "_ZenHubClient__login") @@ -673,7 +698,7 @@ def test___connected_reportingForWork_failure(t, restart, login): type(ex).__name__, ex, ) - t.zhc._ZenHubClient__signalFile.remove.assert_called_once_with() + t.assertFalse(t.zhc._ZenHubClient__zenhub_connected) t.reactor.stop.assert_called_once_with() def test___login(t): diff --git a/Products/ZenHub/zenhubworker.py b/Products/ZenHub/zenhubworker.py index 29a9400b30..05494aa7c8 100755 --- a/Products/ZenHub/zenhubworker.py +++ b/Products/ZenHub/zenhubworker.py @@ -10,10 +10,11 @@ from __future__ import absolute_import +import json import logging -import os import signal import time +import types from collections import defaultdict from contextlib import contextmanager @@ -22,15 +23,22 @@ from metrology import Metrology from twisted.application.internet import ClientService, backoffPolicy from twisted.cred.credentials import UsernamePassword -from twisted.internet.endpoints import clientFromString from twisted.internet import defer, reactor, error, task +from twisted.internet.endpoints import clientFromString, serverFromString from twisted.spread import pb +from twisted.web._responses import INTERNAL_SERVER_ERROR from zope.component import getGlobalSiteManager import Products.ZenHub as ZENHUB_MODULE from Products.DataCollector.Plugins import loadPlugins from Products.ZenHub import PB_PORT +from Products.ZenHub.localserver import ( + ErrorResponse, + LocalServer, + ZenHubStatus, + ZenResource, +) from Products.ZenHub.metricmanager import MetricManager, IMetricManager from Products.ZenHub.server import ( ServiceLoader, @@ -43,16 +51,20 @@ from Products.ZenUtils.debugtools import ContinuousProfiler from Products.ZenUtils.PBUtil import setKeepAlive from Products.ZenUtils.Time import isoDateTime -from Products.ZenUtils.Utils import zenPath, atomicWrite, load_config +from Products.ZenUtils.Utils import load_config from Products.ZenUtils.ZCmdBase import ZCmdBase + IDLE = "None/None" def getLogger(obj): """Return a logger based on the name of the given class.""" - cls = type(obj) - name = "zen.zenhubworker.%s" % (cls.__name__) + if isinstance(obj, types.InstanceType): + name = obj.__class__.__name__ + else: + name = type(obj).__name__ + name = "zen.zenhubworker.%s" % (name.lower()) return logging.getLogger(name) @@ -105,6 +117,18 @@ def __init__(self, reactor): self.worklistId, ) + # bind the server to the localhost interface so only local + # connections can be established. + server_endpoint_descriptor = "tcp:{port}:interface=127.0.0.1".format( + port=self.options.localport + ) + server_endpoint = serverFromString(reactor, server_endpoint_descriptor) + self.__server = LocalServer(reactor, server_endpoint) + self.__server.add_resource( + "zenhub", ZenHubStatus(lambda: self.getZenHubStatus()) + ) + self.__server.add_resource("stats", _ZenHubWorkerStats(self)) + # Setup Metric Reporting self.log.debug("Creating async MetricReporter") self._metric_manager = MetricManager( @@ -133,6 +157,11 @@ def start(self): "before", "shutdown", self.__client.stop ) + self.__server.start() + self.__reactor.addSystemEventTrigger( + "before", "shutdown", self.__server.stop + ) + self._metric_manager.start() self.__reactor.addSystemEventTrigger( "before", "shutdown", self._metric_manager.stop @@ -212,23 +241,53 @@ def _work_finished(self, duration, method): ) self.__reactor.callLater(0, self._shutdown) + def getZenHubStatus(self): + return "connected" if self.__client.is_connected else "disconnected" + + def getStats(self): + results = {"current": self.current} + if self.current != IDLE: + results["current.elapsed"] = time.time() - self.currentStart + + if self.__registry: + sorted_data = sorted( + self.__registry.iteritems(), + key=lambda kv: kv[0][1].rpartition(".")[-1], + ) + summarized_stats = [] + for (_, svc), svcob in sorted_data: + svc = "%s" % svc.rpartition(".")[-1] + for method, stats in sorted(svcob.callStats.items()): + summarized_stats.append( + { + "service": svc, + "method": method, + "count": stats.numoccurrences, + "total": stats.totaltime, + "average": stats.totaltime / stats.numoccurrences + if stats.numoccurrences + else 0.0, + "last-run": isoDateTime(stats.lasttime), + } + ) + results["statistics"] = summarized_stats + + return results + def reportStats(self): """Write zenhubworker's current statistics to the log.""" - now = time.time() - if self.current != IDLE: + stats = self.getStats() + if stats["current"] != IDLE: self.log.info( "Currently performing %s, elapsed %.2f s", - self.current, - now - self.currentStart, + stats["current"], + stats["current.elapsed"], ) else: self.log.info("Currently IDLE") - if self.__registry: + statistics = stats.get("statistics") + if statistics: loglines = ["Running statistics:"] - sorted_data = sorted( - self.__registry.iteritems(), - key=lambda kv: kv[0][1].rpartition(".")[-1], - ) loglines.append( " %-50s %-32s %8s %12s %8s %s" % ( @@ -240,22 +299,18 @@ def reportStats(self): "Last Run", ) ) - for (_, svc), svcob in sorted_data: - svc = "%s" % svc.rpartition(".")[-1] - for method, stats in sorted(svcob.callStats.items()): - loglines.append( - " - %-48s %-32s %8d %12.2f %8.2f %s" - % ( - svc, - method, - stats.numoccurrences, - stats.totaltime, - stats.totaltime / stats.numoccurrences - if stats.numoccurrences - else 0.0, - isoDateTime(stats.lasttime), - ), - ) + for entry in statistics: + loglines.append( + " - %-48s %-32s %8d %12.2f %8.2f %s" + % ( + entry["service"], + entry["method"], + entry["count"], + entry["total"], + entry["average"], + entry["last-run"], + ), + ) self.log.info("\n".join(loglines)) else: self.log.info("no service activity statistics") @@ -306,6 +361,7 @@ def _shutdown(self): def buildOptions(self): """Add optparse options to the options parser.""" ZCmdBase.buildOptions(self) + LocalServer.buildOptions(self.parser) self.parser.add_option( "--hubhost", dest="hubhost", @@ -411,7 +467,11 @@ def __init__( self.__service = None self.__log = getLogger(self) - self.__signalFile = ConnectedToZenHubSignalFile() + self.__zenhub_connected = False + + @property + def is_connected(self): + return self.__zenhub_connected def start(self): """Start connecting to ZenHub.""" @@ -436,13 +496,13 @@ def restart(self): self.start() def __reset(self): + self.__zenhub_connected = False if self.__pinger: self.__pinger.stop() self.__pinger = None if self.__service: self.__service.stopService() self.__service = None - self.__signalFile.remove() def __prepForConnection(self): if not self.__stopping: @@ -459,10 +519,10 @@ def __disconnected(self, *args): "Lost connection to ZenHub: %s", args[0] if args else "", ) + self.__zenhub_connected = False if self.__pinger: self.__pinger.stop() self.__pinger = None - self.__signalFile.remove() self.__prepForConnection() def __notConnected(self, *args): @@ -503,11 +563,11 @@ def __connected(self, broker): self.__log.error( "Unable to report for work: (%s) %s", type(ex).__name__, ex ) - self.__signalFile.remove() + self.__zenhub_connected = False self.__reactor.stop() else: self.__log.info("Logged into ZenHub") - self.__signalFile.touch() + self.__zenhub_connected = True # Connection complete; install a listener to be notified if # the connection is lost. @@ -557,27 +617,22 @@ def __call__(self): self.__client.restart() -class ConnectedToZenHubSignalFile(object): - """Manages a file that indicates successful connection to ZenHub.""" - - def __init__(self): - """Initialize a ConnectedToZenHubSignalFile instance.""" - filename = "zenhub_connected" - self.__signalFilePath = zenPath("var", filename) - self.__log = getLogger(self) - - def touch(self): - """Create the file.""" - atomicWrite(self.__signalFilePath, "") - self.__log.debug("Created file '%s'", self.__signalFilePath) +class _ZenHubWorkerStats(ZenResource): + def __init__(self, worker): + ZenResource.__init__(self) + self._worker = worker - def remove(self): - """Delete the file.""" + def render_GET(self, request): try: - os.remove(self.__signalFilePath) + request.responseHeaders.addRawHeader( + b"content-type", b"application/json; charset=utf-8" + ) + return json.dumps(self._worker.getStats()) except Exception: - pass - self.__log.debug("Removed file '%s'", self.__signalFilePath) + self.log.exception("failed to get zenhubworker stats") + return ErrorResponse( + INTERNAL_SERVER_ERROR, "zenhubworker statistics unavailable" + ) class ServiceReferenceFactory(object): diff --git a/bin/dumpstats b/bin/dumpstats new file mode 100755 index 0000000000..72b5b0f3b9 --- /dev/null +++ b/bin/dumpstats @@ -0,0 +1,18 @@ +#!/bin/sh +# Retrieve statistics from zenhubworker + +set -e + +FILE=/opt/zenoss/etc/global.conf + +getprop() { + grep "${1}" $FILE | cut -d' ' -f2 +} + +_PORT=$(getprop 'localport') +PORT=${_PORT:-14682} +URL=http://localhost:$PORT/stats +curl -f $URL +if [ $? -eq 0 ]; then + echo +fi diff --git a/bin/healthchecks/zenhub_answering b/bin/healthchecks/zenhub_answering index 8cfddc679e..d1459682e7 100755 --- a/bin/healthchecks/zenhub_answering +++ b/bin/healthchecks/zenhub_answering @@ -8,9 +8,16 @@ # ############################################################################## -if [ -e /opt/zenoss/var/zenhub_connected ] -then - exit 0 -else - exit 1 -fi +set -e + +FILE=/opt/zenoss/etc/global.conf + +getprop() { + grep "${1}" $FILE | cut -d' ' -f2 +} + +_PORT=$(getprop 'localport') +PORT=${_PORT:-14682} +URL=http://localhost:$PORT/zenhub + +test "$(curl -sq $URL)" = "connected" From f0e7da69d93e6a7d2fb0638c6db95e3823f496d7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 7 Feb 2024 14:27:17 -0600 Subject: [PATCH 058/147] Add zDeviceConfigMinimumTTL property. This zproperty specifies the number of seconds a configuration must exist before it can be expired. ZEN-34656 --- .../ZenCollector/configcache/cache/model.py | 11 + .../ZenCollector/configcache/cache/storage.py | 217 ++++++--- .../ZenCollector/configcache/invalidator.py | 109 +++-- Products/ZenCollector/configcache/manager.py | 47 +- .../configcache/tests/test_propertymap.py | 14 +- .../tests/test_propertymap_makers.py | 276 +++++++++++ .../configcache/tests/test_storage.py | 451 +++++++++++++++++- .../configcache/utils/__init__.py | 17 +- .../configcache/utils/constants.py | 22 + .../configcache/utils/propertymap.py | 79 ++- .../migrate/addConfigCacheProperties.py | 29 +- Products/ZenRelations/ZenPropertyManager.py | 10 +- Products/ZenRelations/zPropertyCategory.py | 1 + 13 files changed, 1106 insertions(+), 177 deletions(-) create mode 100644 Products/ZenCollector/configcache/tests/test_propertymap_makers.py create mode 100644 Products/ZenCollector/configcache/utils/constants.py diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py index 36c2d7f98f..29eefbf57b 100644 --- a/Products/ZenCollector/configcache/cache/model.py +++ b/Products/ZenCollector/configcache/cache/model.py @@ -95,6 +95,17 @@ def __eq__(self, other): return NotImplemented return self.updated == other.updated + class Retired(object): + """The cofiguration is retired, but not yet expired.""" + + def __init__(self, ts): + self.updated = ts + + def __eq__(self, other): + if not isinstance(other, _ConfigStatus.Retired): + return NotImplemented + return self.updated == other.updated + class Expired(object): """The configuration has expired.""" diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 1d594ee1d7..9c3355179b 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -55,6 +55,9 @@ _app = "configcache" log = logging.getLogger("zen.modelchange.stores") +_EXPIRED_SCORE = 0 +_PENDING_SCORE = -1 + class ConfigStoreFactory(Factory): """ @@ -84,6 +87,7 @@ def __init__(self, client): self.__uids = _DeviceUIDTable() self.__config = _DeviceConfigTable() self.__age = _ConfigMetadataTable("age") + self.__retired = _ConfigMetadataTable("retired") self.__pending = _ConfigMetadataTable("pending") self.__building = _ConfigMetadataTable("building") self.__range = type( @@ -91,6 +95,7 @@ def __init__(self, client): (object,), { "age": partial(_range, self.__client, self.__age), + "retired": partial(_range, self.__client, self.__retired), "pending": partial(_range, self.__client, self.__pending), "building": partial(_range, self.__client, self.__building), }, @@ -119,13 +124,13 @@ def add(self, record): @type record: ConfigRecord """ svc, mon, dvc, uid, updated, config = _from_record(record) - dead_key_parts = tuple( - (key.service, key.monitor, key.device) + + orphaned_keys = tuple( + key for key in self.search(ConfigQuery(service=svc, device=dvc)) if key.monitor != mon ) - dead_keys = [self.__config.make_key(*dkp) for dkp in dead_key_parts] - watch_keys = dead_keys + [svc, mon, dvc] + watch_keys = self._get_watch_keys(orphaned_keys + (record.key,)) add_uid = not self.__uids.exists(self.__client, dvc) def _add_impl(pipe): @@ -134,15 +139,20 @@ def _add_impl(pipe): # monitor. # Note: configs produced by different configuration services # may exist simultaneously. - for dkp in dead_key_parts: - self.__config.delete(pipe, *dkp) - self.__age.delete(pipe, *dkp) - self.__pending.delete(pipe, *dkp) - self.__building.delete(pipe, *dkp) + for key in orphaned_keys: + parts = (key.service, key.monitor, key.device) + self.__config.delete(pipe, *parts) + self.__age.delete(pipe, *parts) + self.__retired.delete(pipe, *parts) + self.__pending.delete(pipe, *parts) + self.__building.delete(pipe, *parts) if add_uid: self.__uids.set(pipe, dvc, uid) self.__config.set(pipe, svc, mon, dvc, config) self.__age.add(pipe, svc, mon, dvc, updated) + self.__retired.delete(pipe, svc, mon, dvc) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.delete(pipe, svc, mon, dvc) self.__client.transaction(_add_impl, *watch_keys) @@ -184,6 +194,7 @@ def remove(self, *keys): svc, mon, dvc = key.service, key.monitor, key.device self.__config.delete(pipe, svc, mon, dvc) self.__age.delete(pipe, svc, mon, dvc) + self.__retired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) pipe.execute() @@ -195,87 +206,159 @@ def remove(self, *keys): if devices: self.__uids.delete(self.__client, *devices) - def set_expired(self, *keys): + def set_retired(self, *keys): """ - Marks the indicated configuration as expired. + Marks the indicated configuration as retired. + + A configuration is retired when its `updated` field is less than the + difference between the current time and zDeviceConfigMinimumTTL. - Attempts to mark pending configurations as expired are ignored. + Only 'current' configurations can be marked as retired. Attempts + to change configurations in other statuses are ignored. @type keys: Sequence[ConfigKey] + @rtype: Sequence[ConfigKey] """ if len(keys) == 0: - return + return () - watch_keys = self._get_watch_keys(*keys) - scores = ( - ( - key, - self.__age.score( - self.__client, key.service, key.monitor, key.device - ), - ) + not_retired = tuple( + key for key in keys + if not self.__retired.exists( + self.__client, key.service, key.monitor, key.device + ) + ) + if len(not_retired) == 0: + return () + + watch_keys = self._get_watch_keys(keys) + targets = self._filter_by_score_keyonly( + lambda x: x > _EXPIRED_SCORE, not_retired ) - targets = tuple( - (key.service, key.monitor, key.device) - for key, score in scores - if score != 0.0 + if len(targets) == 0: + return () + + def _impl(pipe): + pipe.multi() + for svc, mon, dvc, score in targets: + self.__retired.add(pipe, svc, mon, dvc, score) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.delete(pipe, svc, mon, dvc) + + self.__client.transaction(_impl, *watch_keys) + return tuple(ConfigKey(svc, mon, dvc) for svc, mon, dvc, _ in targets) + + def set_expired(self, *keys): + """ + Marks the indicated configuration as expired. + + Attempts to mark configurations that are not 'current' or + 'retired' are ignored. + + @type keys: Sequence[ConfigKey] + @rtype: Sequence[ConfigKey] + """ + if len(keys) == 0: + return () + + watch_keys = self._get_watch_keys(keys) + targets = self._filter_by_score_keyonly( + lambda x: x > _EXPIRED_SCORE, keys ) + if len(targets) == 0: + return () def _impl(pipe): pipe.multi() - for svc, mon, dvc in targets: - self.__age.add(pipe, svc, mon, dvc, 0) + for svc, mon, dvc, _ in targets: + self.__age.add(pipe, svc, mon, dvc, _EXPIRED_SCORE) + self.__retired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) self.__client.transaction(_impl, *watch_keys) + return tuple(ConfigKey(svc, mon, dvc) for svc, mon, dvc, _ in targets) - def set_pending(self, *pending): + def set_pending(self, *pairs): """ Marks an expired configuration as waiting for a new configuration. @type pending: Sequence[(ConfigKey, float)] + @rtype: Sequence[ConfigKey] """ - if len(pending) == 0: - return + if len(pairs) == 0: + return () + + watch_keys = self._get_watch_keys(key for key, _ in pairs) + targets = self._filter_by_score_with_start( + lambda x: x == _EXPIRED_SCORE, pairs + ) - watch_keys = self._get_watch_keys(*(key for key, _ in pending)) - targets = self._get_targets(lambda x: x == 0.0, *pending) + if len(targets) == 0: + return () def _impl(pipe): pipe.multi() - for svc, mon, dvc, ts in targets: + for svc, mon, dvc, ts, _ in targets: score = _to_score(ts) - self.__age.add(pipe, svc, mon, dvc, -1) + self.__age.add(pipe, svc, mon, dvc, _PENDING_SCORE) + self.__retired.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) self.__pending.add(pipe, svc, mon, dvc, score) self.__client.transaction(_impl, *watch_keys) + return tuple( + ConfigKey(svc, mon, dvc) for svc, mon, dvc, _, _ in targets + ) - def set_building(self, *building): + def set_building(self, *pairs): """ Marks a pending configuration as building a new configuration. @type pairs: Sequence[(ConfigKey, float)] + @rtype: Sequence[ConfigKey] """ - if len(building) == 0: - return + if len(pairs) == 0: + return () + + valid = tuple( + (key, ts) + for key, ts in pairs + if self.__pending.exists( + self.__client, key.service, key.monitor, key.device + ) + ) + if len(valid) == 0: + return valid - watch_keys = self._get_watch_keys(*(key for key, _ in building)) - targets = self._get_targets(lambda x: x <= 0.0, *building) + watch_keys = self._get_watch_keys(key for key, _ in valid) def _impl(pipe): pipe.multi() - for svc, mon, dvc, ts in targets: - score = _to_score(ts) - self.__age.add(pipe, svc, mon, dvc, -1) + for key, ts in valid: + svc = key.service + mon = key.monitor + dvc = key.device self.__pending.delete(pipe, svc, mon, dvc) - self.__building.add(pipe, svc, mon, dvc, score) + self.__building.add(pipe, svc, mon, dvc, _to_score(ts)) self.__client.transaction(_impl, *watch_keys) + return tuple(key for key, _ in valid) + + def _filter_by_score_keyonly(self, predicate, keys): + pairs = ((key, None) for key in keys) + return tuple( + (svc, mon, dvc, score) + for svc, mon, dvc, _, score in self._filter_by_score( + predicate, pairs + ) + ) + + def _filter_by_score_with_start(self, predicate, pairs): + return tuple(self._filter_by_score(predicate, pairs)) - def _get_targets(self, predicate, *pairs): + def _filter_by_score(self, predicate, pairs): scores = ( ( key, @@ -286,8 +369,8 @@ def _get_targets(self, predicate, *pairs): ) for key, started in pairs ) - return tuple( - (key.service, key.monitor, key.device, started) + return ( + (key.service, key.monitor, key.device, started, score) for key, started, score in scores if predicate(score) ) @@ -312,7 +395,13 @@ def get_status(self, *keys): def _iter_status(self, scores): for key, score in scores: if score > 0: - yield (key, ConfigStatus.Current(_to_ts(score))) + rscore = self.__retired.score( + self.__client, key.service, key.monitor, key.device + ) + if rscore is not None: + yield (key, ConfigStatus.Retired(_to_ts(rscore))) + else: + yield (key, ConfigStatus.Current(_to_ts(score))) elif score == 0: yield (key, ConfigStatus.Expired()) else: @@ -327,26 +416,26 @@ def _iter_status(self, scores): if bscore is not None: yield (key, ConfigStatus.Building(_to_ts(bscore))) - def get_pending(self, service="*", monitor="*"): + def get_building(self, service="*", monitor="*"): """ - Return an iterator producing (ConfigKey, ConfigStatus.Pending) tuples. + Return an iterator producing (ConfigKey, ConfigStatus.Building) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Pending]] + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Building]] """ return ( - (key, ConfigStatus.Pending(ts)) - for key, ts in self.__range.pending(service, monitor) + (key, ConfigStatus.Building(ts)) + for key, ts in self.__range.building(service, monitor) ) - def get_building(self, service="*", monitor="*"): + def get_pending(self, service="*", monitor="*"): """ - Return an iterator producing (ConfigKey, ConfigStatus.Building) tuples. + Return an iterator producing (ConfigKey, ConfigStatus.Pending) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Building]] + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Pending]] """ return ( - (key, ConfigStatus.Building(ts)) - for key, ts in self.__range.building(service, monitor) + (key, ConfigStatus.Pending(ts)) + for key, ts in self.__range.pending(service, monitor) ) def get_expired(self, service="*", monitor="*"): @@ -362,6 +451,17 @@ def get_expired(self, service="*", monitor="*"): ) ) + def get_retired(self, service="*", monitor="*"): + """ + Return an iterator producing (ConfigKey, ConfigStatus.Retired) tuples. + + @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Expired]] + """ + return ( + (key, ConfigStatus.Retired(ts)) + for key, ts in self.__range.retired(service, monitor) + ) + def get_older(self, maxtimestamp, service="*", monitor="*"): """ Returns an iterator producing (ConfigKey, ConfigStatus.Current) @@ -392,11 +492,12 @@ def get_newer(self, mintimestamp, service="*", monitor="*"): ) ) - def _get_watch_keys(self, *keys): + def _get_watch_keys(self, keys): return set( itertools.chain.from_iterable( ( self.__age.make_key(key.service, key.monitor), + self.__retired.make_key(key.service, key.monitor), self.__pending.make_key(key.service, key.monitor), self.__building.make_key(key.service, key.monitor), ) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index a8f0ccfbda..0f8991a5b0 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -10,6 +10,7 @@ from __future__ import print_function, absolute_import import logging +import time from Products.AdvancedQuery import And, Eq from zenoss.modelindex import constants @@ -21,13 +22,12 @@ from Products.Zuul.catalog.interfaces import IModelCatalogTool from .app import Application -from .cache import ConfigQuery +from .cache import ConfigQuery, ConfigStatus from .debug import Debug as DebugCommand from .misc.args import get_subparser from .modelchange import InvalidationCause from .utils import ( BuildConfigTaskDispatcher, - Constants, DevicePropertyMap, getConfigServices, RelStorageInvalidationPoller, @@ -126,10 +126,8 @@ def _synchronize(self): count = _removeDeleted(self.log, tool, self.store) if count == 0: self.log.info("no dangling configurations found") - timelimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, - Constants.build_timeout_id, - Constants.build_timeout_value, + timelimitmap = DevicePropertyMap.make_build_timeout_map( + self.ctx.dmd.Devices ) new_devices = _addNew( self.log, tool, timelimitmap, self.store, self.dispatcher @@ -145,41 +143,11 @@ def _process(self, invalidation): self.store.search(ConfigQuery(monitor=monitor, device=device.id)) ) if not keys: - timelimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, - Constants.build_timeout_id, - Constants.build_timeout_value, - ) - uid = device.getPrimaryId() - timeout = timelimitmap.get(uid) - self.dispatcher.dispatch_all(monitor, device.id, timeout) - self.log.info( - "submitted build jobs for new device uid=%s monitor=%s", - uid, - monitor, - ) + self._new_device(device, monitor) elif reason is InvalidationCause.Updated: - self.store.set_expired(*keys) - for key in keys: - self.log.info( - "expired configuration of changed device " - "device=%s monitor=%s service=%s device-oid=%r", - key.device, - key.monitor, - key.service, - invalidation.oid, - ) + self._updated_device(device, monitor, keys, invalidation) elif reason is InvalidationCause.Removed: - self.store.remove(*keys) - for key in keys: - self.log.info( - "removed configuration of deleted device " - "device=%s monitor=%s service=%s device-oid=%r", - key.device, - key.monitor, - key.service, - invalidation.oid, - ) + self._removed_device(keys, invalidation) else: self.log.warn( "ignored unexpected reason " @@ -190,6 +158,69 @@ def _process(self, invalidation): invalidation.oid, ) + def _new_device(self, device, monitor): + timelimitmap = DevicePropertyMap.make_build_timeout_map( + self.ctx.dmd.Devices + ) + uid = device.getPrimaryId() + timeout = timelimitmap.get(uid) + self.dispatcher.dispatch_all(monitor, device.id, timeout) + self.log.info( + "submitted build jobs for new device uid=%s monitor=%s", + uid, + monitor, + ) + + def _updated_device(self, device, monitor, keys, invalidation): + minagelimitmap = DevicePropertyMap.make_minimum_ttl_map( + self.ctx.dmd.Devices + ) + statuses = tuple( + (key, status) + for key, status in self.store.get_status(*keys) + if isinstance(status, ConfigStatus.Current) + ) + uid = device.getPrimaryId() + now = time.time() + retired = set( + key + for key, status in statuses + if status.updated >= now - minagelimitmap.get(uid) + ) + expired = set(key for key, _ in statuses if key not in retired) + retired = self.store.set_retired(*retired) + expired = self.store.set_expired(*expired) + for key in retired: + self.log.info( + "retired configuration of changed device " + "device=%s monitor=%s service=%s device-oid=%r", + key.device, + key.monitor, + key.service, + invalidation.oid, + ) + for key in expired: + self.log.info( + "expired configuration of changed device " + "device=%s monitor=%s service=%s device-oid=%r", + key.device, + key.monitor, + key.service, + invalidation.oid, + ) + + def _removed_device(self, keys, invalidation): + self.store.remove(*keys) + for key in keys: + self.log.info( + "removed configuration of deleted device " + "device=%s monitor=%s service=%s device-oid=%r", + key.device, + key.monitor, + key.service, + invalidation.oid, + ) + _solr_fields = ("id", "collector", "uid") diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 44a332d422..6f59b45487 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -89,16 +89,15 @@ def run(self): try: self.ctx.session.sync() self._retry_pending_builds() + self._expire_retired_configs() self._rebuild_older_configs() except Exception as ex: self.log.exception("unexpected error %s", ex) self.ctx.controller.wait(self.interval) def _retry_pending_builds(self): - pendinglimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, - Constants.pending_timeout_id, - Constants.pending_timeout_value, + pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( + self.ctx.dmd.Devices ) now = time() count = 0 @@ -123,31 +122,41 @@ def _retry_pending_builds(self): if count == 0: self.log.debug("no pending configuration builds have timed out") - def _rebuild_older_configs(self): - buildlimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, - Constants.build_timeout_id, - Constants.build_timeout_value, + def _expire_retired_configs(self): + retired = ( + (key, status, self.store.get_uid(key.device)) + for key, status in self.store.get_retired() + ) + minttl_map = DevicePropertyMap.make_minimum_ttl_map( + self.ctx.dmd.Devices + ) + now = time() + expire = tuple( + key + for key, status, uid in retired + if status.updated < now - minttl_map.get(uid) ) - agelimitmap = DevicePropertyMap.from_organizer( - self.ctx.dmd.Devices, - Constants.time_to_live_id, - Constants.time_to_live_value, + self.store.set_expired(*expire) + + def _rebuild_older_configs(self): + buildlimitmap = DevicePropertyMap.make_build_timeout_map( + self.ctx.dmd.Devices ) - min_limit = agelimitmap.smallest_value() + ttlmap = DevicePropertyMap.make_ttl_map(self.ctx.dmd.Devices) + min_ttl = ttlmap.smallest_value() self.log.debug( - "minimum age limit is %s", _formatted_interval(min_limit) + "minimum age limit is %s", _formatted_interval(min_ttl) ) now = time() - min_age = now - min_limit + min_age = now - min_ttl results = chain.from_iterable( (self.store.get_expired(), self.store.get_older(min_age)) ) count = 0 for key, status in results: uid = self.store.get_uid(key.device) - ttl_limit = agelimitmap.get(uid) - expiration_threshold = now - ttl_limit + ttl = ttlmap.get(uid) + expiration_threshold = now - ttl if ( isinstance(status, ConfigStatus.Expired) or status.updated <= expiration_threshold @@ -173,7 +182,7 @@ def _rebuild_older_configs(self): "%Y-%m-%d %H:%M:%S" ), Constants.time_to_live_id, - ttl_limit, + ttl, key.service, key.monitor, key.device, diff --git a/Products/ZenCollector/configcache/tests/test_propertymap.py b/Products/ZenCollector/configcache/tests/test_propertymap.py index a0c63c91c7..51c16b7d68 100644 --- a/Products/ZenCollector/configcache/tests/test_propertymap.py +++ b/Products/ZenCollector/configcache/tests/test_propertymap.py @@ -1,6 +1,6 @@ ############################################################################## # -# Copyright (C) Zenoss, Inc. 2019, all rights reserved. +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. @@ -14,8 +14,8 @@ from ..utils.propertymap import DevicePropertyMap -class EmptyBestMatchMapTest(TestCase): - """Test an empty BestMatchMap object.""" +class EmptyDevicePropertyMapTest(TestCase): + """Test an empty DevicePropertyMap object.""" def setUp(t): t.bmm = DevicePropertyMap({}, None) @@ -30,8 +30,8 @@ def test_smallest_value(t): t.assertIsNone(t.bmm.smallest_value()) -class BestMatchMapTest(TestCase): - """Test a BestMatchMap object.""" +class DevicePropertyMapTest(TestCase): + """Test a DevicePropertyMap object.""" mapping = { "/zport/dmd/Devices": 10, @@ -49,7 +49,7 @@ def setUp(t): def tearDown(t): del t.bmm - def test_get_root(t): + def test_minimal_match(t): value = t.bmm.get("/zport/dmd/Devices/Server-stuff/devices/dev2") t.assertEqual(10, value) @@ -63,7 +63,7 @@ def test_get_best_match(t): value = t.bmm.get("/zport/dmd/Devices/Server/Linux/devices/dev3") t.assertEqual(11, value) - def test_get_too_short_request(t): + def test_no_match(t): value = t.bmm.get("/Devices") t.assertEqual(t._default, value) diff --git a/Products/ZenCollector/configcache/tests/test_propertymap_makers.py b/Products/ZenCollector/configcache/tests/test_propertymap_makers.py new file mode 100644 index 0000000000..8717377fd1 --- /dev/null +++ b/Products/ZenCollector/configcache/tests/test_propertymap_makers.py @@ -0,0 +1,276 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +from Products.ZenTestCase.BaseTestCase import BaseTestCase + +from ..utils.constants import Constants +from ..utils.propertymap import DevicePropertyMap + + +class TestDevicePropertyMapTTLMakers(BaseTestCase): + + ttl_overrides = { + "Server/Linux": 16320, + "Server/Linux/linux0": 68000, + "Power": 8000, + "Network": 32000, + } + + min_ttl_overrides = { + "Server/Linux/linux0": 300, + } + + def afterSetUp(t): + super(TestDevicePropertyMapTTLMakers, t).afterSetUp() + + t.dmd.Devices.createOrganizer("/Server/Linux") + t.dmd.Devices.createOrganizer("/Server/Cmd") + t.dmd.Devices.createOrganizer("/Network") + t.dmd.Devices.createOrganizer("/Power") + + t.dmd.Devices.Server.Linux.setZenProperty( + Constants.time_to_live_id, t.ttl_overrides["Server/Linux"] + ) + t.dmd.Devices.Power.setZenProperty( + Constants.time_to_live_id, t.ttl_overrides["Power"] + ) + t.dmd.Devices.Network.setZenProperty( + Constants.time_to_live_id, t.ttl_overrides["Network"] + ) + + t.linux_dev = t.dmd.Devices.Server.Linux.createInstance("linux0") + t.linux_dev.setZenProperty( + Constants.time_to_live_id, t.ttl_overrides["Server/Linux/linux0"] + ) + t.linux_dev.setZenProperty( + Constants.minimum_time_to_live_id, + t.min_ttl_overrides["Server/Linux/linux0"], + ) + + t.cmd_dev = t.dmd.Devices.Server.Cmd.createInstance("cmd0") + + def test_make_ttl_map(t): + ttlmap = DevicePropertyMap.make_ttl_map(t.dmd.Devices) + + pathid = t.dmd.Devices.Server.getPrimaryId() + expected = Constants.time_to_live_value + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Server.Linux.getPrimaryId() + expected = t.ttl_overrides["Server/Linux"] + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Server.Cmd.getPrimaryId() + expected = Constants.time_to_live_value + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Power.getPrimaryId() + expected = t.ttl_overrides["Power"] + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Network.getPrimaryId() + expected = t.ttl_overrides["Network"] + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.linux_dev.getPrimaryId() + expected = t.ttl_overrides["Server/Linux/linux0"] + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.cmd_dev.getPrimaryId() + expected = Constants.time_to_live_value + actual = ttlmap.get(pathid) + t.assertEqual(expected, actual) + + def test_make_min_ttl_map(t): + minttlmap = DevicePropertyMap.make_minimum_ttl_map(t.dmd.Devices) + + pathid = t.dmd.Devices.Server.getPrimaryId() + expected = Constants.minimum_time_to_live_value + actual = minttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.linux_dev.getPrimaryId() + expected = t.min_ttl_overrides["Server/Linux/linux0"] + actual = minttlmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.cmd_dev.getPrimaryId() + expected = Constants.minimum_time_to_live_value + actual = minttlmap.get(pathid) + t.assertEqual(expected, actual) + + def test_bad_min_ttl_value(t): + bad_minttl_value = Constants.time_to_live_value + 100 + t.cmd_dev.setZenProperty( + Constants.minimum_time_to_live_id, + bad_minttl_value, + ) + + minttlmap = DevicePropertyMap.make_minimum_ttl_map(t.dmd.Devices) + + pathid = t.cmd_dev.getPrimaryId() + expected = Constants.time_to_live_value + actual = minttlmap.get(pathid) + t.assertEqual(expected, actual) + + +class TestDevicePropertyMapPendingTimeout(BaseTestCase): + + pending_overrides = { + "Server/Linux": 500, + "Server/Linux/linux0": 600, + "Power": 800, + "Network": 850, + } + + def afterSetUp(t): + super(TestDevicePropertyMapPendingTimeout, t).afterSetUp() + + t.dmd.Devices.createOrganizer("/Server/Linux") + t.dmd.Devices.createOrganizer("/Server/Cmd") + t.dmd.Devices.createOrganizer("/Network") + t.dmd.Devices.createOrganizer("/Power") + + t.dmd.Devices.Server.Linux.setZenProperty( + Constants.pending_timeout_id, t.pending_overrides["Server/Linux"] + ) + t.dmd.Devices.Power.setZenProperty( + Constants.pending_timeout_id, t.pending_overrides["Power"] + ) + t.dmd.Devices.Network.setZenProperty( + Constants.pending_timeout_id, t.pending_overrides["Network"] + ) + + t.linux_dev = t.dmd.Devices.Server.Linux.createInstance("linux0") + t.linux_dev.setZenProperty( + Constants.pending_timeout_id, + t.pending_overrides["Server/Linux/linux0"], + ) + + t.cmd_dev = t.dmd.Devices.Server.Cmd.createInstance("cmd0") + + def test_make_pending_timeout_map(t): + pendingmap = DevicePropertyMap.make_pending_timeout_map(t.dmd.Devices) + + pathid = t.dmd.Devices.Server.getPrimaryId() + expected = Constants.pending_timeout_value + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Server.Linux.getPrimaryId() + expected = t.pending_overrides["Server/Linux"] + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Server.Cmd.getPrimaryId() + expected = Constants.pending_timeout_value + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Power.getPrimaryId() + expected = t.pending_overrides["Power"] + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Network.getPrimaryId() + expected = t.pending_overrides["Network"] + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.linux_dev.getPrimaryId() + expected = t.pending_overrides["Server/Linux/linux0"] + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.cmd_dev.getPrimaryId() + expected = Constants.pending_timeout_value + actual = pendingmap.get(pathid) + t.assertEqual(expected, actual) + + +class TestDevicePropertyMapBuildTimeout(BaseTestCase): + + build_overrides = { + "Server/Linux": 500, + "Server/Linux/linux0": 600, + "Power": 800, + "Network": 850, + } + + def afterSetUp(t): + super(TestDevicePropertyMapBuildTimeout, t).afterSetUp() + + t.dmd.Devices.createOrganizer("/Server/Linux") + t.dmd.Devices.createOrganizer("/Server/Cmd") + t.dmd.Devices.createOrganizer("/Network") + t.dmd.Devices.createOrganizer("/Power") + + t.dmd.Devices.Server.Linux.setZenProperty( + Constants.build_timeout_id, t.build_overrides["Server/Linux"] + ) + t.dmd.Devices.Power.setZenProperty( + Constants.build_timeout_id, t.build_overrides["Power"] + ) + t.dmd.Devices.Network.setZenProperty( + Constants.build_timeout_id, t.build_overrides["Network"] + ) + + t.linux_dev = t.dmd.Devices.Server.Linux.createInstance("linux0") + t.linux_dev.setZenProperty( + Constants.build_timeout_id, + t.build_overrides["Server/Linux/linux0"], + ) + + t.cmd_dev = t.dmd.Devices.Server.Cmd.createInstance("cmd0") + + def test_make_build_timeout_map(t): + buildmap = DevicePropertyMap.make_build_timeout_map(t.dmd.Devices) + + pathid = t.dmd.Devices.Server.getPrimaryId() + expected = Constants.build_timeout_value + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Server.Linux.getPrimaryId() + expected = t.build_overrides["Server/Linux"] + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Server.Cmd.getPrimaryId() + expected = Constants.build_timeout_value + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Power.getPrimaryId() + expected = t.build_overrides["Power"] + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.dmd.Devices.Network.getPrimaryId() + expected = t.build_overrides["Network"] + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.linux_dev.getPrimaryId() + expected = t.build_overrides["Server/Linux/linux0"] + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) + + pathid = t.cmd_dev.getPrimaryId() + expected = Constants.build_timeout_value + actual = buildmap.get(pathid) + t.assertEqual(expected, actual) diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py index 21b35a6995..226b8e579d 100644 --- a/Products/ZenCollector/configcache/tests/test_storage.py +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -397,16 +397,85 @@ def test_get_newer_greater_multiple(t): t.assertEqual(t.record2.updated, status.updated) -class ConfigLifeCycleTest(_BaseTest): +class TestRetiredStatus(_BaseTest): """ - Test status transitions. + Test APIs regarding the ConfigStatus.Retired status. """ - def test_expired(t): + def test_set_retired(t): + t.store.add(t.record1) + expected = (t.record1.key,) + + actual = t.store.set_retired(t.record1.key) + t.assertTupleEqual(expected, actual) + + def test_set_retired_twice(t): + t.store.add(t.record1) + expected = () + + t.store.set_retired(t.record1.key) + actual = t.store.set_retired(t.record1.key) + t.assertTupleEqual(expected, actual) + + def test_retired_status(t): + t.store.add(t.record1) + t.store.set_retired(t.record1.key) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(status.updated, t.record1.updated) + + def test_get_retired(t): + t.store.add(t.record1) + t.store.set_retired(t.record1.key) + + result = tuple(t.store.get_retired()) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(status.updated, t.record1.updated) + + +class TestExpiredStatus(_BaseTest): + """ + Test APIs regarding the ConfigStatus.Expired status. + """ + + def test_set_expired(t): + t.store.add(t.record1) + expected = (t.record1.key,) + + actual = t.store.set_expired(t.record1.key) + t.assertTupleEqual(expected, actual) + + def test_set_expired_twice(t): + t.store.add(t.record1) + expected = () + + t.store.set_expired(t.record1.key) + actual = t.store.set_expired(t.record1.key) + t.assertTupleEqual(expected, actual) + + def test_expired_status(t): t.store.add(t.record1) t.store.set_expired(t.record1.key) result = tuple(t.store.get_status(t.record1.key)) + + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Expired) + + def test_get_expired(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + + result = tuple(t.store.get_expired()) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) @@ -419,24 +488,192 @@ def test_expired_is_not_older(t): result = tuple(t.store.get_older(t.record1.updated)) t.assertEqual(0, len(result)) - def test_expire_pending(t): + +class TestPendingStatus(_BaseTest): + """ + Test APIs regarding the ConfigStatus.Pending status. + """ + + def test_set_pending(t): t.store.add(t.record1) submitted = t.record1.updated + 500 + expected = (t.record1.key,) + t.store.set_expired(t.record1.key) + actual = t.store.set_pending((t.record1.key, submitted)) + t.assertTupleEqual(expected, actual) + + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) + + def test_set_pending_twice(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + expected = () + t.store.set_pending((t.record1.key, submitted)) + actual = t.store.set_pending((t.record1.key, submitted)) + t.assertTupleEqual(expected, actual) + + def test_pending_status(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Expired) + t.assertIsInstance(status, ConfigStatus.Pending) + t.assertEqual(submitted, status.submitted) - def test_pending_without_expiring(t): + def test_get_pending(t): t.store.add(t.record1) + t.store.set_expired(t.record1.key) submitted = t.record1.updated + 500 t.store.set_pending((t.record1.key, submitted)) + result = tuple(t.store.get_pending()) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Pending) + t.assertEqual(submitted, status.submitted) + + def test_pending_is_not_older(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + submitted = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) + + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(0, len(result)) + + +class TestBuildingStatus(_BaseTest): + """ + Test APIs regarding the ConfigStatus.Building status. + """ + + def test_set_building(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + expected = (t.record1.key,) + + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) + actual = t.store.set_building((t.record1.key, started)) + t.assertTupleEqual(expected, actual) + + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) + + def test_set_building_twice(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + expected = () + + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) + t.store.set_building((t.record1.key, started)) + actual = t.store.set_building((t.record1.key, started)) + t.assertTupleEqual(expected, actual) + + def test_building_status(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) + t.store.set_building((t.record1.key, started)) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + def test_get_building(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + started = t.record1.updated + 500 + t.store.set_pending((t.record1.key, started - 100)) + t.store.set_building((t.record1.key, started)) + + result = tuple(t.store.get_building()) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + def test_building_is_not_older(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + started = t.record1.updated + 500 + t.store.set_building((t.record1.key, started)) + + result = tuple(t.store.get_older(t.record1.updated)) + t.assertEqual(0, len(result)) + + +class TestExpiredTransitions(_BaseTest): + """ + Test transitions to and from ConfigStatus.Expired. + """ + + def test_retired_to_expired(t): + t.store.add(t.record1) + t.store.set_retired(t.record1.key) + + expired_keys = t.store.set_expired(t.record1.key) + t.assertTupleEqual((t.record1.key,), expired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Expired) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) + + def test_expired_to_retired(t): + t.store.add(t.record1) + t.store.set_expired(t.record1.key) + retired_keys = t.store.set_retired(t.record1.key) + t.assertTupleEqual((), retired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Expired) + + +class TestPendingTransitions(_BaseTest): + """ + Test transitions to and from ConfigStatus.Pending. + """ + + def test_current_to_pending(t): + t.store.add(t.record1) + submitted = t.record1.updated + 500 + + pending_keys = t.store.set_pending((t.record1.key, submitted)) + t.assertTupleEqual((), pending_keys) + result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] @@ -444,35 +681,127 @@ def test_pending_without_expiring(t): t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - def test_pending(t): + def test_retired_to_pending(t): + t.store.add(t.record1) + t.store.set_retired(t.record1.key) + submitted = t.record1.updated + 500 + + pending_keys = t.store.set_pending((t.record1.key, submitted)) + t.assertTupleEqual((), pending_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(t.record1.updated, status.updated) + + def test_expired_to_pending(t): t.store.add(t.record1) submitted = t.record1.updated + 500 t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, submitted)) + pending_keys = t.store.set_pending((t.record1.key, submitted)) + t.assertTupleEqual((t.record1.key,), pending_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) t.assertIsInstance(status, ConfigStatus.Pending) - t.assertEqual(submitted, status.submitted) - def test_pending_is_not_older(t): + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) + + def test_pending_to_expired(t): t.store.add(t.record1) + submitted = t.record1.updated + 500 t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, submitted)) + + expired_keys = t.store.set_expired(t.record1.key) + t.assertTupleEqual((), expired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Pending) + + def test_pending_to_retired(t): + t.store.add(t.record1) submitted = t.record1.updated + 500 t.store.set_expired(t.record1.key) t.store.set_pending((t.record1.key, submitted)) - result = tuple(t.store.get_older(t.record1.updated)) - t.assertEqual(0, len(result)) + retired_keys = t.store.set_retired(t.record1.key) + t.assertTupleEqual((), retired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Pending) + + +class TestBuildingTransitions(_BaseTest): + """ + Test transitions to and from ConfigStatus.Building. + """ + + def test_current_to_building(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + + building_keys = t.store.set_building((t.record1.key, started)) + t.assertTupleEqual((), building_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) + + def test_retired_to_building(t): + t.store.add(t.record1) + t.store.set_retired(t.record1.key) + started = t.record1.updated + 500 + + building_keys = t.store.set_building((t.record1.key, started)) + t.assertTupleEqual((), building_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(t.record1.updated, status.updated) - def test_building(t): + def test_expired_to_building(t): t.store.add(t.record1) started = t.record1.updated + 500 t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, started)) - t.store.set_building((t.record1.key, started)) + + building_keys = t.store.set_building((t.record1.key, started)) + t.assertTupleEqual((), building_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Expired) + + def test_pending_to_building(t): + t.store.add(t.record1) + started = t.record1.updated + 500 + t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) + + building_keys = t.store.set_building((t.record1.key, started)) + t.assertTupleEqual((t.record1.key,), building_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) @@ -481,12 +810,25 @@ def test_building(t): t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) - def test_building_without_pending(t): + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) + + def test_building_to_pending(t): t.store.add(t.record1) started = t.record1.updated + 500 t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) + pending_keys = t.store.set_pending((t.record1.key, started)) + t.assertTupleEqual((), pending_keys) + result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] @@ -494,26 +836,75 @@ def test_building_without_pending(t): t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) - def test_building_is_not_older(t): + def test_building_to_expired(t): t.store.add(t.record1) + started = t.record1.updated + 500 t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) + t.store.set_building((t.record1.key, started)) + + expired_keys = t.store.set_expired(t.record1.key) + t.assertTupleEqual((), expired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + def test_building_to_retired(t): + t.store.add(t.record1) started = t.record1.updated + 500 t.store.set_expired(t.record1.key) + t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) - result = tuple(t.store.get_older(t.record1.updated)) - t.assertEqual(0, len(result)) + retired_keys = t.store.set_retired(t.record1.key) + t.assertTupleEqual((), retired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + +class TestAddTransitions(_BaseTest): + """ + Test status changes after adding a config. + """ + + def test_add_overwrites_retired(t): + t.store.add(t.record1) + t.store.set_retired(t.record1.key) + t.store.add(t.record1) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) + + result = tuple(t.store.get_status(t.record1.key)) + t.assertEqual(1, len(result)) + key, status = result[0] + t.assertEqual(t.record1.key, key) + t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) def test_add_overwrites_expired(t): t.store.add(t.record1) t.store.set_expired(t.record1.key) t.store.add(t.record1) + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) def test_add_overwrites_pending(t): t.store.add(t.record1) @@ -522,24 +913,42 @@ def test_add_overwrites_pending(t): t.store.set_pending((t.record1.key, submitted)) t.store.add(t.record1) + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) def test_add_overwrites_building(t): t.store.add(t.record1) - submitted = t.record1.updated + 500 + started = t.record1.updated + 500 t.store.set_expired(t.record1.key) - t.store.set_building((t.record1.key, submitted)) + t.store.set_pending((t.record1.key, started - 100)) + t.store.set_building((t.record1.key, started)) t.store.add(t.record1) + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + + building_keys = tuple(t.store.get_building()) + t.assertTupleEqual((), building_keys) + result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) t.assertIsInstance(status, ConfigStatus.Current) + t.assertEqual(t.record1.updated, status.updated) class DeviceMonitorChangeTest(_BaseTest): diff --git a/Products/ZenCollector/configcache/utils/__init__.py b/Products/ZenCollector/configcache/utils/__init__.py index b15cd0ae42..00910dea16 100644 --- a/Products/ZenCollector/configcache/utils/__init__.py +++ b/Products/ZenCollector/configcache/utils/__init__.py @@ -9,28 +9,17 @@ from __future__ import absolute_import -from .propertymap import DevicePropertyMap +from .constants import Constants from .dispatcher import BuildConfigTaskDispatcher from .pollers import RelStorageInvalidationPoller +from .propertymap import DevicePropertyMap from .services import getConfigServices -class Constants(object): - - build_timeout_id = "zDeviceConfigBuildTimeout" - pending_timeout_id = "zDeviceConfigPendingTimeout" - time_to_live_id = "zDeviceConfigTTL" - - # Default values - build_timeout_value = 7200 - pending_timeout_value = 7200 - time_to_live_value = 43200 - - __all__ = ( "BuildConfigTaskDispatcher", "Constants", "DevicePropertyMap", - "RelStorageInvalidationPoller", "getConfigServices", + "RelStorageInvalidationPoller", ) diff --git a/Products/ZenCollector/configcache/utils/constants.py b/Products/ZenCollector/configcache/utils/constants.py new file mode 100644 index 0000000000..667ec2ffaa --- /dev/null +++ b/Products/ZenCollector/configcache/utils/constants.py @@ -0,0 +1,22 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +class Constants(object): + + build_timeout_id = "zDeviceConfigBuildTimeout" + pending_timeout_id = "zDeviceConfigPendingTimeout" + time_to_live_id = "zDeviceConfigTTL" + minimum_time_to_live_id = "zDeviceConfigMinimumTTL" + + # Default values + build_timeout_value = 7200 + pending_timeout_value = 7200 + time_to_live_value = 43200 + minimum_time_to_live_value = 0 diff --git a/Products/ZenCollector/configcache/utils/propertymap.py b/Products/ZenCollector/configcache/utils/propertymap.py index 5a90c03b20..6567d0330a 100644 --- a/Products/ZenCollector/configcache/utils/propertymap.py +++ b/Products/ZenCollector/configcache/utils/propertymap.py @@ -9,6 +9,8 @@ import logging +from .constants import Constants + log = logging.getLogger("zen.configcache.propertymap") @@ -24,17 +26,56 @@ class DevicePropertyMap(object): """ @classmethod - def from_organizer(cls, obj, propname, default, relName="devices"): + def make_ttl_map(cls, obj): + return cls( + getPropertyValues( + obj, + Constants.time_to_live_id, + Constants.time_to_live_value, + _getZProperty, + ), + Constants.time_to_live_value, + ) + + @classmethod + def make_minimum_ttl_map(cls, obj): return cls( - getPropertyValues(obj, propname, default, relName=relName), - default + getPropertyValues( + obj, + Constants.minimum_time_to_live_id, + Constants.minimum_time_to_live_value, + _getZDeviceConfigMinimumTTL, + ), + Constants.minimum_time_to_live_value, + ) + + @classmethod + def make_pending_timeout_map(cls, obj): + return cls( + getPropertyValues( + obj, + Constants.pending_timeout_id, + Constants.pending_timeout_value, + _getZProperty, + ), + Constants.pending_timeout_value, + ) + + @classmethod + def make_build_timeout_map(cls, obj): + return cls( + getPropertyValues( + obj, + Constants.build_timeout_id, + Constants.build_timeout_value, + _getZProperty, + ), + Constants.build_timeout_value, ) def __init__(self, values, default): self.__values = tuple( - (p.split("/")[1:], v) - for p, v in values.items() - if v is not None + (p.split("/")[1:], v) for p, v in values.items() if v is not None ) self.__default = default @@ -70,18 +111,18 @@ def get(self, request_uid): return self.__default -def getPropertyValues(obj, propname, default, relName="devices"): +def getPropertyValues(obj, propname, default, getter, relName="devices"): """ Returns a mapping of UID -> property-value for the given z-property. """ - values = {obj.getPrimaryId(): _getValue(obj, propname, default)} + values = {obj.getPrimaryId(): getter(obj, propname, default)} values.update( - (inst.getPrimaryId(), _getValue(inst, propname, default)) + (inst.getPrimaryId(), getter(inst, propname, default)) for inst in obj.getSubInstances(relName) if inst.isLocal(propname) ) values.update( - (inst.getPrimaryId(), _getValue(inst, propname, default)) + (inst.getPrimaryId(), getter(inst, propname, default)) for inst in obj.getOverriddenObjects(propname) ) if not values or any(v is None for v in values.values()): @@ -92,8 +133,24 @@ def getPropertyValues(obj, propname, default, relName="devices"): return values -def _getValue(obj, propname, default): +def _getZProperty(obj, propname, default): value = obj.getZ(propname) if value is None: return default return value + + +def _getZDeviceConfigMinimumTTL(obj, propname, default): + """ + Compares zDeviceConfigTTL and zDeviceConfigMinimumTTL and + returns the lesser of the two. + """ + ttl = _getZProperty( + obj, Constants.time_to_live_id, Constants.time_to_live_value + ) + minttl = _getZProperty( + obj, + Constants.minimum_time_to_live_id, + Constants.minimum_time_to_live_value, + ) + return minttl if minttl < ttl else ttl diff --git a/Products/ZenModel/migrate/addConfigCacheProperties.py b/Products/ZenModel/migrate/addConfigCacheProperties.py index aa8bd71d62..4f31cdd6e5 100644 --- a/Products/ZenModel/migrate/addConfigCacheProperties.py +++ b/Products/ZenModel/migrate/addConfigCacheProperties.py @@ -9,6 +9,7 @@ from __future__ import absolute_import +from Products.ZenCollector.configcache.utils import Constants from Products.ZenRelations.zPropertyCategory import setzPropertyCategory from . import Migrate @@ -17,18 +18,32 @@ _properties = ( ( - ("zDeviceConfigTTL", 43200), + (Constants.time_to_live_id, Constants.time_to_live_value), { "type": "int", "label": "Device configuration expiration", "description": ( - "The number of seconds to wait before rebuilding a " + "The maximum number of seconds to wait before rebuilding a " "device configuration." ), - } + }, ), ( - ("zDeviceConfigBuildTimeout", 7200), + ( + Constants.minimum_time_to_live_id, + Constants.minimum_time_to_live_value, + ), + { + "type": "int", + "label": "Device configuration pre-expiration window", + "description": ( + "The number of seconds the configuration is protected " + "from being rebuilt." + ), + }, + ), + ( + (Constants.build_timeout_id, Constants.build_timeout_value), { "type": "int", "label": "Device configuration build timeout", @@ -36,10 +51,10 @@ "The number of seconds allowed for building a device " "configuration." ), - } + }, ), ( - ("zDeviceConfigPendingTimeout", 7200), + (Constants.pending_timeout_id, Constants.pending_timeout_value), { "type": "int", "label": "Device configuration build queued timeout", @@ -47,7 +62,7 @@ "The number of seconds a device configuration build may be " "queued before a timeout." ), - } + }, ), ) diff --git a/Products/ZenRelations/ZenPropertyManager.py b/Products/ZenRelations/ZenPropertyManager.py index 8233511fec..31396f1fa7 100644 --- a/Products/ZenRelations/ZenPropertyManager.py +++ b/Products/ZenRelations/ZenPropertyManager.py @@ -65,9 +65,17 @@ 43200, "int", "Device configuration expiration", - "The number of seconds to wait before rebuilding a " + "The maximum number of seconds to wait before rebuilding a " "device configuration." ), + ( + "zDeviceConfigMinimumTTL", + 0, + "int", + "Device configuration pre-expiration window", + "The number of seconds the configuration is protected " + "from being rebuilt." + ), # zPythonClass maps device class to python classs (separate from device # class name) ( diff --git a/Products/ZenRelations/zPropertyCategory.py b/Products/ZenRelations/zPropertyCategory.py index fc1eb7f06e..cd9d515ae9 100644 --- a/Products/ZenRelations/zPropertyCategory.py +++ b/Products/ZenRelations/zPropertyCategory.py @@ -45,6 +45,7 @@ # Configuration Cache # ------------------- "zDeviceConfigTTL": "Config Cache", + "zDeviceConfigMinimumTTL": "Config Cache", "zDeviceConfigBuildTimeout": "Config Cache", "zDeviceConfigPendingTimeout": "Config Cache", # From 50f6728f43a0d8726dc8345bafafa03f7a8da0d7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 8 Feb 2024 14:20:38 -0600 Subject: [PATCH 059/147] fix: don't import ZenPacks at module level. --- Products/ZenCollector/configcache/utils/services.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Products/ZenCollector/configcache/utils/services.py b/Products/ZenCollector/configcache/utils/services.py index 99870302e5..f685405d7e 100644 --- a/Products/ZenCollector/configcache/utils/services.py +++ b/Products/ZenCollector/configcache/utils/services.py @@ -15,7 +15,6 @@ import pathlib2 as pathlib import Products -import ZenPacks from Products.ZenCollector.services.config import CollectorConfigService @@ -87,6 +86,10 @@ def getConfigServices(): :returns: Tuple of configuration service classes :rtype: tuple[CollectorConfigService] """ + # defer import ZenPacks until here because it doesn't exist during + # an image build. + import ZenPacks + search_paths = ( pathlib.Path(p) for p in itertools.chain(Products.__path__, ZenPacks.__path__) From 2cd2228873e0524c3afed2724d0b0a15ece0aa3b Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 8 Feb 2024 15:38:00 -0600 Subject: [PATCH 060/147] fix: add 'retired' support to configcache list command. ZEN-34656 --- Products/ZenCollector/configcache/cli.py | 5 ++++- Products/ZenCollector/configcache/invalidator.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index f5dc080cbf..5a41d3958e 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -51,7 +51,7 @@ def add_arguments(parser, subparsers): "-f", dest="states", action=MultiChoice, - choices=("current", "expired", "pending", "building"), + choices=("current", "retired", "expired", "pending", "building"), default=argparse.SUPPRESS, help="Only list configurations having these states. One or " "more states may be specified, separated by commas.", @@ -125,6 +125,7 @@ def run(self): _name_state_lookup = { "current": ConfigStatus.Current, + "retired": ConfigStatus.Retired, "expired": ConfigStatus.Expired, "pending": ConfigStatus.Pending, "building": ConfigStatus.Building, @@ -134,6 +135,8 @@ def run(self): def _format_status(status): if isinstance(status, ConfigStatus.Current): return "last updated {}".format(_format_date(status.updated)) + elif isinstance(status, ConfigStatus.Retired): + return "retired" elif isinstance(status, ConfigStatus.Expired): return "expired" elif isinstance(status, ConfigStatus.Pending): diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 0f8991a5b0..c216f16c95 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -181,11 +181,11 @@ def _updated_device(self, device, monitor, keys, invalidation): if isinstance(status, ConfigStatus.Current) ) uid = device.getPrimaryId() + minttl = minagelimitmap.get(uid) now = time.time() + limit = now - minttl retired = set( - key - for key, status in statuses - if status.updated >= now - minagelimitmap.get(uid) + key for key, status in statuses if status.updated >= limit ) expired = set(key for key, _ in statuses if key not in retired) retired = self.store.set_retired(*retired) From 2ffcd45f2e2d28b23ba72f815181802038a1bda1 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 9 Feb 2024 08:28:02 -0600 Subject: [PATCH 061/147] fix: make curl be silent in dumpstats ZEN-34622 --- bin/dumpstats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/dumpstats b/bin/dumpstats index 72b5b0f3b9..2a9cf4a58f 100755 --- a/bin/dumpstats +++ b/bin/dumpstats @@ -12,7 +12,7 @@ getprop() { _PORT=$(getprop 'localport') PORT=${_PORT:-14682} URL=http://localhost:$PORT/stats -curl -f $URL +curl -fs $URL if [ $? -eq 0 ]; then echo fi From c2cab26fe1193d896300439fb8eeee8115dab886 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 12 Feb 2024 10:03:33 -0600 Subject: [PATCH 062/147] fix: skip devices having an undefined monitor. Devices without a monitor cannot be monitored and so do not need configs. ZEN-34701 --- Products/ZenCollector/configcache/invalidator.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index c216f16c95..8721854982 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -139,6 +139,14 @@ def _process(self, invalidation): device = invalidation.device reason = invalidation.reason monitor = device.getPerformanceServerName() + if monitor is None: + self.log.warn( + "ignoring invalidated device having undefined monitor " + "device=%s reason=%s", + device, + reason, + ) + return keys = list( self.store.search(ConfigQuery(monitor=monitor, device=device.id)) ) @@ -262,6 +270,13 @@ def _addNew(log, tool, timelimitmap, store, dispatcher): ).results new_devices = [] for brain in catalog_results: + if brain.collector is None: + log.warn( + "ignoring device having undefined monitor device=%s uid=%s", + brain.id, + brain.uid, + ) + continue keys = tuple( store.search(ConfigQuery(monitor=brain.collector, device=brain.id)) ) From c0a75b3742d6c3f411da50be7e403cc0a6cba58a Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Thu, 1 Feb 2024 16:45:23 +0000 Subject: [PATCH 063/147] Add model API for Config Cache components Implements ZEN-34675. Added model API browser view for Config Cache components. It needs for RMMonitor ZP to be able to model these components. --- .../ZenUI3/browser/modelapi/configure.zcml | 22 ++++++++ Products/ZenUI3/browser/modelapi/modelapi.py | 53 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/Products/ZenUI3/browser/modelapi/configure.zcml b/Products/ZenUI3/browser/modelapi/configure.zcml index c1fca95649..3384ca7f25 100644 --- a/Products/ZenUI3/browser/modelapi/configure.zcml +++ b/Products/ZenUI3/browser/modelapi/configure.zcml @@ -90,4 +90,26 @@ permission="zenoss.Common" /> + + + + + + diff --git a/Products/ZenUI3/browser/modelapi/modelapi.py b/Products/ZenUI3/browser/modelapi/modelapi.py index 484c1c70d7..fceacdef5d 100644 --- a/Products/ZenUI3/browser/modelapi/modelapi.py +++ b/Products/ZenUI3/browser/modelapi/modelapi.py @@ -395,3 +395,56 @@ def _getServices(self, svcName): # ZEN-30188 handle name change of the otsdb services in GCP writers += super(Writer, self)._getServices("writer-bigtable") return writers + +class ImpactDaemons(BaseApiView): + """ + This view emits the Impact daemon services + """ + @property + def _services(self): + return ( + ('impacts', 'Impact'), + ('zenImpactStates', 'zenimpactstate'), + ) + +class Memcached(BaseApiView): + + @property + def _services(self): + return ( + ('memcacheds', 'memcacheds'), + ) + + def _getServiceInstances(self, name): + idFormat = '{}' + titleFormat = '{}' + services = self._appfacade.query(name) + if not services: + return [] + svc = services[0] + data = [dict(id=idFormat.format(name), + title=titleFormat.format(name), + controlplaneServiceId=svc.id, + instanceCount=svc.instances, + RAMCommitment=getattr(svc, 'RAMCommitment', None), + lastModeledState=str(svc.state).lower() + ) + for i in range(svc.instances)] + return data + + def _getServices(self, svcName): + memcacheds = self._getServiceInstances('memcached') + memcacheds += self._getServiceInstances('memcached-session') + return memcacheds + +class ConfigCacheDaemons(BaseApiView): + """ + This view emits info for configcache services: invalidator, builders, and manager + """ + @property + def _services(self): + return ( + ('ConfigCache-Invalidators', 'invalidator'), + ('ConfigCache-Builders', 'builder'), + ('ConfigCache-Managers', 'manager'), + ) From be17c739322c1e65723213ea29ed0e93b62f417c Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Mon, 5 Feb 2024 08:59:46 +0000 Subject: [PATCH 064/147] Change configcache modelapi response keys Implements ZEN-34675. To avoid JS errors and unnecessary code on the RM Monitor ZP side the configcahe response keys are corrected. --- Products/ZenUI3/browser/modelapi/modelapi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Products/ZenUI3/browser/modelapi/modelapi.py b/Products/ZenUI3/browser/modelapi/modelapi.py index fceacdef5d..57d432a2d4 100644 --- a/Products/ZenUI3/browser/modelapi/modelapi.py +++ b/Products/ZenUI3/browser/modelapi/modelapi.py @@ -444,7 +444,7 @@ class ConfigCacheDaemons(BaseApiView): @property def _services(self): return ( - ('ConfigCache-Invalidators', 'invalidator'), - ('ConfigCache-Builders', 'builder'), - ('ConfigCache-Managers', 'manager'), + ('configCacheInvalidators', 'invalidator'), + ('configCacheBuilders', 'builder'), + ('configCacheManagers', 'manager'), ) From 64f4903cd18b566e1d5a60580fbd9326b4b4d147 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 12 Feb 2024 14:01:11 -0600 Subject: [PATCH 065/147] remove zing-related code from modelapi.py --- Products/ZenUI3/browser/modelapi/modelapi.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Products/ZenUI3/browser/modelapi/modelapi.py b/Products/ZenUI3/browser/modelapi/modelapi.py index 57d432a2d4..f83fbdf1ee 100644 --- a/Products/ZenUI3/browser/modelapi/modelapi.py +++ b/Products/ZenUI3/browser/modelapi/modelapi.py @@ -310,7 +310,6 @@ def _services(self): return (("regionServers", "RegionServer"),) def _getServices(self, svcName): - # ZEN-30188 there is bigtable instead of RegionServers svc = next(iter(self._appfacade.query(svcName)), None) if not svc: return [] @@ -376,8 +375,6 @@ def _services(self): def _getServices(self, svcName): readers = super(Reader, self)._getServices("reader") - # ZEN-30188 handle name change of the otsdb services in GCP - readers += super(Reader, self)._getServices("reader-bigtable") return readers @@ -392,8 +389,6 @@ def _services(self): def _getServices(self, svcName): writers = super(Writer, self)._getServices("writer") - # ZEN-30188 handle name change of the otsdb services in GCP - writers += super(Writer, self)._getServices("writer-bigtable") return writers class ImpactDaemons(BaseApiView): From 113a374f98f0f922bfe227ecbf39a8e190532fa7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 13 Feb 2024 08:56:20 -0600 Subject: [PATCH 066/147] fix: add zDeviceConfigMinimumTTL to excluded zprops Update the base invalidation filter to ignore changes to zDeviceConfigMinimumTTL z-property. ZEN-34656 --- Products/ZenCollector/configcache/modelchange/filters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Products/ZenCollector/configcache/modelchange/filters.py b/Products/ZenCollector/configcache/modelchange/filters.py index 6f13ba9207..4b7e72bdc1 100644 --- a/Products/ZenCollector/configcache/modelchange/filters.py +++ b/Products/ZenCollector/configcache/modelchange/filters.py @@ -71,6 +71,7 @@ def include(self, obj): Constants.build_timeout_id, Constants.pending_timeout_id, Constants.time_to_live_id, + Constants.minimum_time_to_live_id, ) From 607f97618692f1253f683d8e11df0acd0d22070c Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 14 Feb 2024 07:27:40 -0600 Subject: [PATCH 067/147] fix: switch creation order to avoid race. Moved creating the DeviceConfigLoader ahead of creating the task stats logger which depends on the DeviceConfigLoader object existing. ZEN-34705 --- Products/ZenCollector/daemon.py | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 273a89320d..dc1ad5111a 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -297,8 +297,8 @@ def connected(self): yield self._initEncryptionKey() yield self._startConfigCycle() yield self._startMaintenance() - yield self._startTaskStatsLogging() yield self._startDeviceConfigLoader() + yield self._startTaskStatsLogging() except Exception as ex: self.log.critical("unrecoverable error: %s", ex) self.log.exception("failed during startup") @@ -354,23 +354,6 @@ def _startMaintenance(self): ) self._maintenanceCycle.start() - def _startTaskStatsLogging(self): - if not (self.options.cycle and self.options.logTaskStats): - return - self._taskstatslogger = task.LoopingCall( - self._displayStatistics, verbose=True - ) - self._taskstatsloggerd = self._taskstatslogger.start( - self.options.logTaskStats, now=False - ) - self.log.debug( - "started logging task statistics interval=%d", - self.options.logTaskStats, - ) - reactor.addSystemEventTrigger( - "before", "shutdown", self._taskstatslogger.stop, "before" - ) - def _startDeviceConfigLoader(self): self.log.info( "running the device config loader every %d seconds", @@ -389,6 +372,23 @@ def _startDeviceConfigLoader(self): "before", "shutdown", self._deviceloadertask.stop, "before" ) + def _startTaskStatsLogging(self): + if not (self.options.cycle and self.options.logTaskStats): + return + self._taskstatslogger = task.LoopingCall( + self._displayStatistics, verbose=True + ) + self._taskstatsloggerd = self._taskstatslogger.start( + self.options.logTaskStats, now=False + ) + self.log.debug( + "started logging task statistics interval=%d", + self.options.logTaskStats, + ) + reactor.addSystemEventTrigger( + "before", "shutdown", self._taskstatslogger.stop, "before" + ) + @defer.inlineCallbacks def getRemoteConfigCacheProxy(self): """Return the remote configuration cache proxy.""" From 72b112444775827a25f493b06d7efd4b70864f5c Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 15 Feb 2024 13:35:30 -0600 Subject: [PATCH 068/147] fix: log sudo nmap command at DEBUG level. --- Products/ZenStatus/nmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenStatus/nmap/util.py b/Products/ZenStatus/nmap/util.py index 5ac40f92dd..a9e1f5aba2 100644 --- a/Products/ZenStatus/nmap/util.py +++ b/Products/ZenStatus/nmap/util.py @@ -135,7 +135,7 @@ def executeNmapCmd( if log.isEnabledFor(logging.DEBUG): log.debug("executing nmap %s", " ".join(args)) args = ["-n", _NMAP_BINARY] + args - log.info("Executing /bin/sudo %s", " ".join(args)) + log.debug("Executing /bin/sudo %s", " ".join(args)) out, err, exitCode = yield utils.getProcessOutputAndValue( "/bin/sudo", args ) From 2be160583c38b5417f220da1e9c9e18824bc9a92 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 15 Feb 2024 13:34:53 -0600 Subject: [PATCH 069/147] fix: remove collection tasks for removed device ZEN-34704 --- Products/ZenCollector/config/task.py | 63 +- Products/ZenCollector/daemon.py | 49 +- Products/ZenCollector/frameworkfactory.py | 4 +- Products/ZenCollector/scheduler/__init__.py | 15 + .../ZenCollector/{ => scheduler}/scheduler.py | 572 +++++------------- Products/ZenCollector/scheduler/statistics.py | 79 +++ Products/ZenCollector/scheduler/task.py | 206 +++++++ Products/ZenCollector/tests/testFactory.py | 4 +- Products/ZenHub/PBDaemon.py | 2 + 9 files changed, 532 insertions(+), 462 deletions(-) create mode 100644 Products/ZenCollector/scheduler/__init__.py rename Products/ZenCollector/{ => scheduler}/scheduler.py (56%) create mode 100644 Products/ZenCollector/scheduler/statistics.py create mode 100644 Products/ZenCollector/scheduler/task.py diff --git a/Products/ZenCollector/config/task.py b/Products/ZenCollector/config/task.py index ee320b7a47..3ca7d7d88f 100644 --- a/Products/ZenCollector/config/task.py +++ b/Products/ZenCollector/config/task.py @@ -174,33 +174,42 @@ def _processConfigs(self, config_data): updated = config_data.get("updated", []) removed = config_data.get("removed", []) try: - if self._options.device: - configs = [ - cfg - for cfg in itertools.chain(new, updated) - if self._options.device in (cfg.id, cfg.configId) - ] - if not configs: - log.error( - "configuration for %s unavailable -- " - "is that the correct name?", - self._options.device, - ) - defer.returnValue(None) - - if not new and not updated: - defer.returnValue(None) - - # self.state = self.STATE_PROCESS_DEVICE_CONFIG - yield self._callback(new, updated, removed) - finally: - self._update_local_cache(new, updated, removed) - lengths = (len(new), len(updated), len(removed)) - logmethod = log.debug if lengths == (0, 0, 0) else log.info - logmethod( - "processed %d new, %d updated, and %d removed device configs", - *lengths - ) + try: + if self._options.device: + config = self._get_specified_config(new, updated) + if not config: + log.error( + "configuration for %s unavailable -- " + "is that the correct name?", + self._options.device, + ) + defer.returnValue(None) + new = [config] + updated = [] + removed = [] + + yield self._callback(new, updated, removed) + finally: + self._update_local_cache(new, updated, removed) + lengths = (len(new), len(updated), len(removed)) + logmethod = log.debug if lengths == (0, 0, 0) else log.info + logmethod( + "processed %d new, %d updated, and %d removed " + "device configs", + *lengths + ) + except Exception: + log.exception("failed to process device configs") + + def _get_specified_config(self, new, updated): + return next( + ( + cfg + for cfg in itertools.chain(new, updated) + if self._options.device == cfg.configId + ), + None + ) def _update_local_cache(self, new, updated, removed): self._deviceIds.difference_update(removed) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index dc1ad5111a..873f59b799 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -62,15 +62,6 @@ class CollectorDaemon(RRDDaemon): _cacheServiceName = "Products.ZenCollector.services.ConfigCache" initialServices = RRDDaemon.initialServices + [_cacheServiceName] - @property - def preferences(self): # type: () -> ICollectorPreferences - """The preferences object of this daemon.""" - return self._prefs - - @property - def frameworkFactoryName(self): - return self._frameworkFactoryName - def __init__( self, preferences, @@ -202,13 +193,20 @@ def __init__( # Flag that indicates the daemon has received the encryption key # from zenhub self.encryptionKeyInitialized = False - # Flag that indicates the daemon is loading the cached configs - self.loadingCachedConfigs = False self._deviceloader = None self._deviceloadertask = None self._deviceloadertaskd = None + @property + def preferences(self): # type: () -> ICollectorPreferences + """The preferences object of this daemon.""" + return self._prefs + + @property + def frameworkFactoryName(self): + return self._frameworkFactoryName + def buildOptions(self): super(CollectorDaemon, self).buildOptions() @@ -331,9 +329,17 @@ def _startConfigCycle(self, startDelay=0): # TODO: should we not run maintenance if running in # non-cycle mode? self._scheduler.addTask(configLoader) - self.log.info("scheduled task task=%s", configLoader.name) + self.log.info( + "scheduled task name=%s config-id=%s", + configLoader.name, + configLoader.configId, + ) else: - self.log.info("task already scheduled task=%s", configLoader.name) + self.log.info( + "task already scheduled name=%s config-id=%s", + configLoader.name, + configLoader.configId, + ) def _startMaintenance(self): if not self.options.cycle: @@ -642,11 +648,11 @@ def _deviceConfigCallback(self, new, updated, removed): ) def _deleteDevice(self, deviceId): - self.log.debug("deleted device device-id=%s", deviceId) self._configListener.deleted(deviceId) self._scheduler.removeTasksForConfig(deviceId) self._deviceGuids.pop(deviceId, None) self._devices.discard(deviceId) + self.log.info("removed device config device-id=%s", deviceId) def _updateConfig(self, cfg): """ @@ -670,7 +676,7 @@ def _updateConfig(self, cfg): return False configId = cfg.configId - self.log.info("processing device config config-id=%s", configId) + self.log.debug("processing device config config-id=%s", configId) guid = getattr(cfg, "_device_guid", None) if guid is not None: @@ -712,7 +718,11 @@ def _updateConfig(self, cfg): try: self._scheduler.addTask(task_, self._taskCompleteCallback, now) except ValueError: - self.log.exception("failed to schedule task task=%r", task_) + self.log.exception( + "failed to schedule task name=%s config-id=%s", + task_.name, + task_.configId, + ) continue # TODO: another hack? @@ -725,7 +735,8 @@ def _updateConfig(self, cfg): self.log.exception( "failed to update thresholds " "config-id=%s thresholds=%r", - configId, cfg.thresholds, + configId, + cfg.thresholds, ) # if we're not running a normal daemon cycle then keep track of the @@ -740,6 +751,10 @@ def _updateConfig(self, cfg): self.log.debug("pausing tasks for device %s", configId) self._scheduler.pauseTasksForConfig(configId) + self.log.debug( + "processed new/updated device config config-id=%s", configId + ) + return True def setPropertyItems(self, items): diff --git a/Products/ZenCollector/frameworkfactory.py b/Products/ZenCollector/frameworkfactory.py index d97a079107..fddaad3d95 100644 --- a/Products/ZenCollector/frameworkfactory.py +++ b/Products/ZenCollector/frameworkfactory.py @@ -12,7 +12,7 @@ from .config import ConfigurationLoaderTask, ConfigurationProxy from .interfaces import IFrameworkFactory, ICollectorPreferences -from .scheduler import Scheduler +from .scheduler import TaskScheduler @implementer(IFrameworkFactory) @@ -29,7 +29,7 @@ def getConfigurationProxy(self): def getScheduler(self): if self.__scheduler is None: - self.__scheduler = Scheduler() + self.__scheduler = TaskScheduler.make() return self.__scheduler def getConfigurationLoaderTask(self): diff --git a/Products/ZenCollector/scheduler/__init__.py b/Products/ZenCollector/scheduler/__init__.py new file mode 100644 index 0000000000..2e299b3734 --- /dev/null +++ b/Products/ZenCollector/scheduler/__init__.py @@ -0,0 +1,15 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from .scheduler import Scheduler, TaskScheduler + + +__all__ = ("Scheduler", "TaskScheduler") diff --git a/Products/ZenCollector/scheduler.py b/Products/ZenCollector/scheduler/scheduler.py similarity index 56% rename from Products/ZenCollector/scheduler.py rename to Products/ZenCollector/scheduler/scheduler.py index 69587abb30..3e78c419c7 100644 --- a/Products/ZenCollector/scheduler.py +++ b/Products/ZenCollector/scheduler/scheduler.py @@ -1,308 +1,38 @@ +############################################################################## # -# -# Copyright (C) Zenoss, Inc. 2009-2017 all rights reserved. +# Copyright (C) Zenoss, Inc. 2024 all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. # -# - -from __future__ import print_function +############################################################################## -""" -Support for scheduling tasks and running them on a periodic interval. Tasks -are associated directly with a device, but multiple tasks may exist for a -single device or other monitored object. -""" +from __future__ import absolute_import import logging -import math -import os import random -import sys import time -from StringIO import StringIO - -import zope.interface +from collections import Sequence from twisted.internet import defer, reactor, task from twisted.python.failure import Failure +from zope.interface import implementer -from Products.ZenEvents import Event from Products.ZenUtils.Executor import TwistedExecutor -from Products.ZenUtils.keyedset import KeyedSet -from Products.ZenUtils.Utils import dumpCallbacks - -from .cyberark import get_cyberark -from .interfaces import ( - IScheduler, - IScheduledTask, - IPausingScheduledTask, -) -from .tasks import TaskStates - -log = logging.getLogger("zen.collector.scheduler") - - -class StateStatistics(object): - def __init__(self, state): - self.state = state - self.reset() - - def addCall(self, elapsedTime): - self.totalElapsedTime += elapsedTime - self.totalElapsedTimeSquared += elapsedTime ** 2 - self.totalCalls += 1 - - if self.totalCalls == 1: - self.minElapsedTime = elapsedTime - self.maxElapsedTime = elapsedTime - else: - self.minElapsedTime = min(self.minElapsedTime, elapsedTime) - self.maxElapsedTime = max(self.maxElapsedTime, elapsedTime) - - def reset(self): - self.totalElapsedTime = 0.0 - self.totalElapsedTimeSquared = 0.0 - self.totalCalls = 0 - self.minElapsedTime = 0xFFFFFFFF - self.maxElapsedTime = 0 - - @property - def mean(self): - return float(self.totalElapsedTime) / float(self.totalCalls) - - @property - def stddev(self): - if self.totalCalls == 1: - return 0 - else: - # see http://www.dspguide.com/ch2/2.htm for stddev of running stats - return math.sqrt( - ( - self.totalElapsedTimeSquared - - self.totalElapsedTime ** 2 / self.totalCalls - ) - / (self.totalCalls - 1) - ) - - -class TaskStatistics(object): - def __init__(self, task): - self.task = task - self.totalRuns = 0 - self.failedRuns = 0 - self.missedRuns = 0 - self.states = {} - self.stateStartTime = None - - def trackStateChange(self, oldState, newState): - now = time.time() - - # record how long we spent in the previous state, if there was one - if oldState is not None and self.stateStartTime: - # TODO: how do we properly handle clockdrift or when the clock - # changes, or is time.time() independent of that? - elapsedTime = now - self.stateStartTime - - if oldState in self.states: - stats = self.states[oldState] - else: - stats = StateStatistics(oldState) - self.states[oldState] = stats - stats.addCall(elapsedTime) - - self.stateStartTime = now - -class CallableTask(object): - """ - A CallableTask wraps an object providing IScheduledTask so that it can be - treated as a callable object. This allows the scheduler to make use of the - Twisted framework's LoopingCall construct for simple interval-based - scheduling. - """ +from ..cyberark import get_cyberark +from ..interfaces import IScheduler, IPausingScheduledTask +from ..tasks import TaskStates - def __init__(self, task, scheduler, executor): - if not IScheduledTask.providedBy(task): - raise TypeError("task must provide IScheduledTask") - else: - self.task = task - - self.task._scheduler = scheduler - self._scheduler = scheduler - self._executor = executor - self.paused = False - self.taskStats = None - - def __repr__(self): - return "CallableTask: %s" % getattr(self.task, "name", self.task) - - def running(self): - """ - Called whenever this task is being run. - """ - try: - if hasattr(self.task, "missed"): - self.task._eventService.sendEvent( - { - "eventClass": "/Perf/MissedRuns", - "component": os.path.basename(sys.argv[0]).replace( - ".py", "" - ), - }, - device=self.task._devId, - summary="Task `{}` is being run.".format(self.task.name), - severity=Event.Clear, - eventKey=self.task.name, - ) - del self.task.missed - except Exception: - pass - self.taskStats.totalRuns += 1 - - def logTwistedTraceback(self, reason): - """ - Twisted errBack to record a traceback and log messages - """ - out = StringIO() - reason.printTraceback(out) - # This shouldn't be necessary except for dev code - log.debug(out.getvalue()) - out.close() +from .statistics import StateStatistics, TaskStatistics +from .task import CallableTaskFactory - def finished(self, result): - """ - Called whenever this task has finished. - """ - if isinstance(result, Failure): - self.taskStats.failedRuns += 1 - self.logTwistedTraceback(result) - - def late(self): - """ - Called whenever this task is late and missed its scheduled run time. - """ - try: - # some tasks we don't want to consider a missed run. - if getattr(self.task, "suppress_late", False): - return - - # send event only for missed runs on devices. - self.task._eventService.sendEvent( - { - "eventClass": "/Perf/MissedRuns", - "component": os.path.basename(sys.argv[0]).replace( - ".py", "" - ), - }, - device=self.task._devId, - summary="Missed run: {}".format(self.task.name), - message=self._scheduler._displayStateStatistics( - "", self.taskStats.states - ), - severity=Event.Warning, - eventKey=self.task.name, - ) - self.task.missed = True - except Exception: - pass - self.taskStats.missedRuns += 1 - - def __call__(self): - if self.task.state is TaskStates.STATE_PAUSED and not self.paused: - self.task.state = TaskStates.STATE_IDLE - elif self.paused and self.task.state is not TaskStates.STATE_PAUSED: - self.task.state = TaskStates.STATE_PAUSED - - self._scheduler.setNextExpectedRun(self.task.name, self.task.interval) - - if self.task.state in [TaskStates.STATE_IDLE, TaskStates.STATE_PAUSED]: - if not self.paused: - self.task.state = TaskStates.STATE_QUEUED - # don't return deferred to looping call. - # If a deferred is returned to looping call - # it won't reschedule on error and will only - # reschedule after the deferred is done. This method - # should be called regardless of whether or - # not the task is still running to keep track - # of "late" tasks - d = self._executor.submit(self._doCall) - - def _callError(failure): - msg = "%s - %s failed %s" % ( - self.task, - self.task.name, - failure, - ) - log.debug(msg) - # don't return failure to prevent - # "Unhandled error in Deferred" message - return msg - - # l last error handler in the chain - d.addErrback(_callError) - else: - self._late() - # don't return a Deferred because we want LoopingCall to keep - # rescheduling so that we can keep track of late intervals - - def _doCall(self): - d = defer.maybeDeferred(self._run) - d.addBoth(self._finished) - - # dump the deferred chain if we're in ludicrous debug mode - if log.getEffectiveLevel() < logging.DEBUG: - print("Callback Chain for Task %s" % self.task.name) - dumpCallbacks(d) - return d - - def _run(self): - self.task.state = TaskStates.STATE_RUNNING - self.running() - - return self.task.doTask() - - def _finished(self, result): - log.debug("Task %s finished, result: %r", self.task.name, result) - - # Unless the task completed or paused itself, make sure - # that we always reset the state to IDLE once the task is finished. - if self.task.state != TaskStates.STATE_COMPLETED: - self.task.state = TaskStates.STATE_IDLE - - self._scheduler.taskDone(self.task.name) - - self.finished(result) - - if self.task.state == TaskStates.STATE_COMPLETED: - self._scheduler.removeTasksForConfig(self.task.configId) - - # return result for executor callbacks - return result - - def _late(self): - log.debug("Task %s skipped because it was not idle", self.task.name) - self.late() - - -class CallableTaskFactory(object): - """ - A factory that creates instances of CallableTask, allowing it to be - easily subclassed or replaced in different scheduler implementations. - """ - - def getCallableTask(self, newTask, scheduler): - return CallableTask(newTask, scheduler, scheduler.executor) - - -def getConfigId(task): - return task.configId +log = logging.getLogger("zen.collector.scheduler") -@zope.interface.implementer(IScheduler) -class Scheduler(object): +@implementer(IScheduler) +class TaskScheduler(object): """ A simple interval-based scheduler that makes use of the Twisted framework's LoopingCall construct. @@ -311,21 +41,29 @@ class Scheduler(object): CLEANUP_TASKS_INTERVAL = 10 # seconds ATTEMPTS = 3 - def __init__(self, callableTaskFactory=CallableTaskFactory()): + @classmethod + def make(cls, factory=None, executor=None): + factory = factory if factory is not None else CallableTaskFactory() + executor = executor if executor is not None else TwistedExecutor(1) + return cls(factory, executor) + + def __init__(self, factory, executor): self._loopingCalls = {} self._tasks = {} self._taskCallback = {} self._taskStats = {} self._displaycounts = () - self._callableTaskFactory = callableTaskFactory self._shuttingDown = False + + self._factory = factory + self._executor = executor + # create a cleanup task that will periodically sweep the # cleanup dictionary for tasks that need to be cleaned - self._tasksToCleanup = KeyedSet(getConfigId) + self._tasksToCleanup = {} self._cleanupTask = task.LoopingCall(self._cleanupTasks) - self._cleanupTask.start(Scheduler.CLEANUP_TASKS_INTERVAL) + self._cleanupTask.start(TaskScheduler.CLEANUP_TASKS_INTERVAL) - self._executor = TwistedExecutor(1) self.cyberark = get_cyberark() # Ensure that we can cleanly shutdown all of our tasks @@ -339,6 +77,18 @@ def __init__(self, callableTaskFactory=CallableTaskFactory()): "after", "shutdown", self.shutdown, "after" ) + @property + def executor(self): + return self._executor + + @property + def maxTasks(self): + return self._executor.limit + + @maxTasks.setter + def maxTasks(self, value): + self._executor.limit = value + def __contains__(self, task): """ Returns True if the task has been added to the scheduler. Otherwise @@ -348,6 +98,41 @@ def __contains__(self, task): name = getattr(task, "name", task) return name in self._tasks + def addTask(self, newTask, callback=None, now=False): + """ + Add a new IScheduledTask object for the scheduler to run. + + @param newTask the task to schedule + @type newTask IScheduledTask + @param callback A callable invoked every time the task completes + @type callback callable + @param now Set True to run the task now + @type now boolean + """ + name = newTask.name + if name in self._tasks: + raise ValueError("Task with same name already exists: %s" % name) + callableTask = self._factory.getCallableTask(newTask, self) + loopingCall = task.LoopingCall(callableTask) + self._loopingCalls[name] = loopingCall + self._tasks[name] = callableTask + self._taskCallback[name] = callback + self.taskAdded(callableTask) + startDelay = getattr(newTask, "startDelay", None) + if startDelay is None: + startDelay = 0 if now else self._getStartDelay(newTask) + reactor.callLater(startDelay, self._startTask, newTask, startDelay) + + # just in case someone does not implement scheduled, lets be careful + scheduled = getattr(newTask, "scheduled", lambda x: None) + scheduled(self) + log.info( + "added new task name=%s config-id=%s interval=%s", + newTask.name, + newTask.configId, + newTask.interval, + ) + def shutdown(self, phase): """ The reactor shutdown has three phases for event types: @@ -372,7 +157,7 @@ def shutdown(self, phase): doomedTasks = [] stopQ = {} log.debug("In shutdown stage %s", phase) - for (taskName, taskWrapper) in self._tasks.iteritems(): + for taskName, taskWrapper in self._tasks.iteritems(): task = taskWrapper.task stopPhase = getattr(task, "stopPhase", "before") if ( @@ -385,7 +170,7 @@ def shutdown(self, phase): queue.append((taskName, taskWrapper, task)) for stopOrder in sorted(stopQ): - for (taskName, taskWrapper, task) in stopQ[stopOrder]: + for taskName, taskWrapper, task in stopQ[stopOrder]: loopTask = self._loopingCalls[taskName] if loopTask.running: log.debug("Stopping running task %s", taskName) @@ -395,7 +180,7 @@ def shutdown(self, phase): self.taskRemoved(taskWrapper) for taskName in doomedTasks: - self._tasksToCleanup.add(self._tasks[taskName].task) + self._tasksToCleanup[taskName] = self._tasks[taskName].task del self._loopingCalls[taskName] del self._tasks[taskName] @@ -404,17 +189,47 @@ def shutdown(self, phase): cleanupList = self._cleanupTasks() return defer.DeferredList(cleanupList) - @property - def executor(self): - return self._executor - - def _getMaxTasks(self): - return self._executor.getMax() - - def _setMaxTasks(self, max): - return self._executor.setMax(max) + def _startTask(self, task, delayed, attempts=0): + # If there's no LoopingCall or the LoopingCall is running, + # then there's nothing to do so return + loopingCall = self._loopingCalls.get(task.name) + if loopingCall is None or loopingCall.running: + return - maxTasks = property(_getMaxTasks, _setMaxTasks) + if task.name in self._tasksToCleanup: + delay = random.randint(0, int(task.interval / 2)) + delayed = delayed + delay + if attempts > TaskScheduler.ATTEMPTS: + del self._tasksToCleanup[task.name] + log.info( + "exceeded max start attempts name=%s config-id=%s", + task.name, + task.configId, + ) + attempts = 0 + attempts += 1 + log.info( + "waiting for cleanup name=%s config-id=%s " + "current-delay=%s delayed-so-far=%s attempts=%s", + task.name, + task.configId, + delay, + delayed, + attempts, + ) + reactor.callLater(delay, self._startTask, task, delayed, attempts) + else: + d = loopingCall.start(task.interval) + d.addBoth(self._ltCallback, task.name) + log.info( + "started task name=%s config-id=%s interval=%s " + "delayed=%s attempts=%s", + task.name, + task.configId, + task.interval, + delayed, + attempts, + ) def _ltCallback(self, result, task_name): """last call back in the chain, if it gets called as an errBack @@ -429,96 +244,6 @@ def _ltCallback(self, result, task_name): ) log.error("%s", result) - def _startTask( - self, result, task_name, interval, configId, delayed, attempts=0 - ): - """start the task using a callback so that its put at the bottom of - the Twisted event queue, to allow other processing to continue and - to support a task start-time jitter""" - if task_name in self._loopingCalls: - loopingCall = self._loopingCalls[task_name] - if not loopingCall.running: - if self._tasksToCleanup.has_key(configId): # noqa W601 - delay = random.randint(0, int(interval / 2)) - delayed = delayed + delay - if attempts > Scheduler.ATTEMPTS: - obj = self._tasksToCleanup.pop_by_key(configId) - log.debug( - "Forced cleanup of %s. Task: %s", - configId, - obj.name, - ) - attempts = 0 - attempts += 1 - log.debug( - "Waiting for cleanup of %s. Task %s postponing its " - "start %d seconds (%d so far). Attempt: %s", - configId, - task_name, - delay, - delayed, - attempts, - ) - d = defer.Deferred() - d.addCallback( - self._startTask, - task_name, - interval, - configId, - delayed, - attempts, - ) - reactor.callLater(delay, d.callback, None) - else: - log.debug( - "Task %s starting (waited %d seconds) on %d " - "second intervals", - task_name, - delayed, - interval, - ) - d = loopingCall.start(interval) - d.addBoth(self._ltCallback, task_name) - - def addTask(self, newTask, callback=None, now=False): - """ - Add a new IScheduledTask to the scheduler for execution. - @param newTask the new task to schedule - @type newTask IScheduledTask - @param callback a callback to be notified each time the task completes - @type callback a Python callable - """ - if newTask.name in self._tasks: - raise ValueError("Task %s already exists" % newTask.name) - log.debug( - "add task %s, %s using %s second interval", - newTask.name, - newTask, - newTask.interval, - ) - callableTask = self._callableTaskFactory.getCallableTask(newTask, self) - loopingCall = task.LoopingCall(callableTask) - self._loopingCalls[newTask.name] = loopingCall - self._tasks[newTask.name] = callableTask - self._taskCallback[newTask.name] = callback - self.taskAdded(callableTask) - startDelay = getattr(newTask, "startDelay", None) - if startDelay is None: - startDelay = 0 if now else self._getStartDelay(newTask) - d = defer.Deferred() - d.addCallback( - self._startTask, - newTask.name, - newTask.interval, - newTask.configId, - startDelay, - ) - reactor.callLater(startDelay, d.callback, None) - - # just in case someone does not implement scheduled, lets be careful - scheduled = getattr(newTask, "scheduled", lambda x: None) - scheduled(self) - def _getStartDelay(self, task): """ amount of time to delay the start of a task. Prevents bunching up of @@ -574,7 +299,7 @@ def getTasksForConfig(self, configId): Get all tasks associated with the specified identifier. """ tasks = [] - for (taskName, taskWrapper) in self._tasks.iteritems(): + for taskName, taskWrapper in self._tasks.iteritems(): task = taskWrapper.task if task.configId == configId: tasks.append(task) @@ -602,34 +327,42 @@ def setNextExpectedRun(self, taskName, taskInterval): ) def removeTasks(self, taskNames): + # type: (Self, Sequence[str]) -> None """ Remove tasks """ + if not isinstance(taskNames, Sequence): + raise ValueError("argument is not a sequence") + doomedTasks = [] # child ids are any task that are children of the current task being # removed childIds = [] - for taskName in taskNames: - taskWrapper = self._tasks[taskName] + for name in taskNames: + taskWrapper = self._tasks[name] task = taskWrapper.task subIds = getattr(task, "childIds", None) if subIds: childIds.extend(subIds) - log.debug("Stopping task %s, %s", taskName, task) - if self._loopingCalls[taskName].running: - self._loopingCalls[taskName].stop() + if self._loopingCalls[name].running: + self._loopingCalls[name].stop() + log.debug( + "stopped task name=%s config-id=%s", name, task.configId + ) - doomedTasks.append(taskName) + doomedTasks.append(name) self.taskRemoved(taskWrapper) for taskName in doomedTasks: task = self._tasks[taskName].task - self._tasksToCleanup.add(task) + self._tasksToCleanup[taskName] = task del self._loopingCalls[taskName] del self._tasks[taskName] self._displayTaskStatistics(task) del self._taskStats[taskName] - # TODO: ponder task statistics and keeping them around? + log.info( + "removed task name=%s config-id=%s", task.name, task.configId + ) map(self.removeTasksForConfig, childIds) @@ -641,13 +374,15 @@ def removeTasksForConfig(self, configId): @type configId: string """ self.removeTasks( - taskName - for taskName, taskWrapper in self._tasks.iteritems() - if taskWrapper.task.configId == configId + tuple( + name + for name, wrapper in self._tasks.iteritems() + if wrapper.task.configId == configId + ) ) def pauseTasksForConfig(self, configId): - for (taskName, taskWrapper) in self._tasks.items(): + for taskName, taskWrapper in self._tasks.items(): task = taskWrapper.task if task.configId == configId: log.debug("Pausing task %s", taskName) @@ -655,7 +390,7 @@ def pauseTasksForConfig(self, configId): self.taskPaused(taskWrapper) def resumeTasksForConfig(self, configId): - for (taskName, taskWrapper) in self._tasks.iteritems(): + for taskName, taskWrapper in self._tasks.iteritems(): task = taskWrapper.task if task.configId == configId: log.debug("Resuming task %s", taskName) @@ -882,21 +617,21 @@ def _cleanupTasks(self): todoList = [ task - for task in self._tasksToCleanup + for task in self._tasksToCleanup.values() if self._isTaskCleanable(task) ] cleanupWaitList = [] - for task in todoList: + for item in todoList: # changing the state of the task will keep the next cleanup run # from processing it again - task.state = TaskStates.STATE_CLEANING + item.state = TaskStates.STATE_CLEANING if self._shuttingDown: # let the task know the scheduler is shutting down - task.state = TaskStates.STATE_SHUTDOWN - log.debug("Cleanup on task %s %s", task.name, task) - d = defer.maybeDeferred(task.cleanup) - d.addBoth(self._cleanupTaskComplete, task) + item.state = TaskStates.STATE_SHUTDOWN + log.debug("Cleanup on task %s %s", item.name, item) + d = defer.maybeDeferred(item.cleanup) + d.addBoth(self._cleanupTaskComplete, item) cleanupWaitList.append(d) return cleanupWaitList @@ -910,7 +645,7 @@ def _cleanupTaskComplete(self, result, task): result, task.name, ) - self._tasksToCleanup.discard(task) + del self._tasksToCleanup[task.name] return result def _isTaskCleanable(self, task): @@ -929,3 +664,12 @@ def resetStats(self, taskName): taskStats.totalRuns = 0 taskStats.failedRuns = 0 taskStats.missedRuns = 0 + + +class Scheduler(TaskScheduler): + """Backward compatibility layer.""" + + def __init__(self): + super(Scheduler, self).__init__( + CallableTaskFactory(), TwistedExecutor(1) + ) diff --git a/Products/ZenCollector/scheduler/statistics.py b/Products/ZenCollector/scheduler/statistics.py new file mode 100644 index 0000000000..90c653bbe9 --- /dev/null +++ b/Products/ZenCollector/scheduler/statistics.py @@ -0,0 +1,79 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +import math +import time + + +class StateStatistics(object): + def __init__(self, state): + self.state = state + self.reset() + + def addCall(self, elapsedTime): + self.totalElapsedTime += elapsedTime + self.totalElapsedTimeSquared += elapsedTime**2 + self.totalCalls += 1 + + if self.totalCalls == 1: + self.minElapsedTime = elapsedTime + self.maxElapsedTime = elapsedTime + else: + self.minElapsedTime = min(self.minElapsedTime, elapsedTime) + self.maxElapsedTime = max(self.maxElapsedTime, elapsedTime) + + def reset(self): + self.totalElapsedTime = 0.0 + self.totalElapsedTimeSquared = 0.0 + self.totalCalls = 0 + self.minElapsedTime = 0xFFFFFFFF + self.maxElapsedTime = 0 + + @property + def mean(self): + return float(self.totalElapsedTime) / float(self.totalCalls) + + @property + def stddev(self): + if self.totalCalls == 1: + return 0 + else: + # see http://www.dspguide.com/ch2/2.htm for stddev of running stats + mean = self.totalElapsedTime**2 / self.totalCalls + return math.sqrt( + (self.totalElapsedTimeSquared - mean) / (self.totalCalls - 1) + ) + + +class TaskStatistics(object): + def __init__(self, task): + self.task = task + self.totalRuns = 0 + self.failedRuns = 0 + self.missedRuns = 0 + self.states = {} + self.stateStartTime = None + + def trackStateChange(self, oldState, newState): + now = time.time() + + # record how long we spent in the previous state, if there was one + if oldState is not None and self.stateStartTime: + # TODO: how do we properly handle clockdrift or when the clock + # changes, or is time.time() independent of that? + elapsedTime = now - self.stateStartTime + + if oldState in self.states: + stats = self.states[oldState] + else: + stats = StateStatistics(oldState) + self.states[oldState] = stats + stats.addCall(elapsedTime) + + self.stateStartTime = now diff --git a/Products/ZenCollector/scheduler/task.py b/Products/ZenCollector/scheduler/task.py new file mode 100644 index 0000000000..ee80337f3e --- /dev/null +++ b/Products/ZenCollector/scheduler/task.py @@ -0,0 +1,206 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +import logging +import os +import sys + +from StringIO import StringIO + +from twisted.internet import defer +from twisted.python.failure import Failure + +from Products.ZenEvents import Event +from Products.ZenUtils.Utils import dumpCallbacks + +from ..interfaces import IScheduledTask +from ..tasks import TaskStates + +log = logging.getLogger("zen.collector.scheduler") + + +class CallableTask(object): + """ + A CallableTask wraps an IScheduledTask object to make it a callable. + This allows the scheduler to make use of the Twisted framework's + LoopingCall construct for simple interval-based scheduling. + """ + + def __init__(self, task, scheduler, executor): + if not IScheduledTask.providedBy(task): + raise TypeError("task must provide IScheduledTask") + + self.task = task + self.task._scheduler = scheduler + self._scheduler = scheduler + self._executor = executor + self.paused = False + self.taskStats = None + + def __repr__(self): + return "CallableTask: %s" % getattr(self.task, "name", self.task) + + def running(self): + """ + Called whenever this task is being run. + """ + try: + if hasattr(self.task, "missed"): + self.task._eventService.sendEvent( + { + "eventClass": "/Perf/MissedRuns", + "component": os.path.basename(sys.argv[0]).replace( + ".py", "" + ), + }, + device=self.task._devId, + summary="Task `{}` is being run.".format(self.task.name), + severity=Event.Clear, + eventKey=self.task.name, + ) + del self.task.missed + except Exception: + pass + self.taskStats.totalRuns += 1 + + def logTwistedTraceback(self, reason): + """ + Twisted errBack to record a traceback and log messages + """ + out = StringIO() + reason.printTraceback(out) + # This shouldn't be necessary except for dev code + log.debug(out.getvalue()) + out.close() + + def finished(self, result): + """ + Called whenever this task has finished. + """ + if isinstance(result, Failure): + self.taskStats.failedRuns += 1 + self.logTwistedTraceback(result) + + def late(self): + """ + Called whenever this task is late and missed its scheduled run time. + """ + try: + # some tasks we don't want to consider a missed run. + if getattr(self.task, "suppress_late", False): + return + + # send event only for missed runs on devices. + self.task._eventService.sendEvent( + { + "eventClass": "/Perf/MissedRuns", + "component": os.path.basename(sys.argv[0]).replace( + ".py", "" + ), + }, + device=self.task._devId, + summary="Missed run: {}".format(self.task.name), + message=self._scheduler._displayStateStatistics( + "", self.taskStats.states + ), + severity=Event.Warning, + eventKey=self.task.name, + ) + self.task.missed = True + except Exception: + pass + self.taskStats.missedRuns += 1 + + def __call__(self): + if self.task.state is TaskStates.STATE_PAUSED and not self.paused: + self.task.state = TaskStates.STATE_IDLE + elif self.paused and self.task.state is not TaskStates.STATE_PAUSED: + self.task.state = TaskStates.STATE_PAUSED + + self._scheduler.setNextExpectedRun(self.task.name, self.task.interval) + + if self.task.state in [TaskStates.STATE_IDLE, TaskStates.STATE_PAUSED]: + if not self.paused: + self.task.state = TaskStates.STATE_QUEUED + # don't return deferred to looping call. + # If a deferred is returned to looping call + # it won't reschedule on error and will only + # reschedule after the deferred is done. This method + # should be called regardless of whether or + # not the task is still running to keep track + # of "late" tasks + d = self._executor.submit(self._doCall) + + def _callError(failure): + msg = "%s - %s failed %s" % ( + self.task, + self.task.name, + failure, + ) + log.debug(msg) + # don't return failure to prevent + # "Unhandled error in Deferred" message + return msg + + # l last error handler in the chain + d.addErrback(_callError) + else: + self._late() + # don't return a Deferred because we want LoopingCall to keep + # rescheduling so that we can keep track of late intervals + + def _doCall(self): + d = defer.maybeDeferred(self._run) + d.addBoth(self._finished) + + # dump the deferred chain if we're in ludicrous debug mode + if log.getEffectiveLevel() < logging.DEBUG: + print("Callback Chain for Task %s" % self.task.name) + dumpCallbacks(d) + return d + + def _run(self): + self.task.state = TaskStates.STATE_RUNNING + self.running() + + return self.task.doTask() + + def _finished(self, result): + log.debug("Task %s finished, result: %r", self.task.name, result) + + # Unless the task completed or paused itself, make sure + # that we always reset the state to IDLE once the task is finished. + if self.task.state != TaskStates.STATE_COMPLETED: + self.task.state = TaskStates.STATE_IDLE + + self._scheduler.taskDone(self.task.name) + + self.finished(result) + + if self.task.state == TaskStates.STATE_COMPLETED: + self._scheduler.removeTasksForConfig(self.task.configId) + + # return result for executor callbacks + return result + + def _late(self): + log.debug("Task %s skipped because it was not idle", self.task.name) + self.late() + + +class CallableTaskFactory(object): + """ + A factory that creates instances of CallableTask, allowing it to be + easily subclassed or replaced in different scheduler implementations. + """ + + def getCallableTask(self, newTask, scheduler): + return CallableTask(newTask, scheduler, scheduler.executor) diff --git a/Products/ZenCollector/tests/testFactory.py b/Products/ZenCollector/tests/testFactory.py index 77c12737ba..d37d7fcfa6 100644 --- a/Products/ZenCollector/tests/testFactory.py +++ b/Products/ZenCollector/tests/testFactory.py @@ -11,7 +11,7 @@ from ..frameworkfactory import CoreCollectorFrameworkFactory from ..config import ConfigurationProxy -from ..scheduler import Scheduler +from ..scheduler import TaskScheduler class TestFactory(BaseTestCase): @@ -25,7 +25,7 @@ def testFactoryInstall(self): self.assertTrue(isinstance(configProxy, ConfigurationProxy)) scheduler = factory.getScheduler() - self.assertTrue(isinstance(scheduler, Scheduler)) + self.assertTrue(isinstance(scheduler, TaskScheduler)) def test_suite(): diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index 2b7ceb745f..35da9af2b6 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -480,6 +480,8 @@ def loadThresholdClasses(self, classnames): self.log.info("imported threshold class class=%s", name) except ImportError: self.log.exception("unable to import threshold class %s", name) + except AttributeError: + self.log.exception("unable to import threshold class %s", name) def buildOptions(self): super(PBDaemon, self).buildOptions() From dc5728f32ea9e8d379411b22c8f5f284ba177215 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 16 Feb 2024 07:50:02 -0600 Subject: [PATCH 070/147] Remove unnecessary execute bit from files in ZenUtils --- Products/ZenUtils/Step.py | 0 Products/ZenUtils/__init__.py | 0 Products/ZenUtils/cstat.py | 0 Products/ZenUtils/requestlogging/ZopeRequestLogger.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 Products/ZenUtils/Step.py mode change 100755 => 100644 Products/ZenUtils/__init__.py mode change 100755 => 100644 Products/ZenUtils/cstat.py mode change 100755 => 100644 Products/ZenUtils/requestlogging/ZopeRequestLogger.py diff --git a/Products/ZenUtils/Step.py b/Products/ZenUtils/Step.py old mode 100755 new mode 100644 diff --git a/Products/ZenUtils/__init__.py b/Products/ZenUtils/__init__.py old mode 100755 new mode 100644 diff --git a/Products/ZenUtils/cstat.py b/Products/ZenUtils/cstat.py old mode 100755 new mode 100644 diff --git a/Products/ZenUtils/requestlogging/ZopeRequestLogger.py b/Products/ZenUtils/requestlogging/ZopeRequestLogger.py old mode 100755 new mode 100644 From 4a58989a4c3a330f6ba2e78b353feb59cad61390 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 16 Feb 2024 10:13:29 -0600 Subject: [PATCH 071/147] fix: check relationships before using them. Sometimes a (ZenRelations) relationship is empty so check whether it exists before trying to use it. ZEN-34714 --- .../configcache/modelchange/oids.py | 53 ++++++++++++++++--- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/Products/ZenCollector/configcache/modelchange/oids.py b/Products/ZenCollector/configcache/modelchange/oids.py index 959950ab7a..9a6808211c 100644 --- a/Products/ZenCollector/configcache/modelchange/oids.py +++ b/Products/ZenCollector/configcache/modelchange/oids.py @@ -87,9 +87,15 @@ class DataPointToDevice(BaseTransform): """Return the device OIDs associated with an RRDDataPoint.""" def transformOid(self, oid): - ds = self._entity.datasource().primaryAq() - template = ds.rrdTemplate().primaryAq() - dc = template.deviceClass().primaryAq() + ds = _getDataSource(self._entity) + if not ds: + return () + template = _getTemplate(ds.primaryAq()) + if not template: + return () + dc = _getDeviceClass(template) + if not dc: + return () log.debug( "[DataPointToDevice] return OIDs of devices associated " "with DataPoint entity=%s ", @@ -103,11 +109,15 @@ class DataSourceToDevice(BaseTransform): """Return the device OIDs associated with an RRDDataSource.""" def transformOid(self, oid): - template = self._entity.rrdTemplate().primaryAq() - dc = template.deviceClass().primaryAq() + template = _getTemplate(self._entity) + if not template: + return () + dc = _getDeviceClass(template) + if not dc: + return () log.debug( "[DataSourceToDevice] return OIDs of devices associated " - "with DataSource entity=%s ", + "with DataSource entity=%s", self._entity, ) return _getDevicesFromDeviceClass(dc) @@ -118,7 +128,9 @@ class TemplateToDevice(BaseTransform): """Return the device OIDs associated with an RRDTemplate.""" def transformOid(self, oid): - dc = self._entity.deviceClass().primaryAq() + dc = _getDeviceClass(self._entity) + if not dc: + return () log.debug( "[TemplateToDevice] return OIDs of devices associated " "with RRDTemplate entity=%s ", @@ -140,6 +152,33 @@ def transformOid(self, oid): return _getDevicesFromDeviceClass(self._entity) +def _getDataSource(dp): + ds = dp.datasource() + if ds is None: + if log.isEnabledFor(logging.DEBUG): + log.warn("no datasource relationship datapoint=%s", dp) + return None + return ds.primaryAq() + + +def _getTemplate(ds): + template = ds.rrdTemplate() + if template is None: + if log.isEnabledFor(logging.DEBUG): + log.warn("no template relationship datasource=%s", ds) + return None + return template.primaryAq() + + +def _getDeviceClass(template): + dc = template.deviceClass() + if dc is None: + if log.isEnabledFor(logging.DEBUG): + log.warn("no device class relationship template=%s", template) + return None + return dc.primaryAq() + + def _getDevicesFromDeviceClass(dc): tool = IModelCatalogTool(dc.dmd.Devices) query, _ = tool._build_query( From f7878dea0dd076a6419f89701cfc7bf905a42c29 Mon Sep 17 00:00:00 2001 From: Oleksandr Dubrovyk Date: Tue, 20 Feb 2024 10:54:08 +0000 Subject: [PATCH 072/147] Fix ImportError for zenvsphere daemon Fixes ZEN-34719. Added CallableTaskFactory and CallableTask classes to scheduler __init__.py --- Products/ZenCollector/scheduler/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Products/ZenCollector/scheduler/__init__.py b/Products/ZenCollector/scheduler/__init__.py index 2e299b3734..3dd71c6b9b 100644 --- a/Products/ZenCollector/scheduler/__init__.py +++ b/Products/ZenCollector/scheduler/__init__.py @@ -10,6 +10,11 @@ from __future__ import absolute_import from .scheduler import Scheduler, TaskScheduler +from .task import CallableTaskFactory, CallableTask - -__all__ = ("Scheduler", "TaskScheduler") +__all__ = ( + "Scheduler", + "TaskScheduler", + "CallableTaskFactory", + "CallableTask" +) From 4e194f0cb3a14788a05c6de391c1105c642ce39c Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 20 Feb 2024 08:56:56 -0600 Subject: [PATCH 073/147] fix: scheduler removeTasks requires a sequence. The TaskScheduler removeTasks method requires its argument to be a sequence, like a list or tuple. ZEN-34717 --- Products/ZenCollector/daemon.py | 4 +++- Products/ZenCollector/scheduler/scheduler.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 873f59b799..2fb61f0333 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -691,7 +691,9 @@ def _updateConfig(self, cfg): ) for taskToRemove in tasksToRemove } - self._scheduler.removeTasks(task.name for task in tasksToRemove) + self._scheduler.removeTasks( + tuple(task.name for task in tasksToRemove) + ) self._configListener.updated(cfg) else: self._devices.add(configId) diff --git a/Products/ZenCollector/scheduler/scheduler.py b/Products/ZenCollector/scheduler/scheduler.py index 3e78c419c7..ae06f94045 100644 --- a/Products/ZenCollector/scheduler/scheduler.py +++ b/Products/ZenCollector/scheduler/scheduler.py @@ -332,7 +332,7 @@ def removeTasks(self, taskNames): Remove tasks """ if not isinstance(taskNames, Sequence): - raise ValueError("argument is not a sequence") + taskNames = tuple(taskNames) doomedTasks = [] # child ids are any task that are children of the current task being From b78e2fa668524d46066d27eebef5638fd34cb0e7 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 20 Feb 2024 09:53:55 -0600 Subject: [PATCH 074/147] fix: avoid race condition involving device config loader The PBDaemon class started a task that was accessing an attribute of the DeviceConfigLoader before the DeviceConfigLoader instance was created. This race condition was fixed by moving the creation of the DeviceConfigLoader instance to an earlier point in CollectorDaemon's startup. ZEN-34705 --- Products/ZenCollector/daemon.py | 29 +++++++++++++++-------------- Products/ZenHub/PBDaemon.py | 4 ++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 2fb61f0333..eb5803519c 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -194,7 +194,13 @@ def __init__( # from zenhub self.encryptionKeyInitialized = False - self._deviceloader = None + # Define _deviceloader to avoid race condition + # with task stats recording. + self._deviceloader = DeviceConfigLoader( + self.options, + self._configProxy, + self._deviceConfigCallback, + ) self._deviceloadertask = None self._deviceloadertaskd = None @@ -361,15 +367,6 @@ def _startMaintenance(self): self._maintenanceCycle.start() def _startDeviceConfigLoader(self): - self.log.info( - "running the device config loader every %d seconds", - self._device_config_update_interval, - ) - self._deviceloader = DeviceConfigLoader( - self.options, - self._configProxy, - self._deviceConfigCallback, - ) self._deviceloadertask = task.LoopingCall(self._deviceloader) self._deviceloadertaskd = self._deviceloadertask.start( self._device_config_update_interval @@ -377,6 +374,10 @@ def _startDeviceConfigLoader(self): reactor.addSystemEventTrigger( "before", "shutdown", self._deviceloadertask.stop, "before" ) + self.log.info( + "started receiving device config changes interval=%d", + self._device_config_update_interval, + ) def _startTaskStatsLogging(self): if not (self.options.cycle and self.options.logTaskStats): @@ -387,13 +388,13 @@ def _startTaskStatsLogging(self): self._taskstatsloggerd = self._taskstatslogger.start( self.options.logTaskStats, now=False ) - self.log.debug( - "started logging task statistics interval=%d", - self.options.logTaskStats, - ) reactor.addSystemEventTrigger( "before", "shutdown", self._taskstatslogger.stop, "before" ) + self.log.info( + "started logging task statistics interval=%d", + self.options.logTaskStats, + ) @defer.inlineCallbacks def getRemoteConfigCacheProxy(self): diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index 35da9af2b6..acbfcfeb53 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -414,7 +414,7 @@ def _setup_event_client(self): self.__eventclient.start() self.__recordQueuedEventsCountLoop.start(2.0, now=False) self.__eventclient.sendEvent(self.startEvent) - self.log.debug("started event client") + self.log.info("started event client") def _setup_stats_recording(self): loop = task.LoopingCall(self.postStatistics) @@ -433,7 +433,7 @@ def _setup_stats_recording(self): reactor.addSystemEventTrigger( "before", "shutdown", self._metrologyReporter.stop ) - self.log.debug("started statistics recording task") + self.log.info("started statistics recording task") def postStatisticsImpl(self): pass From 4f37e3dfffed2ddc6776f1f2873fd885793ed7c6 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 21 Feb 2024 07:42:40 -0600 Subject: [PATCH 075/147] fix: create config proxy before it's used ZEN-34717 --- Products/ZenCollector/daemon.py | 12 +++++++----- Products/ZenCollector/scheduler/scheduler.py | 17 ++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index eb5803519c..5a01ef512e 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -164,9 +164,9 @@ def __init__( self._completedTasks = 0 self._pendingTasks = [] - self._configProxy = None - self._ConfigurationLoaderTask = None framework = _getFramework(self.frameworkFactoryName) + self._configProxy = framework.getConfigurationProxy() + self._scheduler = framework.getScheduler() self._scheduler.maxTasks = self.options.maxTasks @@ -296,8 +296,7 @@ def connected(self): try: yield defer.maybeDeferred(self._getInitializationCallback()) framework = _getFramework(self.frameworkFactoryName) - self.log.debug("using framework factory %s", type(framework)) - self._configProxy = framework.getConfigurationProxy() + self.log.debug("using framework factory %r", framework) yield self._initEncryptionKey() yield self._startConfigCycle() yield self._startMaintenance() @@ -336,9 +335,12 @@ def _startConfigCycle(self, startDelay=0): # non-cycle mode? self._scheduler.addTask(configLoader) self.log.info( - "scheduled task name=%s config-id=%s", + "scheduled task " + "name=%s config-id=%s interval=%s start-delay=%s", configLoader.name, configLoader.configId, + getattr(configLoader, "interval", "n/a"), + configLoader.startDelay, ) else: self.log.info( diff --git a/Products/ZenCollector/scheduler/scheduler.py b/Products/ZenCollector/scheduler/scheduler.py index ae06f94045..89bfe8961a 100644 --- a/Products/ZenCollector/scheduler/scheduler.py +++ b/Products/ZenCollector/scheduler/scheduler.py @@ -126,11 +126,12 @@ def addTask(self, newTask, callback=None, now=False): # just in case someone does not implement scheduled, lets be careful scheduled = getattr(newTask, "scheduled", lambda x: None) scheduled(self) - log.info( - "added new task name=%s config-id=%s interval=%s", + log.debug( + "added new task name=%s config-id=%s interval=%s start-delay=%s", newTask.name, newTask.configId, newTask.interval, + startDelay, ) def shutdown(self, phase): @@ -201,14 +202,14 @@ def _startTask(self, task, delayed, attempts=0): delayed = delayed + delay if attempts > TaskScheduler.ATTEMPTS: del self._tasksToCleanup[task.name] - log.info( + log.warn( "exceeded max start attempts name=%s config-id=%s", task.name, task.configId, ) attempts = 0 attempts += 1 - log.info( + log.debug( "waiting for cleanup name=%s config-id=%s " "current-delay=%s delayed-so-far=%s attempts=%s", task.name, @@ -221,7 +222,7 @@ def _startTask(self, task, delayed, attempts=0): else: d = loopingCall.start(task.interval) d.addBoth(self._ltCallback, task.name) - log.info( + log.debug( "started task name=%s config-id=%s interval=%s " "delayed=%s attempts=%s", task.name, @@ -236,8 +237,7 @@ def _ltCallback(self, result, task_name): the looping will stop - shouldn't be called since CallableTask doesn't return a deferred, here for sanity and debug""" if task_name in self._loopingCalls: - loopingCall = self._loopingCalls[task_name] - log.debug("call finished %s : %s", loopingCall, result) + log.debug("task finished name=%s result=%s", task_name, result) if isinstance(result, Failure): log.warn( "Failure in looping call, will not reschedule %s", task_name @@ -349,7 +349,6 @@ def removeTasks(self, taskNames): log.debug( "stopped task name=%s config-id=%s", name, task.configId ) - doomedTasks.append(name) self.taskRemoved(taskWrapper) @@ -360,7 +359,7 @@ def removeTasks(self, taskNames): del self._tasks[taskName] self._displayTaskStatistics(task) del self._taskStats[taskName] - log.info( + log.debug( "removed task name=%s config-id=%s", task.name, task.configId ) From 1e33555e32f3486141da7af332cff86843a18edd Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 21 Feb 2024 06:58:22 -0600 Subject: [PATCH 076/147] fix: add invalidation transform to handle threshold changes Added an IInvalidationOid implementation to configcache to handle threshold changes. ZEN-34715 --- .../ZenCollector/configcache/modelchange.zcml | 6 ++++++ .../configcache/modelchange/oids.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/Products/ZenCollector/configcache/modelchange.zcml b/Products/ZenCollector/configcache/modelchange.zcml index 16afa81f0c..562346f2ee 100644 --- a/Products/ZenCollector/configcache/modelchange.zcml +++ b/Products/ZenCollector/configcache/modelchange.zcml @@ -82,4 +82,10 @@ License.zenoss under the directory where your Zenoss product is installed. for="Products.ZenModel.DeviceClass.DeviceClass" /> + + diff --git a/Products/ZenCollector/configcache/modelchange/oids.py b/Products/ZenCollector/configcache/modelchange/oids.py index 9a6808211c..4da659b80b 100644 --- a/Products/ZenCollector/configcache/modelchange/oids.py +++ b/Products/ZenCollector/configcache/modelchange/oids.py @@ -16,6 +16,7 @@ from zope.interface import implementer from Products.ZenHub.interfaces import IInvalidationOid +from Products.ZenModel.DeviceClass import DeviceClass from Products.ZenRelations.RelationshipBase import IRelationship from Products.Zuul.catalog.interfaces import IModelCatalogTool @@ -152,6 +153,22 @@ def transformOid(self, oid): return _getDevicesFromDeviceClass(self._entity) +@implementer(IInvalidationOid) +class ThresholdToDevice(BaseTransform): + """Return the device OIDs in the DeviceClass hierarchy.""" + + def transformOid(self, oid): + log.debug( + "[ThresholdToDevice] return OIDs of devices associated " + "with Threshold entity=%s ", + self._entity, + ) + obj = self._entity + while not isinstance(obj, DeviceClass): + obj = obj.getParentNode() + return _getDevicesFromDeviceClass(obj) + + def _getDataSource(dp): ds = dp.datasource() if ds is None: From ff6ce05e509896cd5a9dd53396d014c3013ff5bf Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 21 Feb 2024 11:05:35 -0600 Subject: [PATCH 077/147] The show command displays more device config content. The show command now displays nearly all data in a device configuration and its output adjusts to the width of the terminal. Also, wildcard matching is now used so the commands can be abbreviated to reduce the amount of required typing. The wildcard feature has been extended to list and expire commands. ZEN-34721 --- Products/ZenCollector/configcache/cli.py | 134 ++++++++++++++++++++--- 1 file changed, 118 insertions(+), 16 deletions(-) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 5a41d3958e..a73035e77c 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -10,19 +10,25 @@ from __future__ import absolute_import, print_function import argparse -import pprint +import os import sys from datetime import datetime +import six + +from IPython.lib import pretty +from twisted.spread.jelly import unjellyableRegistry from zope.component import createObject import Products.ZenCollector.configcache as CONFIGCACHE_MODULE +from Products.ZenCollector.services.config import DeviceProxy from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl +from Products.ZenUtils.terminal_size import get_terminal_size from .app import initialize_environment -from .cache import ConfigKey, ConfigQuery, ConfigStatus +from .cache import ConfigQuery, ConfigStatus from .misc.args import get_subparser @@ -59,9 +65,10 @@ def add_arguments(parser, subparsers): subp.set_defaults(factory=List_) def __init__(self, args): - self._monitor = args.monitor - self._service = args.service + self._monitor = "*{}*".format(args.monitor).replace("***", "*") + self._service = "*{}*".format(args.service).replace("***", "*") self._showuid = args.show_uid + self._devices = getattr(args, "device", []) state_names = getattr(args, "states", ()) if state_names: states = set() @@ -72,10 +79,24 @@ def __init__(self, args): self._states = () def run(self): + haswildcard = any("*" in d for d in self._devices) + if haswildcard and len(self._devices) > 1: + print( + "Only one DEVICE argument supported when a wildcard is used.", + file=sys.stderr, + ) + return initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) - query = ConfigQuery(service=self._service, monitor=self._monitor) + if haswildcard: + query = ConfigQuery( + service=self._service, + monitor=self._monitor, + device=self._devices[0], + ) + else: + query = ConfigQuery(service=self._service, monitor=self._monitor) results = store.get_status(*store.search(query)) if self._states: results = ( @@ -85,8 +106,12 @@ def run(self): ) rows = [] maxd, maxs, maxm = 0, 0, 0 + if len(self._devices) > 0: + data = (key for key in results if key[0].device in self._devices) + else: + data = results for key, status in sorted( - results, key=lambda x: (x[0].device, x[0].service) + data, key=lambda x: (x[0].device, x[0].service) ): if self._showuid: uid = store.get_uid(key.device) @@ -162,7 +187,15 @@ class Show(object): @staticmethod def add_arguments(parser, subparsers): - subp = get_subparser(subparsers, "show", "Show a configuration") + subp = get_subparser(subparsers, "show", Show.description) + termsize = get_terminal_size() + subp.add_argument( + "--width", + type=int, + default=termsize.columns, + help="Maxiumum number of columns to use in the output. " + "By default, this is the width of the terminal", + ) subp.add_argument( "service", nargs=1, help="name of the configuration service" ) @@ -176,19 +209,82 @@ def __init__(self, args): self._monitor = args.monitor[0] self._service = args.service[0] self._device = args.device[0] + if _is_output_redirected(): + # when stdout is redirected, default to 79 columns unless + # the --width option has a non-default value. + termsize = get_terminal_size() + if args.width != termsize.columns: + self._columns = args.width + else: + self._columns = 79 + else: + self._columns = args.width + + def run(self): initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) - key = ConfigKey( - service=self._service, monitor=self._monitor, device=self._device + results, err = _query_cache( + store, + service="*{}*".format(self._service), + monitor="*{}*".format(self._monitor), + device="*{}*".format(self._device), ) - results = store.get(key) if results: - pprint.pprint(results.config.__dict__) + for cls in set(unjellyableRegistry.values()): + if cls is DeviceProxy: + pretty.for_type(cls, _pp_DeviceProxy) + else: + pretty.for_type(cls, _pp_default) + pretty.pprint(results.config, max_width=self._columns) else: - print("configuration not found", file=sys.stderr) + print(err, file=sys.stderr) + + +def _query_cache(store, service, monitor, device): + query = ConfigQuery(service=service, monitor=monitor, device=device) + results = store.search(query) + first_key = next(results, None) + if first_key is None: + return (None, "configuration not found") + second_key = next(results, None) + if second_key is not None: + return (None, "more than one configuration matched arguments") + return (store.get(first_key), None) + + +def _pp_DeviceProxy(obj, p, cycle): + _printer( + obj, + p, + cycle, + lambda k, v: v if "password" not in k.lower() else "******", + ) + + +def _pp_default(obj, p, cycle): + _printer(obj, p, cycle, lambda k, v: v) + + +def _printer(obj, p, cycle, vprint): + clsname = obj.__class__.__name__ + if cycle: + p.text("<{}: ...>".format(clsname)) + else: + with p.group(2, "<{}: ".format(clsname), ">"): + attrs = ( + (k, v) + for k, v in sorted(obj.__dict__.items(), key=lambda x: x[0]) + if v not in (None, "", {}, []) + ) + for idx, (k, v) in enumerate(attrs): + if idx: + p.text(",") + p.breakable() + p.text("{}=".format(k)) + p.pretty(vprint(k, v)) class Expire(object): @@ -208,8 +304,8 @@ def add_arguments(parser, subparsers): subp.set_defaults(factory=Expire) def __init__(self, args): - self._monitor = args.monitor - self._service = args.service + self._monitor = "*{}*".format(args.monitor).replace("***", "*") + self._service = "*{}*".format(args.service).replace("***", "*") self._devices = getattr(args, "device", []) def run(self): @@ -298,7 +394,9 @@ def _confirm_inputs(self): def _confirm(mesg): response = None while response not in ["y", "n", ""]: - response = raw_input("%s. Are you sure (y/N)? " % (mesg,)).lower() + response = six.moves.input( + "%s. Are you sure (y/N)? " % (mesg,) + ).lower() return response == "y" @@ -323,7 +421,7 @@ def _split_listed_choices(self, value): return value def __call__(self, parser, namespace, values=None, option_string=None): - if isinstance(values, basestring): + if isinstance(values, six.string_types): values = (values,) setattr(namespace, self.dest, values) @@ -342,6 +440,10 @@ def __iter__(self): return iter(self._choices) +def _is_output_redirected(): + return os.fstat(0) != os.fstat(1) + + _common_parser = None From 5d58194ae5ea3daf11bda8ba0895ed6e39146429 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 22 Feb 2024 07:19:48 -0600 Subject: [PATCH 078/147] fix: only start the 'local server' when the 'cycle' option is set. ZEN-34722 --- Products/ZenHub/PBDaemon.py | 33 ++++++++++++++++---------- Products/ZenHub/tests/test_PBDaemon.py | 2 ++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Products/ZenHub/PBDaemon.py b/Products/ZenHub/PBDaemon.py index acbfcfeb53..fa43bcbf90 100644 --- a/Products/ZenHub/PBDaemon.py +++ b/Products/ZenHub/PBDaemon.py @@ -139,19 +139,25 @@ def __init__( self.__record_queued_events_count ) - self.__server = _getLocalServer(self.options) - self.__server.add_resource( - "zenhub", - ZenHubStatus( - lambda: "connected" - if self.__zenhub_connected - else "disconnected" - ), - ) + if self.options.cycle: + self.__server = _getLocalServer(self.options) + self.__server.add_resource( + "zenhub", + ZenHubStatus( + lambda: ( + "connected" + if self.__zenhub_connected + else "disconnected" + ) + ), + ) + else: + self.__server = None + + self.__zenhub_connected = False self.__zhclient.notifyOnConnect( lambda: self._set_zenhub_connected(True) ) - self.__zenhub_connected = False def _set_zenhub_connected(self, state): self.__zenhub_connected = state @@ -316,8 +322,11 @@ def run(self): self.derivativeTracker(), ) - self.__server.start() - reactor.addSystemEventTrigger("before", "shutdown", self.__server.stop) + if self.options.cycle: + self.__server.start() + reactor.addSystemEventTrigger( + "before", "shutdown", self.__server.stop + ) reactor.addSystemEventTrigger( "after", diff --git a/Products/ZenHub/tests/test_PBDaemon.py b/Products/ZenHub/tests/test_PBDaemon.py index 3ec77745e8..3bc2ba3d12 100644 --- a/Products/ZenHub/tests/test_PBDaemon.py +++ b/Products/ZenHub/tests/test_PBDaemon.py @@ -191,6 +191,7 @@ def setUp(t): "_getZenHubClient", "EventClient", "EventQueueManager", + "LocalServer", "MetricWriter", "publisher", "reactor", @@ -378,6 +379,7 @@ def test_run(t, TwistedMetricReporter, task, sys): t.pbd.connect = create_autospec(t.pbd.connect) t.pbd._customexitcode = 99 t.pbd.options = Mock(name="options", cycle=True) + t.pbd._PBDaemon__server = Mock() host = "localhost" port = 9999 t.pbd.options.redisUrl = "http://{}:{}".format(host, port) From 7afe430e3094ec4dd493f06c68a0f57d3b1cdfbf Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 23 Feb 2024 15:15:42 -0600 Subject: [PATCH 079/147] fix: CollectorDaemon exits for given an unknown device The startup process has also been changed to allow the configuration task to complete a first run before device configs are loaded. ZEN-34681 --- Products/ZenCollector/config/task.py | 31 +--- Products/ZenCollector/daemon.py | 208 +++++++++++++++-------- Products/ZenCollector/services/config.py | 12 +- 3 files changed, 156 insertions(+), 95 deletions(-) diff --git a/Products/ZenCollector/config/task.py b/Products/ZenCollector/config/task.py index 3ca7d7d88f..4b731008da 100644 --- a/Products/ZenCollector/config/task.py +++ b/Products/ZenCollector/config/task.py @@ -145,11 +145,10 @@ def cleanup(self): class DeviceConfigLoader(object): """Handles retrieving devices from the ConfigCache service.""" - def __init__(self, options, proxy, callback): - self._options = options + def __init__(self, proxy, callback): self._proxy = proxy self._callback = callback - self._deviceIds = set([options.device] if options.device else []) + self._deviceIds = set() self._changes_since = 0 @property @@ -170,34 +169,14 @@ def __call__(self): @defer.inlineCallbacks def _processConfigs(self, config_data): - new = config_data.get("new", []) - updated = config_data.get("updated", []) - removed = config_data.get("removed", []) + new = config_data.get("new", ()) + updated = config_data.get("updated", ()) + removed = config_data.get("removed", ()) try: try: - if self._options.device: - config = self._get_specified_config(new, updated) - if not config: - log.error( - "configuration for %s unavailable -- " - "is that the correct name?", - self._options.device, - ) - defer.returnValue(None) - new = [config] - updated = [] - removed = [] - yield self._callback(new, updated, removed) finally: self._update_local_cache(new, updated, removed) - lengths = (len(new), len(updated), len(removed)) - logmethod = log.debug if lengths == (0, 0, 0) else log.info - logmethod( - "processed %d new, %d updated, and %d removed " - "device configs", - *lengths - ) except Exception: log.exception("failed to process device configs") diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 5a01ef512e..1933738230 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -153,7 +153,6 @@ def __init__( self._device_config_update_interval = 300 self._deviceGuids = {} - self._devices = set() # deprecated; kept for vSphere ZP compatibility self._unresponsiveDevices = set() self._rrd = None self.reconfigureTimeout = None @@ -183,7 +182,10 @@ def __init__( # Let the configuration do any additional startup it might need self.preferences.postStartup() - self.addedPostStartupTasks = False + + # Set flag to limit what actions are run after the first run + # of the config loader task. + self.__first_config_task_run = True # Variables used by enterprise collector in resmgr # @@ -196,24 +198,27 @@ def __init__( # Define _deviceloader to avoid race condition # with task stats recording. - self._deviceloader = DeviceConfigLoader( - self.options, - self._configProxy, - self._deviceConfigCallback, - ) + if self.options.device: + callback = self._singleDeviceConfigCallback + else: + callback = self._manyDeviceConfigCallback + self._deviceloader = DeviceConfigLoader(self._configProxy, callback) self._deviceloadertask = None self._deviceloadertaskd = None + # deprecated; kept for vSphere ZP compatibility + self._devices = _DeviceIdProxy(self._deviceloader) + @property - def preferences(self): # type: () -> ICollectorPreferences + def preferences(self): # type: (Self) -> ICollectorPreferences """The preferences object of this daemon.""" return self._prefs @property - def frameworkFactoryName(self): + def frameworkFactoryName(self): # type: (Self) -> str return self._frameworkFactoryName - def buildOptions(self): + def buildOptions(self): # type: (Self) -> None super(CollectorDaemon, self).buildOptions() maxTasks = getattr(self.preferences, "maxTasks", None) @@ -266,13 +271,13 @@ def buildOptions(self): # give the collector configuration a chance to add options, too self.preferences.buildOptions(self.parser) - def parseOptions(self): + def parseOptions(self): # type: (Self) -> None """Overrides base class to process configuration options.""" super(CollectorDaemon, self).parseOptions() self.preferences.options = self.options # @deprecated - def getInitialServices(self): + def getInitialServices(self): # type: (Self) -> Sequence[str] # Retained for compatibility with ZenPacks fixing CollectorDaemon's old # behavior regarding the `initialServices` attribute. This new # CollectorDaemon respects changes made to the `initialServices` @@ -281,7 +286,7 @@ def getInitialServices(self): # to avoid AttributeError exceptions. return self.initialServices - def watchdogCycleTime(self): + def watchdogCycleTime(self): # type: (Self) -> float """ Return our cycle time (in minutes) @@ -291,7 +296,7 @@ def watchdogCycleTime(self): return self.preferences.cycleInterval * 2 @defer.inlineCallbacks - def connected(self): + def connected(self): # type: (Self) -> Deferred """Invoked after a connection to ZenHub is established.""" try: yield defer.maybeDeferred(self._getInitializationCallback()) @@ -300,7 +305,6 @@ def connected(self): yield self._initEncryptionKey() yield self._startConfigCycle() yield self._startMaintenance() - yield self._startDeviceConfigLoader() yield self._startTaskStatsLogging() except Exception as ex: self.log.critical("unrecoverable error: %s", ex) @@ -313,7 +317,7 @@ def _getInitializationCallback(self): return lambda: None @defer.inlineCallbacks - def _initEncryptionKey(self): + def _initEncryptionKey(self): # type: (Self) -> Deferred # Encrypt dummy msg in order to initialize the encryption key. # The 'yield' does not return until the key is initialized. data = yield self._configProxy.encrypt("Hello") @@ -321,7 +325,7 @@ def _initEncryptionKey(self): self.encryptionKeyInitialized = True self.log.debug("initialized encryption key") - def _startConfigCycle(self, startDelay=0): + def _startConfigCycle(self, startDelay=0): # type: (Self, float) -> None framework = _getFramework(self.frameworkFactoryName) configLoader = framework.getConfigurationLoaderTask()( CONFIG_LOADER_NAME, taskConfig=self.preferences @@ -349,7 +353,7 @@ def _startConfigCycle(self, startDelay=0): configLoader.configId, ) - def _startMaintenance(self): + def _startMaintenance(self): # type: (Self) -> None if not self.options.cycle: return interval = self.preferences.cycleInterval @@ -630,31 +634,90 @@ def _taskCompleteCallback(self, taskName): self._displayStatistics() self.stop() - def _deviceConfigCallback(self, new, updated, removed): + def _singleDeviceConfigCallback(self, new, updated, removed): + # type: ( + # Self, + # Sequence[DeviceProxy], + # Sequence[DeviceProxy], + # Sequence[str] + # ) -> None """ - Update the device configs for the devices this collector manages. + Update the device configs for the devices this collector manages + when a device is specified on the command line. + + :param new: a list of new device configurations + :type new: Sequence[DeviceProxy] + :param updated: a list of updated device configurations + :type updated: Sequence[DeviceProxy] + :param removed: ignored + :type removed: Sequence[str] + """ + config = next( + ( + cfg + for cfg in itertools.chain(new, updated) + if self.options.device == cfg.configId + ), + None, + ) + if not config: + self.log.error( + "configuration for %s unavailable -- " + "is that the correct name?", + self.options.device, + ) + self.stop() + return + + guid = config.deviceGuid + if guid is not None: + self._deviceGuids[config.configId] = guid - :param deviceConfigs: a list of device configurations - :type deviceConfigs: list of name,value tuples + self._updateConfig(config) + + def _manyDeviceConfigCallback(self, new, updated, removed): + # type: ( + # Self, + # Sequence[DeviceProxy], + # Sequence[DeviceProxy], + # Sequence[str] + # ) -> None + """ + Update the device configs for the devices this collector manages + when no device is specified on the command line. + + :param new: a list of new device configurations + :type new: Sequence[DeviceProxy] + :param updated: a list of updated device configurations + :type updated: Sequence[DeviceProxy] + :param removed: a list of devices removed from this collector + :type removed: Sequence[str] """ for deviceId in removed: self._deleteDevice(deviceId) for cfg in itertools.chain(new, updated): + # guard against parsing updates during a disconnect + if cfg is None: + continue + + guid = cfg.deviceGuid + if guid is not None: + self._deviceGuids[cfg.configId] = guid + self._updateConfig(cfg) - self.log.debug( - "processed %d new, %d updated, %d removed device configs", - len(new), - len(updated), - len(removed), + lengths = (len(new), len(updated), len(removed)) + logmethod = self.log.debug if lengths == (0, 0, 0) else self.log.info + logmethod( + "processed %d new, %d updated, and %d removed device configs", + *lengths ) def _deleteDevice(self, deviceId): self._configListener.deleted(deviceId) self._scheduler.removeTasksForConfig(deviceId) self._deviceGuids.pop(deviceId, None) - self._devices.discard(deviceId) self.log.info("removed device config device-id=%s", deviceId) def _updateConfig(self, cfg): @@ -664,27 +727,12 @@ def _updateConfig(self, cfg): Returns True if the configuration was processed, otherwise, False is returned. """ - # guard against parsing updates during a disconnect - if cfg is None: - return False - - configFilter = getattr(self.preferences, "configFilter", _always_ok) - if not ( - (not self.options.device and configFilter(cfg)) - or self.options.device in (cfg.id, cfg.configId) - ): - self.log.info( - "filtered out device config config-id=%s", cfg.configId - ) + if self._is_config_excluded(cfg): return False configId = cfg.configId self.log.debug("processing device config config-id=%s", configId) - guid = getattr(cfg, "_device_guid", None) - if guid is not None: - self._deviceGuids[configId] = guid - nextExpectedRuns = {} if configId in self._deviceloader.deviceIds: tasksToRemove = self._scheduler.getTasksForConfig(configId) @@ -699,9 +747,10 @@ def _updateConfig(self, cfg): ) self._configListener.updated(cfg) else: - self._devices.add(configId) self._configListener.added(cfg) + self._update_thresholds(configId, cfg) + newTasks = self._taskSplitter.splitConfiguration([cfg]) self.log.debug("tasks for config %s: %s", configId, newTasks) @@ -730,20 +779,6 @@ def _updateConfig(self, cfg): ) continue - # TODO: another hack? - if hasattr(cfg, "thresholds"): - try: - self.getThresholds().updateForDevice( - configId, cfg.thresholds - ) - except Exception: - self.log.exception( - "failed to update thresholds " - "config-id=%s thresholds=%r", - configId, - cfg.thresholds, - ) - # if we're not running a normal daemon cycle then keep track of the # tasks we just added for this device so that we can shutdown once # all pending tasks have completed @@ -756,12 +791,33 @@ def _updateConfig(self, cfg): self.log.debug("pausing tasks for device %s", configId) self._scheduler.pauseTasksForConfig(configId) - self.log.debug( - "processed new/updated device config config-id=%s", configId - ) - + self.log.info("processed device config config-id=%s", configId) return True + def _update_thresholds(self, configId, cfg): + thresholds = getattr(cfg, "thresholds", None) + if thresholds: + try: + self.getThresholds().updateForDevice(configId, thresholds) + except Exception: + self.log.exception( + "failed to update thresholds config-id=%s thresholds=%r", + configId, + thresholds, + ) + + def _is_config_excluded(self, cfg): + configFilter = getattr(self.preferences, "configFilter", _always_ok) + if not ( + (not self.options.device and configFilter(cfg)) + or self.options.device in (cfg.id, cfg.configId) + ): + self.log.info( + "filtered out device config config-id=%s", cfg.configId + ) + return True + return False + def setPropertyItems(self, items): """Override so that preferences are updated.""" super(CollectorDaemon, self).setPropertyItems(items) @@ -823,16 +879,15 @@ def _pauseUnreachableDevices(self): def runPostConfigTasks(self): """ Add post-startup tasks from the preferences. - - This may be called with the failure code as well. """ - if not self.addedPostStartupTasks: + if self.__first_config_task_run: postStartupTasks = getattr( self.preferences, "postStartupTasks", lambda: [] ) for _task in postStartupTasks(): self._scheduler.addTask(_task, now=True) - self.addedPostStartupTasks = True + self._startDeviceConfigLoader() + self.__first_config_task_run = False def postStatisticsImpl(self): self._displayStatistics() @@ -897,6 +952,25 @@ def worker_id(self): return getattr(self.options, "workerid", 0) +class _DeviceIdProxy(object): + """ + Exists to maintain an API for ZenPacks that accessed CollectorDaemon's + _devices attribute. + """ + + def __init__(self, loader): + self.__loader = loader + + def __contains__(self, deviceId): + return deviceId in self.__loader.deviceIds + + def add(self, deviceId): + pass + + def discard(self, deviceId): + pass + + def _always_ok(*args): return True diff --git a/Products/ZenCollector/services/config.py b/Products/ZenCollector/services/config.py index 10b9d800be..9e55ba38c1 100644 --- a/Products/ZenCollector/services/config.py +++ b/Products/ZenCollector/services/config.py @@ -46,11 +46,19 @@ def configId(self): def deviceGuid(self): return getattr(self, "_device_guid", None) + def __eq__(self, other): + if isinstance(other, DeviceProxy): + return self.configId == other.configId + return NotImplemented + + def __hash__(self): + return hash(self.configId) + def __str__(self): - return self.id + return self.configId def __repr__(self): - return "%s:%s" % (self.__class__.__name__, self.id) + return "%s:%s" % (self.__class__.__name__, self.configId) pb.setUnjellyableForClass(DeviceProxy, DeviceProxy) From a73064b02e8baad5a61b27f482e347b70a16ef3f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 27 Feb 2024 12:28:01 -0600 Subject: [PATCH 080/147] fix: retry config builds that have expired ZEN-34725 --- .../ZenCollector/configcache/cache/model.py | 3 + .../ZenCollector/configcache/cache/storage.py | 553 +++++------------- .../configcache/cache/table/__init__.py | 19 + .../configcache/cache/table/config.py | 94 +++ .../configcache/cache/table/metadata.py | 114 ++++ .../configcache/cache/table/uid.py | 71 +++ Products/ZenCollector/configcache/manager.py | 53 +- .../configcache/tests/test_storage.py | 166 +++--- 8 files changed, 605 insertions(+), 468 deletions(-) create mode 100644 Products/ZenCollector/configcache/cache/table/__init__.py create mode 100644 Products/ZenCollector/configcache/cache/table/config.py create mode 100644 Products/ZenCollector/configcache/cache/table/metadata.py create mode 100644 Products/ZenCollector/configcache/cache/table/uid.py diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py index 29eefbf57b..8836b36e94 100644 --- a/Products/ZenCollector/configcache/cache/model.py +++ b/Products/ZenCollector/configcache/cache/model.py @@ -109,6 +109,9 @@ def __eq__(self, other): class Expired(object): """The configuration has expired.""" + def __init__(self, ts): + self.expired = ts + def __eq__(self, other): if not isinstance(other, _ConfigStatus.Expired): return NotImplemented diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 9c3355179b..c8d1fc5db3 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -1,6 +1,6 @@ ############################################################################## # -# Copyright (C) Zenoss, Inc. 2019, all rights reserved. +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. @@ -12,20 +12,43 @@ # modelchange:device:uid: # modelchange:device:config::: # modelchange:device:age:: [(, ), ...] +# modelchange:device:retired:: [(, ), ...] +# modelchange:device:expired:: [(, ), ...] # modelchange:device:pending:: [(, ), ...] # modelchange:device:building:: [(, ), ...] # # While "device" seems redundant, other values in this position could be # "threshold" and "property". # -# The "config" segment identifies a key storing a device configuration. -# The "age" segment identifies a key storing a set of device IDs sorted by -# a score that has an simple encoding. The score is segmented by value. -# Values above zero are timestamps of when the current configuration was -# stored in redis. A score of zero means the configuration is outdated -# and needs to be replaced with an updated version. A score less than zero -# is the negated timestamp of when a request was submitted for a device -# configuration build. +# * uid - Maps a device to its object path in ZODB +# * config - Maps a key (::) to a configuration +# * age - Stores the timestamp for when the configuration was created +# +# The following keys store the state of a device's config. +# +# * retired - devices with a 'retired' config. +# +# The is a copy from the 'age' key. Since retirement is +# controlled by a z-property, storing the time when the config +# transitioned to 'retire' is not useful because the z-property +# can change dynamically. +# +# * expired - devices with an 'expired' config. +# +# The is the timestamp when the config was expired. +# +# * pending - devices with a 'pending' config +# +# The is the timestamp when the build_device_config job +# was submitted. +# +# * building - devices with a 'building' config +# +# The is the timestamp when the build_device_config job +# began execution. +# +# A device may exist in only one of 'retired', 'expired', 'pending', +# 'building', or none of them. # # names the configuration service class used to generate the # configuration. @@ -51,13 +74,11 @@ from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl from .model import ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus +from .table import DeviceUIDTable, DeviceConfigTable, ConfigMetadataTable _app = "configcache" log = logging.getLogger("zen.modelchange.stores") -_EXPIRED_SCORE = 0 -_PENDING_SCORE = -1 - class ConfigStoreFactory(Factory): """ @@ -84,18 +105,20 @@ def make(cls): def __init__(self, client): """Initialize a ConfigStore instance.""" self.__client = client - self.__uids = _DeviceUIDTable() - self.__config = _DeviceConfigTable() - self.__age = _ConfigMetadataTable("age") - self.__retired = _ConfigMetadataTable("retired") - self.__pending = _ConfigMetadataTable("pending") - self.__building = _ConfigMetadataTable("building") + self.__uids = DeviceUIDTable(_app) + self.__config = DeviceConfigTable(_app) + self.__age = ConfigMetadataTable(_app, "age") + self.__retired = ConfigMetadataTable(_app, "retired") + self.__expired = ConfigMetadataTable(_app, "expired") + self.__pending = ConfigMetadataTable(_app, "pending") + self.__building = ConfigMetadataTable(_app, "building") self.__range = type( "rangefuncs", (object,), { "age": partial(_range, self.__client, self.__age), "retired": partial(_range, self.__client, self.__retired), + "expired": partial(_range, self.__client, self.__expired), "pending": partial(_range, self.__client, self.__pending), "building": partial(_range, self.__client, self.__building), }, @@ -144,6 +167,7 @@ def _add_impl(pipe): self.__config.delete(pipe, *parts) self.__age.delete(pipe, *parts) self.__retired.delete(pipe, *parts) + self.__expired.delete(pipe, *parts) self.__pending.delete(pipe, *parts) self.__building.delete(pipe, *parts) if add_uid: @@ -151,6 +175,7 @@ def _add_impl(pipe): self.__config.set(pipe, svc, mon, dvc, config) self.__age.add(pipe, svc, mon, dvc, updated) self.__retired.delete(pipe, svc, mon, dvc) + self.__expired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) @@ -195,6 +220,7 @@ def remove(self, *keys): self.__config.delete(pipe, svc, mon, dvc) self.__age.delete(pipe, svc, mon, dvc) self.__retired.delete(pipe, svc, mon, dvc) + self.__expired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) pipe.execute() @@ -213,9 +239,6 @@ def set_retired(self, *keys): A configuration is retired when its `updated` field is less than the difference between the current time and zDeviceConfigMinimumTTL. - Only 'current' configurations can be marked as retired. Attempts - to change configurations in other statuses are ignored. - @type keys: Sequence[ConfigKey] @rtype: Sequence[ConfigKey] """ @@ -223,62 +246,73 @@ def set_retired(self, *keys): return () not_retired = tuple( - key - for key in keys - if not self.__retired.exists( - self.__client, key.service, key.monitor, key.device - ) + self._filter_existing(self.__retired, keys, lambda x: x) ) if len(not_retired) == 0: return () - watch_keys = self._get_watch_keys(keys) - targets = self._filter_by_score_keyonly( - lambda x: x > _EXPIRED_SCORE, not_retired + watch_keys = self._get_watch_keys(not_retired) + scores = ( + ( + key, + self.__age.score( + self.__client, key.service, key.monitor, key.device + ), + ) + for key in not_retired + ) + targets = ( + (key.service, key.monitor, key.device, score) + for key, score in scores ) - if len(targets) == 0: - return () def _impl(pipe): pipe.multi() for svc, mon, dvc, score in targets: self.__retired.add(pipe, svc, mon, dvc, score) + self.__expired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) self.__client.transaction(_impl, *watch_keys) - return tuple(ConfigKey(svc, mon, dvc) for svc, mon, dvc, _ in targets) + return not_retired - def set_expired(self, *keys): + def set_expired(self, *pairs): """ Marks the indicated configuration as expired. Attempts to mark configurations that are not 'current' or 'retired' are ignored. - @type keys: Sequence[ConfigKey] + @type keys: Sequence[(ConfigKey, float)] @rtype: Sequence[ConfigKey] """ - if len(keys) == 0: + if len(pairs) == 0: return () - watch_keys = self._get_watch_keys(keys) - targets = self._filter_by_score_keyonly( - lambda x: x > _EXPIRED_SCORE, keys + not_expired = tuple( + self._filter_existing(self.__expired, pairs, lambda x: x[0]) ) - if len(targets) == 0: + if len(not_expired) == 0: return () + watch_keys = self._get_watch_keys(key for key, _ in not_expired) + targets = ( + (key.service, key.monitor, key.device, ts) + for key, ts in not_expired + ) + def _impl(pipe): pipe.multi() - for svc, mon, dvc, _ in targets: - self.__age.add(pipe, svc, mon, dvc, _EXPIRED_SCORE) + for svc, mon, dvc, ts in targets: + score = _to_score(ts) self.__retired.delete(pipe, svc, mon, dvc) + self.__expired.add(pipe, svc, mon, dvc, score) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) self.__client.transaction(_impl, *watch_keys) - return tuple(ConfigKey(svc, mon, dvc) for svc, mon, dvc, _ in targets) + return tuple(key for key, _ in not_expired) def set_pending(self, *pairs): """ @@ -290,27 +324,29 @@ def set_pending(self, *pairs): if len(pairs) == 0: return () - watch_keys = self._get_watch_keys(key for key, _ in pairs) - targets = self._filter_by_score_with_start( - lambda x: x == _EXPIRED_SCORE, pairs + not_pending = tuple( + self._filter_existing(self.__pending, pairs, lambda x: x[0]) ) - - if len(targets) == 0: + if len(not_pending) == 0: return () + watch_keys = self._get_watch_keys(key for key, _ in not_pending) + targets = ( + (key.service, key.monitor, key.device, ts) + for key, ts in not_pending + ) + def _impl(pipe): pipe.multi() - for svc, mon, dvc, ts, _ in targets: + for svc, mon, dvc, ts in targets: score = _to_score(ts) - self.__age.add(pipe, svc, mon, dvc, _PENDING_SCORE) self.__retired.delete(pipe, svc, mon, dvc) - self.__building.delete(pipe, svc, mon, dvc) + self.__expired.delete(pipe, svc, mon, dvc) self.__pending.add(pipe, svc, mon, dvc, score) + self.__building.delete(pipe, svc, mon, dvc) self.__client.transaction(_impl, *watch_keys) - return tuple( - ConfigKey(svc, mon, dvc) for svc, mon, dvc, _, _ in targets - ) + return tuple(key for key, _ in not_pending) def set_building(self, *pairs): """ @@ -322,58 +358,37 @@ def set_building(self, *pairs): if len(pairs) == 0: return () - valid = tuple( - (key, ts) - for key, ts in pairs - if self.__pending.exists( - self.__client, key.service, key.monitor, key.device - ) + not_building = tuple( + self._filter_existing(self.__building, pairs, lambda x: x[0]) ) - if len(valid) == 0: - return valid + if len(not_building) == 0: + return () - watch_keys = self._get_watch_keys(key for key, _ in valid) + watch_keys = self._get_watch_keys(key for key, _ in not_building) + targets = ( + (key.service, key.monitor, key.device, ts) + for key, ts in not_building + ) def _impl(pipe): pipe.multi() - for key, ts in valid: - svc = key.service - mon = key.monitor - dvc = key.device + for svc, mon, dvc, ts in targets: + score = _to_score(ts) + self.__retired.delete(pipe, svc, mon, dvc) + self.__expired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) - self.__building.add(pipe, svc, mon, dvc, _to_score(ts)) + self.__building.add(pipe, svc, mon, dvc, score) self.__client.transaction(_impl, *watch_keys) - return tuple(key for key, _ in valid) - - def _filter_by_score_keyonly(self, predicate, keys): - pairs = ((key, None) for key in keys) - return tuple( - (svc, mon, dvc, score) - for svc, mon, dvc, _, score in self._filter_by_score( - predicate, pairs - ) - ) - - def _filter_by_score_with_start(self, predicate, pairs): - return tuple(self._filter_by_score(predicate, pairs)) + return tuple(key for key, _ in not_building) - def _filter_by_score(self, predicate, pairs): - scores = ( - ( - key, - started, - self.__age.score( - self.__client, key.service, key.monitor, key.device - ), - ) - for key, started in pairs - ) - return ( - (key.service, key.monitor, key.device, started, score) - for key, started, score in scores - if predicate(score) - ) + def _filter_existing(self, table, items, getkey): + for item in items: + key = getkey(item) + if not table.exists( + self.__client, key.service, key.monitor, key.device + ): + yield item def get_status(self, *keys): """ @@ -381,40 +396,11 @@ def get_status(self, *keys): @rtype: Iterable[Tuple[ConfigKey, ConfigStatus]] """ - scores = ( - ( - key, - self.__age.score( - self.__client, key.service, key.monitor, key.device - ), - ) - for key in keys - ) - return iter(self._iter_status(scores)) - - def _iter_status(self, scores): - for key, score in scores: - if score > 0: - rscore = self.__retired.score( - self.__client, key.service, key.monitor, key.device - ) - if rscore is not None: - yield (key, ConfigStatus.Retired(_to_ts(rscore))) - else: - yield (key, ConfigStatus.Current(_to_ts(score))) - elif score == 0: - yield (key, ConfigStatus.Expired()) - else: - pscore = self.__pending.score( - self.__client, key.service, key.monitor, key.device - ) - if pscore is not None: - yield (key, ConfigStatus.Pending(_to_ts(pscore))) - bscore = self.__building.score( - self.__client, key.service, key.monitor, key.device - ) - if bscore is not None: - yield (key, ConfigStatus.Building(_to_ts(bscore))) + for key in keys: + scores = self._get_scores(key) + status = self._get_status(scores) + if status is not None: + yield (key, status) def get_building(self, service="*", monitor="*"): """ @@ -445,10 +431,8 @@ def get_expired(self, service="*", monitor="*"): @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Expired]] """ return ( - (key, ConfigStatus.Expired()) - for key, _ in self.__range.age( - service, monitor, minv=0.0, maxv=0.0 - ) + (key, ConfigStatus.Expired(ts)) + for key, ts in self.__range.expired(service, monitor) ) def get_retired(self, service="*", monitor="*"): @@ -470,12 +454,17 @@ def get_older(self, maxtimestamp, service="*", monitor="*"): @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Current]] """ # NOTE: 'older' means timestamps > 0 and <= `maxtimestamp`. - return ( - (key, ConfigStatus.Current(ts)) - for key, ts in self.__range.age( + selection = tuple( + (key, age) + for key, age in self.__range.age( service, monitor, minv="(0", maxv=_to_score(maxtimestamp) ) ) + for key, age in selection: + scores = self._get_scores(key)[1:] + if any(score is not None for score in scores): + continue + yield (key, ConfigStatus.Current(age)) def get_newer(self, mintimestamp, service="*", monitor="*"): """ @@ -485,12 +474,40 @@ def get_newer(self, mintimestamp, service="*", monitor="*"): @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Current]] """ # NOTE: 'newer' means timestamps to `maxtimestamp`. - return ( - (key, ConfigStatus.Current(ts)) - for key, ts in self.__range.age( + selection = tuple( + (key, age) + for key, age in self.__range.age( service, monitor, minv="(%s" % (_to_score(mintimestamp),) ) ) + for key, age in selection: + scores = self._get_scores(key)[1:] + if any(score is not None for score in scores): + continue + yield (key, ConfigStatus.Current(age)) + + def _get_scores(self, key): + service, monitor, device = attr.astuple(key) + with self.__client.pipeline() as pipe: + self.__age.score(pipe, service, monitor, device), + self.__retired.score(pipe, service, monitor, device), + self.__expired.score(pipe, service, monitor, device), + self.__pending.score(pipe, service, monitor, device), + self.__building.score(pipe, service, monitor, device), + return pipe.execute() + + def _get_status(self, scores): + age, retired, expired, pending, building = scores + if building is not None: + return ConfigStatus.Building(_to_ts(building)) + elif pending is not None: + return ConfigStatus.Pending(_to_ts(pending)) + elif expired is not None: + return ConfigStatus.Expired(_to_ts(expired)) + elif retired is not None: + return ConfigStatus.Retired(_to_ts(retired)) + elif age is not None: + return ConfigStatus.Current(_to_ts(age)) def _get_watch_keys(self, keys): return set( @@ -498,6 +515,7 @@ def _get_watch_keys(self, keys): ( self.__age.make_key(key.service, key.monitor), self.__retired.make_key(key.service, key.monitor), + self.__expired.make_key(key.service, key.monitor), self.__pending.make_key(key.service, key.monitor), self.__building.make_key(key.service, key.monitor), ) @@ -546,261 +564,6 @@ def _from_record(record): ) -class _DeviceUIDTable(object): - """ - Manages mapping device names to their ZODB UID. - """ - - def __init__(self, scan_page_size=1000, mget_page_size=10): - """Initialize a _DeviceUIDTable instance.""" - self.__template = "{app}:device:uid:{{device}}".format(app=_app) - self.__scan_count = scan_page_size - self.__mget_count = mget_page_size - - def make_key(self, device): - return self.__template.format(device=device) - - def exists(self, client, device): - """Return True if configuration data exists for the given ID. - - :param device: The ID of the device - :type device: str - :rtype: boolean - """ - return client.exists(self.make_key(device)) - - def scan(self, client, device="*"): - """ - Return an iterable of tuples of device names. - """ - pattern = self.make_key(device) - result = client.scan_iter(match=pattern, count=self.__scan_count) - return (key.rsplit(":", 1)[-1] for key in result) - - def get(self, client, device): - """Return the UID of the given device name. - - :type device: str - :rtype: str - """ - key = self.make_key(device) - return client.get(key) - - def set(self, client, device, uid): - """Insert or replace the UID for the given device. - - :param device: The ID of the configuration - :type device: str - :param uid: The ZODB UID of the device - :type uid: str - :raises: ValueError - """ - key = self.make_key(device) - client.set(key, uid) - - def delete(self, client, *devices): - """Delete one or more keys. - - This method does not fail if the key doesn't exist. - - :type uids: Sequence[str] - """ - keys = tuple(self.make_key(dvc) for dvc in devices) - client.delete(*keys) - - -class _DeviceConfigTable(object): - """ - Manages device configuration data for a specific configuration service. - """ - - def __init__(self, scan_page_size=1000, mget_page_size=10): - """Initialize a _DeviceConfigTable instance.""" - self.__template = ( - "{app}:device:config:{{service}}:{{monitor}}:{{device}}".format( - app=_app - ) - ) - self.__scan_count = scan_page_size - self.__mget_count = mget_page_size - - def make_key(self, service, monitor, device): - return self.__template.format( - service=service, monitor=monitor, device=device - ) - - def exists(self, client, service, monitor, device): - """Return True if configuration data exists for the given ID. - - :param service: Name of the configuration service. - :type service: str - :param monitor: Name of the monitor the device is a member of. - :type monitor: str - :param device: The ID of the device - :type device: str - :rtype: boolean - """ - return client.exists(self.make_key(service, monitor, device)) - - def scan(self, client, service="*", monitor="*", device="*"): - """ - Return an iterable of tuples of (service, monitor, device). - """ - pattern = self.make_key(service, monitor, device) - result = client.scan_iter(match=pattern, count=self.__scan_count) - return (tuple(key.rsplit(":", 3)[1:]) for key in result) - - def get(self, client, service, monitor, device): - """Return the config data for the given config ID. - - If the config ID is not found, the default argument is returned. - - :type service: str - :type monitor: str - :type device: str - :rtype: Union[IJellyable, None] - """ - key = self.make_key(service, monitor, device) - return client.get(key) - - def set(self, client, service, monitor, device, data): - """Insert or replace the config data for the given config ID. - - If existing data for the device exists under a different monitor, - it will be deleted. - - :param service: The name of the configuration service. - :type service: str - :param monitor: The ID of the performance monitor - :type monitor: str - :param device: The ID of the configuration - :type device: str - :param data: The serialized configuration data - :type data: str - :raises: ValueError - """ - key = self.make_key(service, monitor, device) - client.set(key, data) - - def delete(self, client, service, monitor, device): - """Delete a key. - - This method does not fail if the key doesn't exist. - - :type service: str - :type monitor: str - :type device: str - """ - key = self.make_key(service, monitor, device) - client.delete(key) - - -class _ConfigMetadataTable(object): - """ - Manages the mapping of device configurations to monitors. - - Configuration IDs are mapped to service ID/monitor ID pairs. - - A Service ID/monitor ID pair are used as a key to retrieve the - Configuration IDs mapped to the pair. - """ - - def __init__(self, category): - """Initialize a ConfigMetadataStore instance.""" - self.__template = ( - "{app}:device:{category}:{{service}}:{{monitor}}".format( - app=_app, category=category - ) - ) - self.__scan_count = 1000 - - def make_key(self, service, monitor): - return self.__template.format(service=service, monitor=monitor) - - def get_pairs(self, client, service="*", monitor="*"): - pattern = self.make_key(service, monitor) - return ( - key.rsplit(":", 2)[1:] - for key in client.scan_iter(match=pattern, count=self.__scan_count) - ) - - def scan(self, client, pairs): - """ - Return an iterable of tuples of (service, monitor, device, score). - - @type client: redis client - @type pairs: Iterable[Tuple[str, str]] - @rtype Iterator[Tuple[str, str, str, float]] - """ - return ( - (service, monitor, dvc, score) - for service, monitor in pairs - for dvc, score in client.zscan_iter( - self.make_key(service, monitor), count=self.__scan_count - ) - ) - - def range(self, client, pairs, maxscore=None, minscore=None): - """ - Return an iterable of tuples of (service, monitor, device, score). - - @type client: redis client - @type pairs: Iterable[Tuple[str, str]] - @type minscore: Union[float, None] - @type maxscore: Union[float, None] - @rtype Iterator[Tuple[str, str, str, float]] - """ - maxv = maxscore if maxscore is not None else "+inf" - minv = minscore if minscore is not None else "-inf" - return ( - (service, monitor, device, score) - for service, monitor in pairs - for device, score in client.zrangebyscore( - self.make_key(service, monitor), minv, maxv, withscores=True - ) - ) - - def exists(self, client, service, monitor, device): - """Return True if a score for the key and device exists. - - @type client: RedisClient - @type service: str - @type monitor: str - @type device: str - """ - key = self.make_key(service, monitor) - return client.zscore(key, device) is not None - - def add(self, client, service, monitor, device, score): - """ - Add a (device, score) -> (monitor, serviceid) mapping. - This method will replace any existing mapping for device. - - @type client: RedisClient - @type service: str - @type monitor: str - @type device: str - @type score: float - """ - key = self.make_key(service, monitor) - client.zadd(key, score, device) - - def score(self, client, service, monitor, device): - """ - Returns the timestamp associated with the device ID. - Returns None of the device ID is not found. - """ - key = self.make_key(service, monitor) - return client.zscore(key, device) - - def delete(self, client, service, monitor, device): - """ - Removes a device from a (service, monitor) key. - """ - key = self.make_key(service, monitor) - client.zrem(key, device) - - def _batched(iterable, n): """ Batch data into tuples of length `n`. The last batch may be shorter. diff --git a/Products/ZenCollector/configcache/cache/table/__init__.py b/Products/ZenCollector/configcache/cache/table/__init__.py new file mode 100644 index 0000000000..fb4a7136d8 --- /dev/null +++ b/Products/ZenCollector/configcache/cache/table/__init__.py @@ -0,0 +1,19 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from .uid import DeviceUIDTable +from .config import DeviceConfigTable +from .metadata import ConfigMetadataTable + + +__all__ = ( + "DeviceUIDTable", + "DeviceConfigTable", + "ConfigMetadataTable", +) diff --git a/Products/ZenCollector/configcache/cache/table/config.py b/Products/ZenCollector/configcache/cache/table/config.py new file mode 100644 index 0000000000..ef2bed2b5f --- /dev/null +++ b/Products/ZenCollector/configcache/cache/table/config.py @@ -0,0 +1,94 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +class DeviceConfigTable(object): + """ + Manages device configuration data for a specific configuration service. + """ + + def __init__(self, app, scan_page_size=1000, mget_page_size=10): + """Initialize a DeviceConfigTable instance.""" + self.__template = ( + "{app}:device:config:{{service}}:{{monitor}}:{{device}}".format( + app=app + ) + ) + self.__scan_count = scan_page_size + self.__mget_count = mget_page_size + + def make_key(self, service, monitor, device): + return self.__template.format( + service=service, monitor=monitor, device=device + ) + + def exists(self, client, service, monitor, device): + """Return True if configuration data exists for the given ID. + + :param service: Name of the configuration service. + :type service: str + :param monitor: Name of the monitor the device is a member of. + :type monitor: str + :param device: The ID of the device + :type device: str + :rtype: boolean + """ + return client.exists(self.make_key(service, monitor, device)) + + def scan(self, client, service="*", monitor="*", device="*"): + """ + Return an iterable of tuples of (service, monitor, device). + """ + pattern = self.make_key(service, monitor, device) + result = client.scan_iter(match=pattern, count=self.__scan_count) + return (tuple(key.rsplit(":", 3)[1:]) for key in result) + + def get(self, client, service, monitor, device): + """Return the config data for the given config ID. + + If the config ID is not found, the default argument is returned. + + :type service: str + :type monitor: str + :type device: str + :rtype: Union[IJellyable, None] + """ + key = self.make_key(service, monitor, device) + return client.get(key) + + def set(self, client, service, monitor, device, data): + """Insert or replace the config data for the given config ID. + + If existing data for the device exists under a different monitor, + it will be deleted. + + :param service: The name of the configuration service. + :type service: str + :param monitor: The ID of the performance monitor + :type monitor: str + :param device: The ID of the configuration + :type device: str + :param data: The serialized configuration data + :type data: str + :raises: ValueError + """ + key = self.make_key(service, monitor, device) + client.set(key, data) + + def delete(self, client, service, monitor, device): + """Delete a key. + + This method does not fail if the key doesn't exist. + + :type service: str + :type monitor: str + :type device: str + """ + key = self.make_key(service, monitor, device) + client.delete(key) diff --git a/Products/ZenCollector/configcache/cache/table/metadata.py b/Products/ZenCollector/configcache/cache/table/metadata.py new file mode 100644 index 0000000000..b88411eac0 --- /dev/null +++ b/Products/ZenCollector/configcache/cache/table/metadata.py @@ -0,0 +1,114 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +class ConfigMetadataTable(object): + """ + Manages the mapping of device configurations to monitors. + + Configuration IDs are mapped to service ID/monitor ID pairs. + + A Service ID/monitor ID pair are used as a key to retrieve the + Configuration IDs mapped to the pair. + """ + + def __init__(self, app, category): + """Initialize a ConfigMetadataStore instance.""" + self.__template = ( + "{app}:device:{category}:{{service}}:{{monitor}}".format( + app=app, category=category + ) + ) + self.__scan_count = 1000 + + def make_key(self, service, monitor): + return self.__template.format(service=service, monitor=monitor) + + def get_pairs(self, client, service="*", monitor="*"): + pattern = self.make_key(service, monitor) + return ( + key.rsplit(":", 2)[1:] + for key in client.scan_iter(match=pattern, count=self.__scan_count) + ) + + def scan(self, client, pairs): + """ + Return an iterable of tuples of (service, monitor, device, score). + + @type client: redis client + @type pairs: Iterable[Tuple[str, str]] + @rtype Iterator[Tuple[str, str, str, float]] + """ + return ( + (service, monitor, dvc, score) + for service, monitor in pairs + for dvc, score in client.zscan_iter( + self.make_key(service, monitor), count=self.__scan_count + ) + ) + + def range(self, client, pairs, maxscore=None, minscore=None): + """ + Return an iterable of tuples of (service, monitor, device, score). + + @type client: redis client + @type pairs: Iterable[Tuple[str, str]] + @type minscore: Union[float, None] + @type maxscore: Union[float, None] + @rtype Iterator[Tuple[str, str, str, float]] + """ + maxv = maxscore if maxscore is not None else "+inf" + minv = minscore if minscore is not None else "-inf" + return ( + (service, monitor, device, score) + for service, monitor in pairs + for device, score in client.zrangebyscore( + self.make_key(service, monitor), minv, maxv, withscores=True + ) + ) + + def exists(self, client, service, monitor, device): + """Return True if a score for the key and device exists. + + @type client: RedisClient + @type service: str + @type monitor: str + @type device: str + """ + key = self.make_key(service, monitor) + return client.zscore(key, device) is not None + + def add(self, client, service, monitor, device, score): + """ + Add a (device, score) -> (monitor, serviceid) mapping. + This method will replace any existing mapping for device. + + @type client: RedisClient + @type service: str + @type monitor: str + @type device: str + @type score: float + """ + key = self.make_key(service, monitor) + client.zadd(key, score, device) + + def score(self, client, service, monitor, device): + """ + Returns the timestamp associated with the device ID. + Returns None of the device ID is not found. + """ + key = self.make_key(service, monitor) + return client.zscore(key, device) + + def delete(self, client, service, monitor, device): + """ + Removes a device from a (service, monitor) key. + """ + key = self.make_key(service, monitor) + client.zrem(key, device) diff --git a/Products/ZenCollector/configcache/cache/table/uid.py b/Products/ZenCollector/configcache/cache/table/uid.py new file mode 100644 index 0000000000..7942b60fcd --- /dev/null +++ b/Products/ZenCollector/configcache/cache/table/uid.py @@ -0,0 +1,71 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +class DeviceUIDTable(object): + """ + Manages mapping device names to their ZODB UID. + """ + + def __init__(self, app, scan_page_size=1000, mget_page_size=10): + """Initialize a DeviceUIDTable instance.""" + self.__template = "{app}:device:uid:{{device}}".format(app=app) + self.__scan_count = scan_page_size + self.__mget_count = mget_page_size + + def make_key(self, device): + return self.__template.format(device=device) + + def exists(self, client, device): + """Return True if configuration data exists for the given ID. + + :param device: The ID of the device + :type device: str + :rtype: boolean + """ + return client.exists(self.make_key(device)) + + def scan(self, client, device="*"): + """ + Return an iterable of tuples of device names. + """ + pattern = self.make_key(device) + result = client.scan_iter(match=pattern, count=self.__scan_count) + return (key.rsplit(":", 1)[-1] for key in result) + + def get(self, client, device): + """Return the UID of the given device name. + + :type device: str + :rtype: str + """ + key = self.make_key(device) + return client.get(key) + + def set(self, client, device, uid): + """Insert or replace the UID for the given device. + + :param device: The ID of the configuration + :type device: str + :param uid: The ZODB UID of the device + :type uid: str + :raises: ValueError + """ + key = self.make_key(device) + client.set(key, uid) + + def delete(self, client, *devices): + """Delete one or more keys. + + This method does not fail if the key doesn't exist. + + :type uids: Sequence[str] + """ + keys = tuple(self.make_key(dvc) for dvc in devices) + client.delete(*keys) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 6f59b45487..7cd7f3d811 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -88,14 +88,48 @@ def run(self): while not self.ctx.controller.shutdown: try: self.ctx.session.sync() - self._retry_pending_builds() + self._retry_build() + self._retry_pending() self._expire_retired_configs() self._rebuild_older_configs() except Exception as ex: self.log.exception("unexpected error %s", ex) self.ctx.controller.wait(self.interval) - def _retry_pending_builds(self): + def _retry_build(self): + buildlimitmap = DevicePropertyMap.make_build_timeout_map( + self.ctx.dmd.Devices + ) + # Test against a time 10 minutes earlier to minimize interfering + # with builder working on the same config. + now = time() - 600 + count = 0 + for key, status in self.store.get_building(): + uid = self.store.get_uid(key.device) + if uid is None: + self.log.warn( + "No UID found for device device=%s", key.device + ) + continue + duration = buildlimitmap.get(uid) + if status.started < (now - duration): + self.store.set_expired(key) + self.log.info( + "expired configuration due to build timeout " + "started=%s timeout=%s service=%s monitor=%s device=%s", + datetime.fromtimestamp(status.started).strftime( + "%Y-%m-%d %H:%M:%S" + ), + duration, + key.service, + key.monitor, + key.device, + ) + count += 1 + if count == 0: + self.log.debug("no configuration builds have timed out") + + def _retry_pending(self): pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( self.ctx.dmd.Devices ) @@ -103,16 +137,20 @@ def _retry_pending_builds(self): count = 0 for key, status in self.store.get_pending(): uid = self.store.get_uid(key.device) + if uid is None: + self.log.warn( + "No UID found for device device=%s", key.device + ) + continue duration = pendinglimitmap.get(uid) if status.submitted < (now - duration): self.store.set_expired(key) self.log.info( - "pending configuration build has timed out " - "submitted=%s service=%s monitor=%s device=%s", + "expired pending configuration build due to timeout " + "submitted=%s timeout=%s service=%s monitor=%s device=%s", datetime.fromtimestamp(status.submitted).strftime( "%Y-%m-%d %H:%M:%S" ), - Constants.build_timeout_id, duration, key.service, key.monitor, @@ -155,6 +193,11 @@ def _rebuild_older_configs(self): count = 0 for key, status in results: uid = self.store.get_uid(key.device) + if uid is None: + self.log.warn( + "No UID found for device device=%s", key.device + ) + continue ttl = ttlmap.get(uid) expiration_threshold = now - ttl if ( diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py index 226b8e579d..d96bf2ee77 100644 --- a/Products/ZenCollector/configcache/tests/test_storage.py +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -447,22 +447,26 @@ class TestExpiredStatus(_BaseTest): def test_set_expired(t): t.store.add(t.record1) - expected = (t.record1.key,) + ts = t.record1.updated + 500 - actual = t.store.set_expired(t.record1.key) + expected = (t.record1.key,) + actual = t.store.set_expired((t.record1.key, ts)) t.assertTupleEqual(expected, actual) def test_set_expired_twice(t): t.store.add(t.record1) + ts = t.record1.updated + 500 + expected = () - t.store.set_expired(t.record1.key) - actual = t.store.set_expired(t.record1.key) + t.store.set_expired((t.record1.key, ts)) + actual = t.store.set_expired((t.record1.key, ts)) t.assertTupleEqual(expected, actual) def test_expired_status(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) + ts = t.record1.updated + 500 + t.store.set_expired((t.record1.key, ts)) result = tuple(t.store.get_status(t.record1.key)) @@ -473,7 +477,8 @@ def test_expired_status(t): def test_get_expired(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) + ts = t.record1.updated + 500 + t.store.set_expired((t.record1.key, ts)) result = tuple(t.store.get_expired()) t.assertEqual(1, len(result)) @@ -483,7 +488,8 @@ def test_get_expired(t): def test_expired_is_not_older(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) + ts = t.record1.updated + 500 + t.store.set_expired((t.record1.key, ts)) result = tuple(t.store.get_older(t.record1.updated)) t.assertEqual(0, len(result)) @@ -499,7 +505,6 @@ def test_set_pending(t): submitted = t.record1.updated + 500 expected = (t.record1.key,) - t.store.set_expired(t.record1.key) actual = t.store.set_pending((t.record1.key, submitted)) t.assertTupleEqual(expected, actual) @@ -521,7 +526,6 @@ def test_set_pending_twice(t): def test_pending_status(t): t.store.add(t.record1) submitted = t.record1.updated + 500 - t.store.set_expired(t.record1.key) t.store.set_pending((t.record1.key, submitted)) result = tuple(t.store.get_status(t.record1.key)) @@ -533,7 +537,6 @@ def test_pending_status(t): def test_get_pending(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) submitted = t.record1.updated + 500 t.store.set_pending((t.record1.key, submitted)) @@ -546,9 +549,7 @@ def test_get_pending(t): def test_pending_is_not_older(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) submitted = t.record1.updated + 500 - t.store.set_expired(t.record1.key) t.store.set_pending((t.record1.key, submitted)) result = tuple(t.store.get_older(t.record1.updated)) @@ -565,7 +566,6 @@ def test_set_building(t): started = t.record1.updated + 500 expected = (t.record1.key,) - t.store.set_expired(t.record1.key) t.store.set_pending((t.record1.key, started - 100)) actual = t.store.set_building((t.record1.key, started)) t.assertTupleEqual(expected, actual) @@ -584,8 +584,6 @@ def test_set_building_twice(t): started = t.record1.updated + 500 expected = () - t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) actual = t.store.set_building((t.record1.key, started)) t.assertTupleEqual(expected, actual) @@ -593,8 +591,6 @@ def test_set_building_twice(t): def test_building_status(t): t.store.add(t.record1) started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) result = tuple(t.store.get_status(t.record1.key)) @@ -606,9 +602,7 @@ def test_building_status(t): def test_get_building(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) started = t.record1.updated + 500 - t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) result = tuple(t.store.get_building()) @@ -620,7 +614,6 @@ def test_get_building(t): def test_building_is_not_older(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) started = t.record1.updated + 500 t.store.set_building((t.record1.key, started)) @@ -636,8 +629,9 @@ class TestExpiredTransitions(_BaseTest): def test_retired_to_expired(t): t.store.add(t.record1) t.store.set_retired(t.record1.key) + ts = t.record1.updated + 300 - expired_keys = t.store.set_expired(t.record1.key) + expired_keys = t.store.set_expired((t.record1.key, ts)) t.assertTupleEqual((t.record1.key,), expired_keys) result = tuple(t.store.get_status(t.record1.key)) @@ -651,15 +645,17 @@ def test_retired_to_expired(t): def test_expired_to_retired(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) + ts = t.record1.updated + 300 + t.store.set_expired((t.record1.key, ts)) retired_keys = t.store.set_retired(t.record1.key) - t.assertTupleEqual((), retired_keys) + t.assertTupleEqual((t.record1.key,), retired_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Expired) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(t.record1.updated, status.updated) class TestPendingTransitions(_BaseTest): @@ -672,14 +668,14 @@ def test_current_to_pending(t): submitted = t.record1.updated + 500 pending_keys = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual((), pending_keys) + t.assertTupleEqual((t.record1.key,), pending_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Current) - t.assertEqual(t.record1.updated, status.updated) + t.assertIsInstance(status, ConfigStatus.Pending) + t.assertEqual(submitted, status.submitted) def test_retired_to_pending(t): t.store.add(t.record1) @@ -687,19 +683,20 @@ def test_retired_to_pending(t): submitted = t.record1.updated + 500 pending_keys = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual((), pending_keys) + t.assertTupleEqual((t.record1.key,), pending_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(t.record1.updated, status.updated) + t.assertIsInstance(status, ConfigStatus.Pending) + t.assertEqual(submitted, status.submitted) def test_expired_to_pending(t): t.store.add(t.record1) + ts = t.record1.updated + 300 submitted = t.record1.updated + 500 - t.store.set_expired(t.record1.key) + t.store.set_expired((t.record1.key, ts)) pending_keys = t.store.set_pending((t.record1.key, submitted)) t.assertTupleEqual((t.record1.key,), pending_keys) @@ -715,35 +712,39 @@ def test_expired_to_pending(t): retired_keys = tuple(t.store.get_retired()) t.assertTupleEqual((), retired_keys) + building_keys = tuple(t.store.get_building()) + t.assertTupleEqual((), building_keys) + def test_pending_to_expired(t): t.store.add(t.record1) + ts = t.record1.updated + 300 submitted = t.record1.updated + 500 - t.store.set_expired(t.record1.key) t.store.set_pending((t.record1.key, submitted)) - expired_keys = t.store.set_expired(t.record1.key) - t.assertTupleEqual((), expired_keys) + expired_keys = t.store.set_expired((t.record1.key, ts)) + t.assertTupleEqual((t.record1.key,), expired_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Pending) + t.assertIsInstance(status, ConfigStatus.Expired) + t.assertEqual(ts, status.expired) def test_pending_to_retired(t): t.store.add(t.record1) submitted = t.record1.updated + 500 - t.store.set_expired(t.record1.key) t.store.set_pending((t.record1.key, submitted)) retired_keys = t.store.set_retired(t.record1.key) - t.assertTupleEqual((), retired_keys) + t.assertTupleEqual((t.record1.key,), retired_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Pending) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(t.record1.updated, status.updated) class TestBuildingTransitions(_BaseTest): @@ -756,14 +757,23 @@ def test_current_to_building(t): started = t.record1.updated + 500 building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((), building_keys) + t.assertTupleEqual((t.record1.key,), building_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Current) - t.assertEqual(t.record1.updated, status.updated) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) def test_retired_to_building(t): t.store.add(t.record1) @@ -771,33 +781,54 @@ def test_retired_to_building(t): started = t.record1.updated + 500 building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((), building_keys) + t.assertTupleEqual((t.record1.key,), building_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(t.record1.updated, status.updated) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) def test_expired_to_building(t): t.store.add(t.record1) + ts = t.record1.updated + 300 started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) + t.store.set_expired((t.record1.key, ts)) building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((), building_keys) + t.assertTupleEqual((t.record1.key,), building_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Expired) + t.assertIsInstance(status, ConfigStatus.Building) + t.assertEqual(started, status.started) + + pending_keys = tuple(t.store.get_pending()) + t.assertTupleEqual((), pending_keys) + + expired_keys = tuple(t.store.get_expired()) + t.assertTupleEqual((), expired_keys) + + retired_keys = tuple(t.store.get_retired()) + t.assertTupleEqual((), retired_keys) def test_pending_to_building(t): t.store.add(t.record1) + ts = t.record1.updated + 300 started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) + t.store.set_expired((t.record1.key, ts)) t.store.set_pending((t.record1.key, started - 100)) building_keys = t.store.set_building((t.record1.key, started)) @@ -821,54 +852,50 @@ def test_pending_to_building(t): def test_building_to_pending(t): t.store.add(t.record1) + submitted = t.record1.updated + 300 started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) - pending_keys = t.store.set_pending((t.record1.key, started)) - t.assertTupleEqual((), pending_keys) + pending_keys = t.store.set_pending((t.record1.key, submitted)) + t.assertTupleEqual((t.record1.key,), pending_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + t.assertIsInstance(status, ConfigStatus.Pending) + t.assertEqual(submitted, status.submitted) def test_building_to_expired(t): t.store.add(t.record1) + expired = t.record1.updated + 300 started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) - expired_keys = t.store.set_expired(t.record1.key) - t.assertTupleEqual((), expired_keys) + expired_keys = t.store.set_expired((t.record1.key, expired)) + t.assertTupleEqual((t.record1.key,), expired_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + t.assertIsInstance(status, ConfigStatus.Expired) + t.assertEqual(expired, status.expired) def test_building_to_retired(t): t.store.add(t.record1) started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) - t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) retired_keys = t.store.set_retired(t.record1.key) - t.assertTupleEqual((), retired_keys) + t.assertTupleEqual((t.record1.key,), retired_keys) result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) key, status = result[0] t.assertEqual(t.record1.key, key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + t.assertIsInstance(status, ConfigStatus.Retired) + t.assertEqual(t.record1.updated, status.updated) class TestAddTransitions(_BaseTest): @@ -893,7 +920,8 @@ def test_add_overwrites_retired(t): def test_add_overwrites_expired(t): t.store.add(t.record1) - t.store.set_expired(t.record1.key) + ts = t.record1.updated + 300 + t.store.set_expired((t.record1.key, ts)) t.store.add(t.record1) expired_keys = tuple(t.store.get_expired()) @@ -908,8 +936,9 @@ def test_add_overwrites_expired(t): def test_add_overwrites_pending(t): t.store.add(t.record1) + ts = t.record1.updated + 300 submitted = t.record1.updated + 500 - t.store.set_expired(t.record1.key) + t.store.set_expired((t.record1.key, ts)) t.store.set_pending((t.record1.key, submitted)) t.store.add(t.record1) @@ -928,8 +957,9 @@ def test_add_overwrites_pending(t): def test_add_overwrites_building(t): t.store.add(t.record1) + ts = t.record1.updated + 300 started = t.record1.updated + 500 - t.store.set_expired(t.record1.key) + t.store.set_expired((t.record1.key, ts)) t.store.set_pending((t.record1.key, started - 100)) t.store.set_building((t.record1.key, started)) t.store.add(t.record1) From 00277c62939e5f8510265efa3204b60002bb65e4 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 27 Feb 2024 14:23:05 -0600 Subject: [PATCH 081/147] fix: force garbage collection ZEN-34712 --- Products/ZenCollector/configcache/invalidator.py | 5 ++++- Products/ZenCollector/configcache/manager.py | 2 ++ Products/ZenCollector/configcache/utils/pollers.py | 4 +--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 8721854982..92e62fca54 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -9,6 +9,7 @@ from __future__ import print_function, absolute_import +import gc import logging import time @@ -98,12 +99,14 @@ def run(self): self._synchronize() poller = RelStorageInvalidationPoller( - self.ctx.db.storage, self.ctx.session, self.ctx.dmd + self.ctx.db.storage, self.ctx.dmd ) self.log.info( "polling for device changes every %s seconds", self.interval ) while not self.ctx.controller.shutdown: + self.ctx.session.sync() + gc.collect() result = poller.poll() if result: self.log.debug("found %d relevant invalidations", len(result)) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 7cd7f3d811..0b2f311ff8 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -9,6 +9,7 @@ from __future__ import print_function +import gc import logging from datetime import datetime @@ -88,6 +89,7 @@ def run(self): while not self.ctx.controller.shutdown: try: self.ctx.session.sync() + gc.collect() self._retry_build() self._retry_pending() self._expire_retired_configs() diff --git a/Products/ZenCollector/configcache/utils/pollers.py b/Products/ZenCollector/configcache/utils/pollers.py index 3345b57499..db2501e41f 100644 --- a/Products/ZenCollector/configcache/utils/pollers.py +++ b/Products/ZenCollector/configcache/utils/pollers.py @@ -28,7 +28,7 @@ class RelStorageInvalidationPoller(object): API to return the latest database invalidations. """ - def __init__(self, storage, session, dmd): + def __init__(self, storage, dmd): """ Initialize a RelStorageInvalidationPoller instance. @@ -36,7 +36,6 @@ def __init__(self, storage, session, dmd): :type storage: :class:`relstorage.storage.RelStorage` """ self.__storage = storage - self.__session = session app = dmd.getPhysicalRoot() filters = initialize_invalidation_filters(dmd) self.__processor = InvalidationProcessor(app, filters) @@ -48,7 +47,6 @@ def poll(self): :rtype: Iterable[ZODB object] """ - self.__session.sync() oids = self.__storage.poll_invalidations() if not oids: return () From 10ab2fc5dd594785e09e03a6f7c83e425f329894 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 28 Feb 2024 09:51:53 -0600 Subject: [PATCH 082/147] fix: build_device_config skips duplicate jobs Fixing this issue also caused circular import issues, so the code in 'misc' and some code in 'utils' were moved into other locations to mitigate the occurrence of circular imports. Also updated set_expired calls with the correct arguments. ZEN-34698 ZEN-34735 --- .../configcache/{misc => app}/args.py | 0 .../ZenCollector/configcache/app/genconf.py | 2 +- Products/ZenCollector/configcache/cli.py | 6 +- .../ZenCollector/configcache/configcache.py | 2 +- .../configcache/{utils => }/constants.py | 0 .../ZenCollector/configcache/invalidator.py | 14 ++-- Products/ZenCollector/configcache/manager.py | 34 +++------ .../configcache/modelchange/filters.py | 2 +- .../configcache/modelchange/pipeline.py | 2 +- .../configcache/modelchange/processor.py | 3 +- .../__init__.py => modelchange/utils.py} | 13 ++-- .../configcache/{utils => }/propertymap.py | 0 Products/ZenCollector/configcache/task.py | 73 ++++++++++++++++++- .../configcache/tests/test_dispatcher.py | 4 +- .../configcache/tests/test_propertymap.py | 2 +- .../tests/test_propertymap_makers.py | 4 +- .../configcache/utils/__init__.py | 6 -- .../configcache/utils/dispatcher.py | 65 ----------------- Products/ZenCollector/configcache/version.py | 12 ++- .../migrate/addConfigCacheProperties.py | 2 +- 20 files changed, 119 insertions(+), 127 deletions(-) rename Products/ZenCollector/configcache/{misc => app}/args.py (100%) rename Products/ZenCollector/configcache/{utils => }/constants.py (100%) rename Products/ZenCollector/configcache/{misc/__init__.py => modelchange/utils.py} (75%) rename Products/ZenCollector/configcache/{utils => }/propertymap.py (100%) delete mode 100644 Products/ZenCollector/configcache/utils/dispatcher.py diff --git a/Products/ZenCollector/configcache/misc/args.py b/Products/ZenCollector/configcache/app/args.py similarity index 100% rename from Products/ZenCollector/configcache/misc/args.py rename to Products/ZenCollector/configcache/app/args.py diff --git a/Products/ZenCollector/configcache/app/genconf.py b/Products/ZenCollector/configcache/app/genconf.py index 9e5123619d..2f2b223f5d 100644 --- a/Products/ZenCollector/configcache/app/genconf.py +++ b/Products/ZenCollector/configcache/app/genconf.py @@ -2,7 +2,7 @@ import textwrap -from ..misc.args import get_subparser +from .args import get_subparser # List of options to not include when generating a config file. _ARGS_TO_IGNORE = ( diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index a73035e77c..73bce90a98 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -12,6 +12,7 @@ import argparse import os import sys +import time from datetime import datetime @@ -28,8 +29,8 @@ from Products.ZenUtils.terminal_size import get_terminal_size from .app import initialize_environment +from .app.args import get_subparser from .cache import ConfigQuery, ConfigStatus -from .misc.args import get_subparser class List_(object): @@ -319,7 +320,8 @@ def run(self): results = store.get_status(*store.search(query)) method = self._no_devices if not self._devices else self._with_devices keys = method(results) - store.set_expired(*keys) + now = time.time() + store.set_expired(*((key, now) for key in keys)) count = len(keys) print( "expired %d device configuration%s" diff --git a/Products/ZenCollector/configcache/configcache.py b/Products/ZenCollector/configcache/configcache.py index 53a061dffb..a77a8812f2 100644 --- a/Products/ZenCollector/configcache/configcache.py +++ b/Products/ZenCollector/configcache/configcache.py @@ -9,10 +9,10 @@ from __future__ import absolute_import, print_function +from .app.args import get_arg_parser from .cli import List_, Show, Expire from .invalidator import Invalidator from .manager import Manager -from .misc.args import get_arg_parser from .version import Version diff --git a/Products/ZenCollector/configcache/utils/constants.py b/Products/ZenCollector/configcache/constants.py similarity index 100% rename from Products/ZenCollector/configcache/utils/constants.py rename to Products/ZenCollector/configcache/constants.py diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 92e62fca54..c02293fb9a 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -23,16 +23,13 @@ from Products.Zuul.catalog.interfaces import IModelCatalogTool from .app import Application +from .app.args import get_subparser from .cache import ConfigQuery, ConfigStatus from .debug import Debug as DebugCommand -from .misc.args import get_subparser from .modelchange import InvalidationCause -from .utils import ( - BuildConfigTaskDispatcher, - DevicePropertyMap, - getConfigServices, - RelStorageInvalidationPoller, -) +from .propertymap import DevicePropertyMap +from .task import BuildConfigTaskDispatcher +from .utils import getConfigServices, RelStorageInvalidationPoller _default_interval = 30.0 @@ -200,7 +197,8 @@ def _updated_device(self, device, monitor, keys, invalidation): ) expired = set(key for key, _ in statuses if key not in retired) retired = self.store.set_retired(*retired) - expired = self.store.set_expired(*expired) + now = time.time() + expired = self.store.set_expired(*((key, now) for key in expired)) for key in retired: self.log.info( "retired configuration of changed device " diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 0b2f311ff8..b0d13c2cef 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -21,15 +21,13 @@ from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl from .app import Application +from .app.args import get_subparser from .cache import ConfigStatus +from .constants import Constants from .debug import Debug as DebugCommand -from .misc.args import get_subparser -from .utils import ( - BuildConfigTaskDispatcher, - Constants, - DevicePropertyMap, - getConfigServices, -) +from .propertymap import DevicePropertyMap +from .task import BuildConfigTaskDispatcher +from .utils import getConfigServices _default_interval = 30.0 # seconds @@ -109,13 +107,11 @@ def _retry_build(self): for key, status in self.store.get_building(): uid = self.store.get_uid(key.device) if uid is None: - self.log.warn( - "No UID found for device device=%s", key.device - ) + self.log.warn("No UID found for device device=%s", key.device) continue duration = buildlimitmap.get(uid) if status.started < (now - duration): - self.store.set_expired(key) + self.store.set_expired((key, now)) self.log.info( "expired configuration due to build timeout " "started=%s timeout=%s service=%s monitor=%s device=%s", @@ -140,13 +136,11 @@ def _retry_pending(self): for key, status in self.store.get_pending(): uid = self.store.get_uid(key.device) if uid is None: - self.log.warn( - "No UID found for device device=%s", key.device - ) + self.log.warn("No UID found for device device=%s", key.device) continue duration = pendinglimitmap.get(uid) if status.submitted < (now - duration): - self.store.set_expired(key) + self.store.set_expired((key, now)) self.log.info( "expired pending configuration build due to timeout " "submitted=%s timeout=%s service=%s monitor=%s device=%s", @@ -176,7 +170,7 @@ def _expire_retired_configs(self): for key, status, uid in retired if status.updated < now - minttl_map.get(uid) ) - self.store.set_expired(*expire) + self.store.set_expired(*((key, now) for key in expire)) def _rebuild_older_configs(self): buildlimitmap = DevicePropertyMap.make_build_timeout_map( @@ -184,9 +178,7 @@ def _rebuild_older_configs(self): ) ttlmap = DevicePropertyMap.make_ttl_map(self.ctx.dmd.Devices) min_ttl = ttlmap.smallest_value() - self.log.debug( - "minimum age limit is %s", _formatted_interval(min_ttl) - ) + self.log.debug("minimum age limit is %s", _formatted_interval(min_ttl)) now = time() min_age = now - min_ttl results = chain.from_iterable( @@ -196,9 +188,7 @@ def _rebuild_older_configs(self): for key, status in results: uid = self.store.get_uid(key.device) if uid is None: - self.log.warn( - "No UID found for device device=%s", key.device - ) + self.log.warn("No UID found for device device=%s", key.device) continue ttl = ttlmap.get(uid) expiration_threshold = now - ttl diff --git a/Products/ZenCollector/configcache/modelchange/filters.py b/Products/ZenCollector/configcache/modelchange/filters.py index 4b7e72bdc1..e752c4e345 100644 --- a/Products/ZenCollector/configcache/modelchange/filters.py +++ b/Products/ZenCollector/configcache/modelchange/filters.py @@ -34,7 +34,7 @@ from Products.ZenWidgets.Portlet import Portlet from Products.Zuul.catalog.interfaces import IModelCatalogTool -from ..utils import Constants +from ..constants import Constants log = logging.getLogger("zen.{}".format(__name__.split(".")[-1].lower())) diff --git a/Products/ZenCollector/configcache/modelchange/pipeline.py b/Products/ZenCollector/configcache/modelchange/pipeline.py index a1f4d6387f..2b8e63ed1f 100644 --- a/Products/ZenCollector/configcache/modelchange/pipeline.py +++ b/Products/ZenCollector/configcache/modelchange/pipeline.py @@ -11,7 +11,7 @@ import logging -from ..misc import coroutine, into_tuple +from .utils import coroutine, into_tuple log = logging.getLogger("zen.configcache.modelchange.pipeline") diff --git a/Products/ZenCollector/configcache/modelchange/processor.py b/Products/ZenCollector/configcache/modelchange/processor.py index 94d410d3e2..9128883adb 100644 --- a/Products/ZenCollector/configcache/modelchange/processor.py +++ b/Products/ZenCollector/configcache/modelchange/processor.py @@ -27,10 +27,9 @@ PrimaryPathObjectManager, ) -from ..misc import into_tuple - from .invalidation import Invalidation, InvalidationCause from .pipeline import Pipe, IterablePipe, Action +from .utils import into_tuple log = logging.getLogger("zen.configcache.modelchange") diff --git a/Products/ZenCollector/configcache/misc/__init__.py b/Products/ZenCollector/configcache/modelchange/utils.py similarity index 75% rename from Products/ZenCollector/configcache/misc/__init__.py rename to Products/ZenCollector/configcache/modelchange/utils.py index 5fa6665ad9..ff881d8339 100644 --- a/Products/ZenCollector/configcache/misc/__init__.py +++ b/Products/ZenCollector/configcache/modelchange/utils.py @@ -1,16 +1,18 @@ ############################################################################## # -# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. # ############################################################################## -import sys as _sys +from __future__ import absolute_import from functools import wraps as _wraps +import six as _six + def coroutine(func): """Decorator for initializing a generator as a coroutine.""" @@ -25,13 +27,8 @@ def start(*args, **kw): def into_tuple(args): - if isinstance(args, basestring): + if isinstance(args, _six.string_types): return (args,) elif not hasattr(args, "__iter__"): return (args,) return args - - -def app_name(): - fn = _sys.argv[0].rsplit("/", 1)[-1] - return fn.rsplit(".", 1)[0] if fn.endswith(".py") else fn diff --git a/Products/ZenCollector/configcache/utils/propertymap.py b/Products/ZenCollector/configcache/propertymap.py similarity index 100% rename from Products/ZenCollector/configcache/utils/propertymap.py rename to Products/ZenCollector/configcache/propertymap.py diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 433f1d7a40..9117f14df2 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -20,7 +20,8 @@ from Products.Jobber.task import requires, DMD, Abortable from Products.Jobber.zenjobs import app -from .cache import ConfigKey, ConfigQuery, ConfigRecord +from .cache import ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus +from .propertymap import DevicePropertyMap @app.task( @@ -47,6 +48,23 @@ def build_device_config(self, monitorname, deviceid, configclassname): svcconfigclass = resolve(configclassname) svcname = configclassname.rsplit(".", 1)[0] store = _getStore() + key = ConfigKey(svcname, monitorname, deviceid) + + # Check whether this is an old job, i.e. job pending timeout. + # If it is an old job, skip it, manager already sent another one. + statuses = tuple(store.get_status(key)) + if statuses: + key, status = statuses[0] + if isinstance(status, ConfigStatus.Pending): + pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( + self.dmd.Devices + ) + now = time() + uid = store.get_uid(key.device) + duration = pendinglimitmap.get(uid) + if status.submitted < (now - duration): + return + # Change the configuration's status from 'pending' to 'building' so # that configcache-manager doesn't prematurely timeout the build. store.set_building((ConfigKey(svcname, monitorname, deviceid), time())) @@ -101,6 +119,59 @@ def build_device_config(self, monitorname, deviceid, configclassname): ) +class BuildConfigTaskDispatcher(object): + """Encapsulates the act of dispatching the build_device_config task.""" + + def __init__(self, configClasses): + """ + Initialize a BuildConfigTaskDispatcher instance. + + The `configClasses` parameter should be the classes used to create + the device configurations. + + @type configClasses: Sequence[Class] + """ + self._classnames = { + cls.__module__: ".".join((cls.__module__, cls.__name__)) + for cls in configClasses + } + + def dispatch_all(self, monitorid, deviceid, timeout): + """ + Submit a task to build a device configuration from each + configuration service. + """ + soft_limit, hard_limit = _get_limits(timeout) + for name in self._classnames.values(): + build_device_config.apply_async( + args=(monitorid, deviceid, name), + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) + + def dispatch(self, servicename, monitorid, deviceid, timeout): + """ + Submit a task to build device configurations for the specified device. + + @type servicename: str + @type monitorid: str + @type deviceId: str + """ + name = self._classnames.get(servicename) + if name is None: + raise ValueError("service name '%s' not found" % servicename) + soft_limit, hard_limit = _get_limits(timeout) + build_device_config.apply_async( + args=(monitorid, deviceid, name), + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) + + +def _get_limits(timeout): + return timeout, (timeout + (timeout * 0.1)) + + def _getStore(): client = getRedisClient(url=getRedisUrl()) return createObject("configcache-store", client) diff --git a/Products/ZenCollector/configcache/tests/test_dispatcher.py b/Products/ZenCollector/configcache/tests/test_dispatcher.py index 13ac95abd1..e2eb79d86e 100644 --- a/Products/ZenCollector/configcache/tests/test_dispatcher.py +++ b/Products/ZenCollector/configcache/tests/test_dispatcher.py @@ -13,10 +13,10 @@ from mock import call, patch -from ..utils.dispatcher import BuildConfigTaskDispatcher, build_device_config +from ..task import BuildConfigTaskDispatcher, build_device_config -PATH = {"src": "Products.ZenCollector.configcache.utils.dispatcher"} +PATH = {"src": "Products.ZenCollector.configcache.task"} class BuildConfigTaskDispatcherTest(TestCase): diff --git a/Products/ZenCollector/configcache/tests/test_propertymap.py b/Products/ZenCollector/configcache/tests/test_propertymap.py index 51c16b7d68..9d6455455d 100644 --- a/Products/ZenCollector/configcache/tests/test_propertymap.py +++ b/Products/ZenCollector/configcache/tests/test_propertymap.py @@ -11,7 +11,7 @@ from unittest import TestCase -from ..utils.propertymap import DevicePropertyMap +from ..propertymap import DevicePropertyMap class EmptyDevicePropertyMapTest(TestCase): diff --git a/Products/ZenCollector/configcache/tests/test_propertymap_makers.py b/Products/ZenCollector/configcache/tests/test_propertymap_makers.py index 8717377fd1..f72f0d0fa0 100644 --- a/Products/ZenCollector/configcache/tests/test_propertymap_makers.py +++ b/Products/ZenCollector/configcache/tests/test_propertymap_makers.py @@ -11,8 +11,8 @@ from Products.ZenTestCase.BaseTestCase import BaseTestCase -from ..utils.constants import Constants -from ..utils.propertymap import DevicePropertyMap +from ..propertymap import DevicePropertyMap +from ..constants import Constants class TestDevicePropertyMapTTLMakers(BaseTestCase): diff --git a/Products/ZenCollector/configcache/utils/__init__.py b/Products/ZenCollector/configcache/utils/__init__.py index 00910dea16..b59560283b 100644 --- a/Products/ZenCollector/configcache/utils/__init__.py +++ b/Products/ZenCollector/configcache/utils/__init__.py @@ -9,17 +9,11 @@ from __future__ import absolute_import -from .constants import Constants -from .dispatcher import BuildConfigTaskDispatcher from .pollers import RelStorageInvalidationPoller -from .propertymap import DevicePropertyMap from .services import getConfigServices __all__ = ( - "BuildConfigTaskDispatcher", - "Constants", - "DevicePropertyMap", "getConfigServices", "RelStorageInvalidationPoller", ) diff --git a/Products/ZenCollector/configcache/utils/dispatcher.py b/Products/ZenCollector/configcache/utils/dispatcher.py deleted file mode 100644 index 1b864e2016..0000000000 --- a/Products/ZenCollector/configcache/utils/dispatcher.py +++ /dev/null @@ -1,65 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2023, all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -from __future__ import absolute_import - -from ..task import build_device_config - - -class BuildConfigTaskDispatcher(object): - """Encapsulates the act of dispatching the build_device_config task.""" - - def __init__(self, configClasses): - """ - Initialize a BuildConfigTaskDispatcher instance. - - The `configClasses` parameter should be the classes used to create - the device configurations. - - @type configClasses: Sequence[Class] - """ - self._classnames = { - cls.__module__: ".".join((cls.__module__, cls.__name__)) - for cls in configClasses - } - - def dispatch_all(self, monitorid, deviceid, timeout): - """ - Submit a task to build a device configuration from each - configuration service. - """ - soft_limit, hard_limit = _get_limits(timeout) - for name in self._classnames.values(): - build_device_config.apply_async( - args=(monitorid, deviceid, name), - soft_time_limit=soft_limit, - time_limit=hard_limit, - ) - - def dispatch(self, servicename, monitorid, deviceid, timeout): - """ - Submit a task to build device configurations for the specified device. - - @type servicename: str - @type monitorid: str - @type deviceId: str - """ - name = self._classnames.get(servicename) - if name is None: - raise ValueError("service name '%s' not found" % servicename) - soft_limit, hard_limit = _get_limits(timeout) - build_device_config.apply_async( - args=(monitorid, deviceid, name), - soft_time_limit=soft_limit, - time_limit=hard_limit, - ) - - -def _get_limits(timeout): - return timeout, (timeout + (timeout * 0.1)) diff --git a/Products/ZenCollector/configcache/version.py b/Products/ZenCollector/configcache/version.py index 1b488684a3..1604104a88 100644 --- a/Products/ZenCollector/configcache/version.py +++ b/Products/ZenCollector/configcache/version.py @@ -7,8 +7,9 @@ # ############################################################################## -from .misc import app_name -from .misc.args import get_subparser +import sys as _sys + +from .app.args import get_subparser class Version(object): @@ -30,4 +31,9 @@ def run(self): zinfo = ZenossInfo("") version = zinfo.getZenossVersion().short() - print("{} {}".format(app_name(), version)) + print("{} {}".format(_app_name(), version)) + + +def _app_name(): + fn = _sys.argv[0].rsplit("/", 1)[-1] + return fn.rsplit(".", 1)[0] if fn.endswith(".py") else fn diff --git a/Products/ZenModel/migrate/addConfigCacheProperties.py b/Products/ZenModel/migrate/addConfigCacheProperties.py index 4f31cdd6e5..0b4cdbfef7 100644 --- a/Products/ZenModel/migrate/addConfigCacheProperties.py +++ b/Products/ZenModel/migrate/addConfigCacheProperties.py @@ -9,7 +9,7 @@ from __future__ import absolute_import -from Products.ZenCollector.configcache.utils import Constants +from Products.ZenCollector.configcache.constants import Constants from Products.ZenRelations.zPropertyCategory import setzPropertyCategory from . import Migrate From 278ec93efdf3975b7cb73bb41ffcda2f5dd4df0b Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 28 Feb 2024 15:31:28 -0600 Subject: [PATCH 083/147] Add ConfigId class to configcache model. Also fix build_device_config to look at correct 'submitted' value. --- .../configcache/cache/__init__.py | 2 + .../ZenCollector/configcache/cache/model.py | 21 +-- .../ZenCollector/configcache/cache/storage.py | 87 ++++++--- Products/ZenCollector/configcache/cli.py | 8 +- .../ZenCollector/configcache/invalidator.py | 4 +- Products/ZenCollector/configcache/manager.py | 74 ++++---- Products/ZenCollector/configcache/task.py | 30 ++- .../configcache/tests/test_dispatcher.py | 13 +- .../configcache/tests/test_storage.py | 171 +++++++++--------- 9 files changed, 231 insertions(+), 179 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/__init__.py b/Products/ZenCollector/configcache/cache/__init__.py index 733d6a1805..8082ec8d0b 100644 --- a/Products/ZenCollector/configcache/cache/__init__.py +++ b/Products/ZenCollector/configcache/cache/__init__.py @@ -10,6 +10,7 @@ from __future__ import absolute_import from .model import ( + ConfigId, ConfigKey, ConfigQuery, ConfigRecord, @@ -17,6 +18,7 @@ ) __all__ = ( + "ConfigId", "ConfigKey", "ConfigQuery", "ConfigRecord", diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py index 8836b36e94..20ef08b6fe 100644 --- a/Products/ZenCollector/configcache/cache/model.py +++ b/Products/ZenCollector/configcache/cache/model.py @@ -9,8 +9,6 @@ from __future__ import absolute_import, print_function -import time - import attr from attr.validators import instance_of @@ -33,24 +31,9 @@ class ConfigKey(object): @attr.s(frozen=True, slots=True) -class ConfigPending(object): +class ConfigId(object): key = attr.ib(validator=[instance_of(ConfigKey)]) - started = attr.ib(validator=[instance_of(float)]) - - @classmethod - def make(cls, svc, mon, dev, started=None): - if started is None: - started = time.time() - return cls(ConfigKey(svc, mon, dev), started) - - @classmethod - def from_key(cls, key, started=None): - if started is None: - started = time.time() - return cls(key, started) - - def astuple(self): - return attr.astuple(self, recurse=False) + uid = attr.ib(validator=[instance_of(str)]) @attr.s(slots=True) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index c8d1fc5db3..8657f62211 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -73,7 +73,7 @@ from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl -from .model import ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus +from .model import ConfigId, ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus from .table import DeviceUIDTable, DeviceConfigTable, ConfigMetadataTable _app = "configcache" @@ -194,16 +194,14 @@ def get(self, key, default=None): @type key: ConfigKey @rtype: ConfigRecord """ - conf = self.__config.get( - self.__client, key.service, key.monitor, key.device - ) + with self.__client.pipeline() as pipe: + self.__config.get(pipe, key.service, key.monitor, key.device) + self.__age.score(pipe, key.service, key.monitor, key.device) + self.__uids.get(pipe, key.device) + conf, score, uid = pipe.execute() if conf is None: return default - score = self.__age.score( - self.__client, key.service, key.monitor, key.device - ) score = 0 if score < 0 else score - uid = self.__uids.get(self.__client, key.device) return _to_record( key.service, key.monitor, key.device, uid, score, conf ) @@ -236,8 +234,11 @@ def set_retired(self, *keys): """ Marks the indicated configuration as retired. - A configuration is retired when its `updated` field is less than the - difference between the current time and zDeviceConfigMinimumTTL. + The keys that were retired are returned to the caller. The returned + keys may match or be a subset of the keys that were passed in. + + If a config is already retired, it will not be among the keys + that are returned. @type keys: Sequence[ConfigKey] @rtype: Sequence[ConfigKey] @@ -281,8 +282,11 @@ def set_expired(self, *pairs): """ Marks the indicated configuration as expired. - Attempts to mark configurations that are not 'current' or - 'retired' are ignored. + The keys that were expired are returned to the caller. The returned + keys may match or be a subset of the keys that were passed in. + + If a config is already expired, it will not be among the keys + that are returned. @type keys: Sequence[(ConfigKey, float)] @rtype: Sequence[ConfigKey] @@ -316,7 +320,14 @@ def _impl(pipe): def set_pending(self, *pairs): """ - Marks an expired configuration as waiting for a new configuration. + Marks a configuration as waiting for a new configuration. + + The keys that were marked pending are returned to the caller. + The returned keys may match or be a subset of the keys that + were passed in. + + If a config is already marked pending, it will not be among the + keys that are returned. @type pending: Sequence[(ConfigKey, float)] @rtype: Sequence[ConfigKey] @@ -350,7 +361,14 @@ def _impl(pipe): def set_building(self, *pairs): """ - Marks a pending configuration as building a new configuration. + Marks a configuration as building a new configuration. + + The keys that were marked building are returned to the caller. + The returned keys may match or be a subset of the keys that + were passed in. + + If a config is already marked building, it will not be among the + keys that are returned. @type pairs: Sequence[(ConfigKey, float)] @rtype: Sequence[ConfigKey] @@ -394,22 +412,26 @@ def get_status(self, *keys): """ Returns an interable of (ConfigKey, ConfigStatus) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus]] + @rtype: Iterable[Tuple[ConfigId, ConfigStatus]] """ for key in keys: scores = self._get_scores(key) status = self._get_status(scores) + uid = self.__uids.get(self.__client, key.device) if status is not None: - yield (key, status) + yield (ConfigId(key, uid), status) def get_building(self, service="*", monitor="*"): """ Return an iterator producing (ConfigKey, ConfigStatus.Building) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Building]] + @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Building]] """ return ( - (key, ConfigStatus.Building(ts)) + ( + ConfigId(key, self.__uids.get(self.__client, key.device)), + ConfigStatus.Building(ts) + ) for key, ts in self.__range.building(service, monitor) ) @@ -417,10 +439,13 @@ def get_pending(self, service="*", monitor="*"): """ Return an iterator producing (ConfigKey, ConfigStatus.Pending) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Pending]] + @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Pending]] """ return ( - (key, ConfigStatus.Pending(ts)) + ( + ConfigId(key, self.__uids.get(self.__client, key.device)), + ConfigStatus.Pending(ts) + ) for key, ts in self.__range.pending(service, monitor) ) @@ -428,10 +453,13 @@ def get_expired(self, service="*", monitor="*"): """ Return an iterator producing (ConfigKey, ConfigStatus.Expired) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Expired]] + @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Expired]] """ return ( - (key, ConfigStatus.Expired(ts)) + ( + ConfigId(key, self.__uids.get(self.__client, key.device)), + ConfigStatus.Expired(ts) + ) for key, ts in self.__range.expired(service, monitor) ) @@ -439,10 +467,13 @@ def get_retired(self, service="*", monitor="*"): """ Return an iterator producing (ConfigKey, ConfigStatus.Retired) tuples. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Expired]] + @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Expired]] """ return ( - (key, ConfigStatus.Retired(ts)) + ( + ConfigId(key, self.__uids.get(self.__client, key.device)), + ConfigStatus.Retired(ts) + ) for key, ts in self.__range.retired(service, monitor) ) @@ -451,7 +482,7 @@ def get_older(self, maxtimestamp, service="*", monitor="*"): Returns an iterator producing (ConfigKey, ConfigStatus.Current) tuples where current timestamp <= `maxtimestamp`. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Current]] + @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Current]] """ # NOTE: 'older' means timestamps > 0 and <= `maxtimestamp`. selection = tuple( @@ -464,7 +495,8 @@ def get_older(self, maxtimestamp, service="*", monitor="*"): scores = self._get_scores(key)[1:] if any(score is not None for score in scores): continue - yield (key, ConfigStatus.Current(age)) + uid = self.__uids.get(self.__client, key.device) + yield (ConfigId(key, uid), ConfigStatus.Current(age)) def get_newer(self, mintimestamp, service="*", monitor="*"): """ @@ -484,7 +516,8 @@ def get_newer(self, mintimestamp, service="*", monitor="*"): scores = self._get_scores(key)[1:] if any(score is not None for score in scores): continue - yield (key, ConfigStatus.Current(age)) + uid = self.__uids.get(self.__client, key.device) + yield (ConfigId(key, uid), ConfigStatus.Current(age)) def _get_scores(self, key): service, monitor, device = attr.astuple(key) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 73bce90a98..2ea572feaa 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -221,8 +221,6 @@ def __init__(self, args): else: self._columns = args.width - - def run(self): initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) @@ -329,11 +327,13 @@ def run(self): ) def _no_devices(self, results): - return tuple(key for key, state in results) + return tuple(ident.key for ident, state in results) def _with_devices(self, results): return tuple( - key for key, state in results if key.device in self._devices + ident.key + for ident, state in results + if ident.key.device in self._devices ) def _confirm_inputs(self): diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index c02293fb9a..8aff3f20eb 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -184,8 +184,8 @@ def _updated_device(self, device, monitor, keys, invalidation): self.ctx.dmd.Devices ) statuses = tuple( - (key, status) - for key, status in self.store.get_status(*keys) + (ident.key, status) + for ident, status in self.store.get_status(*keys) if isinstance(status, ConfigStatus.Current) ) uid = device.getPrimaryId() diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index b0d13c2cef..235cab1aed 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -104,14 +104,15 @@ def _retry_build(self): # with builder working on the same config. now = time() - 600 count = 0 - for key, status in self.store.get_building(): - uid = self.store.get_uid(key.device) - if uid is None: - self.log.warn("No UID found for device device=%s", key.device) + for ident, status in self.store.get_building(): + if ident.uid is None: + self.log.warn( + "No UID found for device device=%s", ident.key.device + ) continue - duration = buildlimitmap.get(uid) + duration = buildlimitmap.get(ident.uid) if status.started < (now - duration): - self.store.set_expired((key, now)) + self.store.set_expired((ident.key, now)) self.log.info( "expired configuration due to build timeout " "started=%s timeout=%s service=%s monitor=%s device=%s", @@ -119,9 +120,9 @@ def _retry_build(self): "%Y-%m-%d %H:%M:%S" ), duration, - key.service, - key.monitor, - key.device, + ident.key.service, + ident.key.monitor, + ident.key.device, ) count += 1 if count == 0: @@ -133,14 +134,15 @@ def _retry_pending(self): ) now = time() count = 0 - for key, status in self.store.get_pending(): - uid = self.store.get_uid(key.device) - if uid is None: - self.log.warn("No UID found for device device=%s", key.device) + for ident, status in self.store.get_pending(): + if ident.uid is None: + self.log.warn( + "No UID found for device device=%s", ident.key.device + ) continue - duration = pendinglimitmap.get(uid) + duration = pendinglimitmap.get(ident.uid) if status.submitted < (now - duration): - self.store.set_expired((key, now)) + self.store.set_expired((ident.key, now)) self.log.info( "expired pending configuration build due to timeout " "submitted=%s timeout=%s service=%s monitor=%s device=%s", @@ -148,9 +150,9 @@ def _retry_pending(self): "%Y-%m-%d %H:%M:%S" ), duration, - key.service, - key.monitor, - key.device, + ident.key.service, + ident.key.monitor, + ident.key.device, ) count += 1 if count == 0: @@ -158,8 +160,8 @@ def _retry_pending(self): def _expire_retired_configs(self): retired = ( - (key, status, self.store.get_uid(key.device)) - for key, status in self.store.get_retired() + (ident.key, status, ident.uid) + for ident, status in self.store.get_retired() ) minttl_map = DevicePropertyMap.make_minimum_ttl_map( self.ctx.dmd.Devices @@ -185,29 +187,33 @@ def _rebuild_older_configs(self): (self.store.get_expired(), self.store.get_older(min_age)) ) count = 0 - for key, status in results: - uid = self.store.get_uid(key.device) - if uid is None: - self.log.warn("No UID found for device device=%s", key.device) + for ident, status in results: + if ident.uid is None: + self.log.warn( + "No UID found for device device=%s", ident.key.device + ) continue - ttl = ttlmap.get(uid) + ttl = ttlmap.get(ident.uid) expiration_threshold = now - ttl if ( isinstance(status, ConfigStatus.Expired) or status.updated <= expiration_threshold ): - timeout = buildlimitmap.get(uid) - self.store.set_pending((key, time())) + timeout = buildlimitmap.get(ident.uid) + self.store.set_pending((ident.key, time())) self.dispatcher.dispatch( - key.service, key.monitor, key.device, timeout + ident.key.service, + ident.key.monitor, + ident.key.device, + timeout, ) if isinstance(status, ConfigStatus.Expired): self.log.info( "submitted job to rebuild expired config " "service=%s monitor=%s device=%s", - key.service, - key.monitor, - key.device, + ident.key.service, + ident.key.monitor, + ident.key.device, ) else: self.log.info( @@ -218,9 +224,9 @@ def _rebuild_older_configs(self): ), Constants.time_to_live_id, ttl, - key.service, - key.monitor, - key.device, + ident.key.service, + ident.key.monitor, + ident.key.device, ) count += 1 if count == 0: diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 9117f14df2..929bdcd59d 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -21,6 +21,7 @@ from Products.Jobber.zenjobs import app from .cache import ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus +from .constants import Constants from .propertymap import DevicePropertyMap @@ -32,7 +33,9 @@ description_template="Create the configuration for device {2}.", ignore_result=True, ) -def build_device_config(self, monitorname, deviceid, configclassname): +def build_device_config( + self, monitorname, deviceid, configclassname, submitted=None +): """ Create a configuration for the given device. @@ -44,6 +47,8 @@ def build_device_config(self, monitorname, deviceid, configclassname): @param configclassname: The fully qualified name of the class that will create the device configuration. @type configclassname: str + @param submitted: timestamp of when the job was submitted + @type submitted: float """ svcconfigclass = resolve(configclassname) svcname = configclassname.rsplit(".", 1)[0] @@ -52,17 +57,25 @@ def build_device_config(self, monitorname, deviceid, configclassname): # Check whether this is an old job, i.e. job pending timeout. # If it is an old job, skip it, manager already sent another one. - statuses = tuple(store.get_status(key)) - if statuses: - key, status = statuses[0] + ident, status = next((store.get_status(key)), (None, None)) + if status is not None and submitted is not None: if isinstance(status, ConfigStatus.Pending): pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( self.dmd.Devices ) now = time() - uid = store.get_uid(key.device) - duration = pendinglimitmap.get(uid) - if status.submitted < (now - duration): + duration = pendinglimitmap.get(ident.uid) + if submitted < (now - duration): + self.log.warn( + "dropped this job in favor of newer job " + "device=%s monitor=%s service=%s submitted=%f %s=%s", + deviceid, + monitorname, + svcname, + submitted, + Constants.pending_timeout_id, + duration, + ) return # Change the configuration's status from 'pending' to 'building' so @@ -142,9 +155,11 @@ def dispatch_all(self, monitorid, deviceid, timeout): configuration service. """ soft_limit, hard_limit = _get_limits(timeout) + now = time() for name in self._classnames.values(): build_device_config.apply_async( args=(monitorid, deviceid, name), + kwargs={"submitted": now}, soft_time_limit=soft_limit, time_limit=hard_limit, ) @@ -163,6 +178,7 @@ def dispatch(self, servicename, monitorid, deviceid, timeout): soft_limit, hard_limit = _get_limits(timeout) build_device_config.apply_async( args=(monitorid, deviceid, name), + kwargs={"submitted": time()}, soft_time_limit=soft_limit, time_limit=hard_limit, ) diff --git a/Products/ZenCollector/configcache/tests/test_dispatcher.py b/Products/ZenCollector/configcache/tests/test_dispatcher.py index e2eb79d86e..10baf4eed9 100644 --- a/Products/ZenCollector/configcache/tests/test_dispatcher.py +++ b/Products/ZenCollector/configcache/tests/test_dispatcher.py @@ -34,11 +34,14 @@ def setUp(t): t.bctd = BuildConfigTaskDispatcher((t.class_a, t.class_b)) + @patch("{src}.time".format(**PATH), autospec=True) @patch.object(build_device_config, "apply_async") - def test_dispatch_all(t, _apply_async): + def test_dispatch_all(t, _apply_async, _time): timeout = 100.0 soft = 100.0 hard = 110.0 + submitted = 111.0 + _time.return_value = submitted monitor = "local" device = "linux" t.bctd.dispatch_all(monitor, device, timeout) @@ -47,22 +50,27 @@ def test_dispatch_all(t, _apply_async): ( call( args=(monitor, device, t.class_a_name), + kwargs={"submitted": submitted}, soft_time_limit=soft, time_limit=hard, ), call( args=(monitor, device, t.class_b_name), + kwargs={"submitted": submitted}, soft_time_limit=soft, time_limit=hard, ), ) ) + @patch("{src}.time".format(**PATH), autospec=True) @patch.object(build_device_config, "apply_async") - def test_dispatch(t, _apply_async): + def test_dispatch(t, _apply_async, _time): timeout = 100.0 soft = 100.0 hard = 110.0 + submitted = 111.0 + _time.return_value = submitted monitor = "local" device = "linux" svcname = t.class_a.__module__ @@ -70,6 +78,7 @@ def test_dispatch(t, _apply_async): _apply_async.assert_called_once_with( args=(monitor, device, t.class_a_name), + kwargs={"submitted": submitted}, soft_time_limit=soft, time_limit=hard, ) diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py index d96bf2ee77..f4919d9028 100644 --- a/Products/ZenCollector/configcache/tests/test_storage.py +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -18,6 +18,7 @@ from ..cache.storage import ConfigStore from ..cache import ( + ConfigId, ConfigKey, ConfigQuery, ConfigRecord, @@ -111,6 +112,7 @@ def setUp(t): t.fields[0].updated, t.config1, ) + t.cid1 = ConfigId(t.record1.key, t.record1.uid) t.record2 = ConfigRecord.make( t.fields[1].service, t.fields[1].monitor, @@ -119,6 +121,7 @@ def setUp(t): t.fields[1].updated, t.config2, ) + t.cid2 = ConfigId(t.record2.key, t.record2.uid) def tearDown(t): del t.store @@ -231,15 +234,15 @@ def test_get_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.fields[0].updated, status.updated) result = tuple(t.store.get_status(t.record2.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record2.key, key) + cid, status = result[0] + t.assertEqual(t.cid2, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.fields[1].updated, status.updated) @@ -260,17 +263,17 @@ def test_get_older_less_multiple(t): t.assertEqual(0, len(result)) result = tuple(t.store.get_older(t.record2.updated - 1)) - key, status = result[0] t.assertEqual(1, len(result)) - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertEqual(t.record1.updated, status.updated) def test_get_older_equal_single(t): t.store.add(t.record1) result = tuple(t.store.get_older(t.record1.updated)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -280,8 +283,8 @@ def test_get_older_equal_multiple(t): result = tuple(t.store.get_older(t.record1.updated)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -289,13 +292,13 @@ def test_get_older_equal_multiple(t): t.store.get_older(t.record2.updated), key=lambda x: x[1].updated ) t.assertEqual(2, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - key, status = result[1] - t.assertEqual(t.record2.key, key) + cid, status = result[1] + t.assertEqual(t.cid2, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -303,8 +306,8 @@ def test_get_older_greater_single(t): t.store.add(t.record1) result = tuple(t.store.get_older(t.record1.updated + 1)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -314,8 +317,8 @@ def test_get_older_greater_multiple(t): result = tuple(t.store.get_older(t.record1.updated + 1)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -324,13 +327,13 @@ def test_get_older_greater_multiple(t): key=lambda x: x[1].updated, ) t.assertEqual(2, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - key, status = result[1] - t.assertEqual(t.record2.key, key) + cid, status = result[1] + t.assertEqual(t.cid2, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -342,8 +345,8 @@ def test_get_newer_less_single(t): t.store.add(t.record1) result = tuple(t.store.get_newer(t.record1.updated - 1)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -356,12 +359,12 @@ def test_get_newer_less_multiple(t): key=lambda x: x[1].updated, ) t.assertEqual(2, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - key, status = result[1] - t.assertEqual(t.record2.key, key) + cid, status = result[1] + t.assertEqual(t.cid2, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -376,8 +379,8 @@ def test_get_newer_equal_multiple(t): result = tuple(t.store.get_newer(t.record1.updated)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record2.key, key) + cid, status = result[0] + t.assertEqual(t.cid2, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -391,8 +394,8 @@ def test_get_newer_greater_multiple(t): t.store.add(t.record2) result = tuple(t.store.get_newer(t.record1.updated + 1)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record2.key, key) + cid, status = result[0] + t.assertEqual(t.cid2, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -423,8 +426,8 @@ def test_retired_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(status.updated, t.record1.updated) @@ -434,8 +437,8 @@ def test_get_retired(t): result = tuple(t.store.get_retired()) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(status.updated, t.record1.updated) @@ -471,8 +474,8 @@ def test_expired_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Expired) def test_get_expired(t): @@ -482,8 +485,8 @@ def test_get_expired(t): result = tuple(t.store.get_expired()) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Expired) def test_expired_is_not_older(t): @@ -530,8 +533,8 @@ def test_pending_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -542,8 +545,8 @@ def test_get_pending(t): result = tuple(t.store.get_pending()) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -595,8 +598,8 @@ def test_building_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -607,8 +610,8 @@ def test_get_building(t): result = tuple(t.store.get_building()) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -636,8 +639,8 @@ def test_retired_to_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Expired) retired_keys = tuple(t.store.get_retired()) @@ -652,8 +655,8 @@ def test_expired_to_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(t.record1.updated, status.updated) @@ -672,8 +675,8 @@ def test_current_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -687,8 +690,8 @@ def test_retired_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -702,8 +705,8 @@ def test_expired_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Pending) expired_keys = tuple(t.store.get_expired()) @@ -726,8 +729,8 @@ def test_pending_to_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Expired) t.assertEqual(ts, status.expired) @@ -741,8 +744,8 @@ def test_pending_to_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(t.record1.updated, status.updated) @@ -761,8 +764,8 @@ def test_current_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -785,8 +788,8 @@ def test_retired_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -810,8 +813,8 @@ def test_expired_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -836,8 +839,8 @@ def test_pending_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -861,8 +864,8 @@ def test_building_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -877,8 +880,8 @@ def test_building_to_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Expired) t.assertEqual(expired, status.expired) @@ -892,8 +895,8 @@ def test_building_to_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(t.record1.updated, status.updated) @@ -913,8 +916,8 @@ def test_add_overwrites_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -929,8 +932,8 @@ def test_add_overwrites_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -950,8 +953,8 @@ def test_add_overwrites_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -975,8 +978,8 @@ def test_add_overwrites_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - key, status = result[0] - t.assertEqual(t.record1.key, key) + cid, status = result[0] + t.assertEqual(t.cid1, cid) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) From 369f5046c0aca65916fdc5ba1ac89833ba6f52c4 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 29 Feb 2024 09:28:56 -0600 Subject: [PATCH 084/147] Simplified the config cache storage API. ConfigKey, ConfigQuery, and ConfigRecord renamed to CacheKey, CacheQuery, and CacheRecord. The ConfigStatus classes now include key and uid attributes. The ConfigStore's get_* methods (except for get_uid) are modified to return only ConfigStatus class instances. --- .../configcache/cache/__init__.py | 16 +- .../ZenCollector/configcache/cache/model.py | 90 +++---- .../ZenCollector/configcache/cache/storage.py | 118 +++++----- Products/ZenCollector/configcache/cli.py | 46 ++-- .../ZenCollector/configcache/invalidator.py | 16 +- Products/ZenCollector/configcache/manager.py | 74 +++--- Products/ZenCollector/configcache/task.py | 14 +- .../configcache/tests/test_storage.py | 222 +++++++++--------- Products/ZenCollector/services/ConfigCache.py | 12 +- 9 files changed, 277 insertions(+), 331 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/__init__.py b/Products/ZenCollector/configcache/cache/__init__.py index 8082ec8d0b..ce41b3d5c4 100644 --- a/Products/ZenCollector/configcache/cache/__init__.py +++ b/Products/ZenCollector/configcache/cache/__init__.py @@ -9,18 +9,6 @@ from __future__ import absolute_import -from .model import ( - ConfigId, - ConfigKey, - ConfigQuery, - ConfigRecord, - ConfigStatus, -) +from .model import CacheKey, CacheQuery, CacheRecord, ConfigStatus -__all__ = ( - "ConfigId", - "ConfigKey", - "ConfigQuery", - "ConfigRecord", - "ConfigStatus", -) +__all__ = ("CacheKey", "CacheQuery", "CacheRecord", "ConfigStatus") diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py index 20ef08b6fe..e3e98c4060 100644 --- a/Products/ZenCollector/configcache/cache/model.py +++ b/Products/ZenCollector/configcache/cache/model.py @@ -17,29 +17,23 @@ @attr.s(frozen=True, slots=True) -class ConfigQuery(object): +class CacheQuery(object): service = attr.ib(validator=[instance_of(str)], default="*") monitor = attr.ib(validator=[instance_of(str)], default="*") device = attr.ib(validator=[instance_of(str)], default="*") @attr.s(frozen=True, slots=True) -class ConfigKey(object): +class CacheKey(object): service = attr.ib(validator=[instance_of(str)]) monitor = attr.ib(validator=[instance_of(str)]) device = attr.ib(validator=[instance_of(str)]) -@attr.s(frozen=True, slots=True) -class ConfigId(object): - key = attr.ib(validator=[instance_of(ConfigKey)]) - uid = attr.ib(validator=[instance_of(str)]) - - @attr.s(slots=True) -class ConfigRecord(object): +class CacheRecord(object): key = attr.ib( - validator=[instance_of(ConfigKey)], on_setattr=attr.setters.NO_OP + validator=[instance_of(CacheKey)], on_setattr=attr.setters.NO_OP ) uid = attr.ib(validator=[instance_of(str)], on_setattr=attr.setters.NO_OP) updated = attr.ib(validator=[instance_of(float)]) @@ -47,7 +41,7 @@ class ConfigRecord(object): @classmethod def make(cls, svc, mon, dev, uid, updated, config): - return cls(ConfigKey(svc, mon, dev), uid, updated, config) + return cls(CacheKey(svc, mon, dev), uid, updated, config) @property def service(self): @@ -62,74 +56,58 @@ def device(self): return self.key.device +@attr.s(slots=True) +class _Status(object): + """Base class for status classes.""" + + key = attr.ib(validator=[instance_of(CacheKey)]) + uid = attr.ib(validator=[instance_of(str)]) + + class _ConfigStatus(object): """ - Namespace class for Current, Building, Expired, and Pending types. + Namespace class for Current, Retired, Expired, Pending, and Building types. """ - class Current(object): + @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") + class Current(_Status): """The configuration is current.""" - def __init__(self, ts): - self.updated = ts - - def __eq__(self, other): - if not isinstance(other, _ConfigStatus.Current): - return NotImplemented - return self.updated == other.updated + updated = attr.ib(validator=[instance_of(float)]) - class Retired(object): + @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") + class Retired(_Status): """The cofiguration is retired, but not yet expired.""" - def __init__(self, ts): - self.updated = ts + updated = attr.ib(validator=[instance_of(float)]) - def __eq__(self, other): - if not isinstance(other, _ConfigStatus.Retired): - return NotImplemented - return self.updated == other.updated - - class Expired(object): + @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") + class Expired(_Status): """The configuration has expired.""" - def __init__(self, ts): - self.expired = ts - - def __eq__(self, other): - if not isinstance(other, _ConfigStatus.Expired): - return NotImplemented - return True + expired = attr.ib(validator=[instance_of(float)]) - class Pending(object): + @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") + class Pending(_Status): """The configuration is waiting for a rebuild.""" - def __init__(self, ts): - self.submitted = ts + submitted = attr.ib(validator=[instance_of(float)]) - def __eq__(self, other): - if not isinstance(other, _ConfigStatus.Pending): - return NotImplemented - return self.submitted == other.submitted - - class Building(object): + @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") + class Building(_Status): """The configuration is rebuilding.""" - def __init__(self, ts): - self.started = ts - - def __eq__(self, other): - if not isinstance(other, _ConfigStatus.Building): - return NotImplemented - return self.started == other.started + started = attr.ib(validator=[instance_of(float)]) def __contains__(self, value): return isinstance( value, ( - _ConfigStatus.Building, _ConfigStatus.Current, + _ConfigStatus.Retired, _ConfigStatus.Expired, _ConfigStatus.Pending, + _ConfigStatus.Building, ), ) @@ -137,8 +115,8 @@ def __contains__(self, value): ConfigStatus = _ConfigStatus() __all__ = ( - "ConfigKey", - "ConfigQuery", - "ConfigRecord", + "CacheKey", + "CacheQuery", + "CacheRecord", "ConfigStatus", ) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 8657f62211..cd5d6e20df 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -73,7 +73,7 @@ from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl -from .model import ConfigId, ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus +from .model import CacheKey, CacheQuery, CacheRecord, ConfigStatus from .table import DeviceUIDTable, DeviceConfigTable, ConfigMetadataTable _app = "configcache" @@ -124,19 +124,19 @@ def __init__(self, client): }, )() - def search(self, query=ConfigQuery()): + def search(self, query=CacheQuery()): """ Returns the configuration keys matching the search criteria. - @type query: ConfigQuery - @rtype: Iterator[ConfigKey] + @type query: CacheQuery + @rtype: Iterator[CacheKey] @raises TypeError: Unsupported value given for a field @raises AttributeError: Unknown field """ - if not isinstance(query, ConfigQuery): - raise TypeError("'{!r} is not a ConfigQuery".format(query)) + if not isinstance(query, CacheQuery): + raise TypeError("'{!r} is not a CacheQuery".format(query)) return ( - ConfigKey(svc, mon, dvc) + CacheKey(svc, mon, dvc) for svc, mon, dvc in self.__config.scan( self.__client, **attr.asdict(query) ) @@ -144,13 +144,13 @@ def search(self, query=ConfigQuery()): def add(self, record): """ - @type record: ConfigRecord + @type record: CacheRecord """ svc, mon, dvc, uid, updated, config = _from_record(record) orphaned_keys = tuple( key - for key in self.search(ConfigQuery(service=svc, device=dvc)) + for key in self.search(CacheQuery(service=svc, device=dvc)) if key.monitor != mon ) watch_keys = self._get_watch_keys(orphaned_keys + (record.key,)) @@ -191,8 +191,8 @@ def get_uid(self, device): def get(self, key, default=None): """ - @type key: ConfigKey - @rtype: ConfigRecord + @type key: CacheKey + @rtype: CacheRecord """ with self.__client.pipeline() as pipe: self.__config.get(pipe, key.service, key.monitor, key.device) @@ -210,7 +210,7 @@ def remove(self, *keys): """ Delete the configurations identified by `keys`. - @type keys: Sequence[ConfigKey] + @type keys: Sequence[CacheKey] """ with self.__client.pipeline() as pipe: for key in keys: @@ -240,8 +240,8 @@ def set_retired(self, *keys): If a config is already retired, it will not be among the keys that are returned. - @type keys: Sequence[ConfigKey] - @rtype: Sequence[ConfigKey] + @type keys: Sequence[CacheKey] + @rtype: Sequence[CacheKey] """ if len(keys) == 0: return () @@ -288,8 +288,8 @@ def set_expired(self, *pairs): If a config is already expired, it will not be among the keys that are returned. - @type keys: Sequence[(ConfigKey, float)] - @rtype: Sequence[ConfigKey] + @type keys: Sequence[(CacheKey, float)] + @rtype: Sequence[CacheKey] """ if len(pairs) == 0: return () @@ -329,8 +329,8 @@ def set_pending(self, *pairs): If a config is already marked pending, it will not be among the keys that are returned. - @type pending: Sequence[(ConfigKey, float)] - @rtype: Sequence[ConfigKey] + @type pending: Sequence[(CacheKey, float)] + @rtype: Sequence[CacheKey] """ if len(pairs) == 0: return () @@ -370,8 +370,8 @@ def set_building(self, *pairs): If a config is already marked building, it will not be among the keys that are returned. - @type pairs: Sequence[(ConfigKey, float)] - @rtype: Sequence[ConfigKey] + @type pairs: Sequence[(CacheKey, float)] + @rtype: Sequence[CacheKey] """ if len(pairs) == 0: return () @@ -410,79 +410,75 @@ def _filter_existing(self, table, items, getkey): def get_status(self, *keys): """ - Returns an interable of (ConfigKey, ConfigStatus) tuples. + Returns an interable of ConfigStatus objects. - @rtype: Iterable[Tuple[ConfigId, ConfigStatus]] + @rtype: Iterable[ConfigStatus] """ for key in keys: scores = self._get_scores(key) - status = self._get_status(scores) uid = self.__uids.get(self.__client, key.device) + status = self._get_status(scores, key, uid) if status is not None: - yield (ConfigId(key, uid), status) + yield status def get_building(self, service="*", monitor="*"): """ - Return an iterator producing (ConfigKey, ConfigStatus.Building) tuples. + Return an iterator producing ConfigStatus.Building objects. - @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Building]] + @rtype: Iterable[ConfigStatus.Building] """ return ( - ( - ConfigId(key, self.__uids.get(self.__client, key.device)), - ConfigStatus.Building(ts) + ConfigStatus.Building( + key, self.__uids.get(self.__client, key.device), ts ) for key, ts in self.__range.building(service, monitor) ) def get_pending(self, service="*", monitor="*"): """ - Return an iterator producing (ConfigKey, ConfigStatus.Pending) tuples. + Return an iterator producing ConfigStatus.Pending objects. - @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Pending]] + @rtype: Iterable[ConfigStatus.Pending] """ return ( - ( - ConfigId(key, self.__uids.get(self.__client, key.device)), - ConfigStatus.Pending(ts) + ConfigStatus.Pending( + key, self.__uids.get(self.__client, key.device), ts ) for key, ts in self.__range.pending(service, monitor) ) def get_expired(self, service="*", monitor="*"): """ - Return an iterator producing (ConfigKey, ConfigStatus.Expired) tuples. + Return an iterator producing ConfigStatus.Expired objects. - @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Expired]] + @rtype: Iterable[ConfigStatus.Expired] """ return ( - ( - ConfigId(key, self.__uids.get(self.__client, key.device)), - ConfigStatus.Expired(ts) + ConfigStatus.Expired( + key, self.__uids.get(self.__client, key.device), ts ) for key, ts in self.__range.expired(service, monitor) ) def get_retired(self, service="*", monitor="*"): """ - Return an iterator producing (ConfigKey, ConfigStatus.Retired) tuples. + Return an iterator producing ConfigStatus.Retired objects. - @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Expired]] + @rtype: Iterable[ConfigStatus.Retired] """ return ( - ( - ConfigId(key, self.__uids.get(self.__client, key.device)), - ConfigStatus.Retired(ts) + ConfigStatus.Retired( + key, self.__uids.get(self.__client, key.device), ts ) for key, ts in self.__range.retired(service, monitor) ) def get_older(self, maxtimestamp, service="*", monitor="*"): """ - Returns an iterator producing (ConfigKey, ConfigStatus.Current) - tuples where current timestamp <= `maxtimestamp`. + Returns an iterator producing ConfigStatus.Current objects + where current timestamp <= `maxtimestamp`. - @rtype: Iterable[Tuple[ConfigId, ConfigStatus.Current]] + @rtype: Iterable[ConfigStatus.Current] """ # NOTE: 'older' means timestamps > 0 and <= `maxtimestamp`. selection = tuple( @@ -496,14 +492,14 @@ def get_older(self, maxtimestamp, service="*", monitor="*"): if any(score is not None for score in scores): continue uid = self.__uids.get(self.__client, key.device) - yield (ConfigId(key, uid), ConfigStatus.Current(age)) + yield ConfigStatus.Current(key, uid, age) def get_newer(self, mintimestamp, service="*", monitor="*"): """ - Returns an iterator producing (ConfigKey, ConfigStatus.Current) - tuples where current timestamp > `mintimestamp`. + Returns an iterator producing ConfigStatus.Current objects + where current timestamp > `mintimestamp`. - @rtype: Iterable[Tuple[ConfigKey, ConfigStatus.Current]] + @rtype: Iterable[ConfigStatus.Current] """ # NOTE: 'newer' means timestamps to `maxtimestamp`. selection = tuple( @@ -517,7 +513,7 @@ def get_newer(self, mintimestamp, service="*", monitor="*"): if any(score is not None for score in scores): continue uid = self.__uids.get(self.__client, key.device) - yield (ConfigId(key, uid), ConfigStatus.Current(age)) + yield ConfigStatus.Current(key, uid, age) def _get_scores(self, key): service, monitor, device = attr.astuple(key) @@ -529,18 +525,18 @@ def _get_scores(self, key): self.__building.score(pipe, service, monitor, device), return pipe.execute() - def _get_status(self, scores): + def _get_status(self, scores, key, uid): age, retired, expired, pending, building = scores if building is not None: - return ConfigStatus.Building(_to_ts(building)) + return ConfigStatus.Building(key, uid, _to_ts(building)) elif pending is not None: - return ConfigStatus.Pending(_to_ts(pending)) + return ConfigStatus.Pending(key, uid, _to_ts(pending)) elif expired is not None: - return ConfigStatus.Expired(_to_ts(expired)) + return ConfigStatus.Expired(key, uid, _to_ts(expired)) elif retired is not None: - return ConfigStatus.Retired(_to_ts(retired)) + return ConfigStatus.Retired(key, uid, _to_ts(retired)) elif age is not None: - return ConfigStatus.Current(_to_ts(age)) + return ConfigStatus.Current(key, uid, _to_ts(age)) def _get_watch_keys(self, keys): return set( @@ -560,7 +556,7 @@ def _get_watch_keys(self, keys): def _range(client, metadata, svc, mon, minv=None, maxv=None): pairs = metadata.get_pairs(client, svc, mon) return ( - (ConfigKey(svcId, monId, devId), _to_ts(score)) + (CacheKey(svcId, monId, devId), _to_ts(score)) for svcId, monId, devId, score in metadata.range( client, pairs, minscore=minv, maxscore=maxv ) @@ -580,10 +576,10 @@ def _to_ts(score): def _to_record(svc, mon, dvc, uid, updated, config): - key = ConfigKey(svc, mon, dvc) + key = CacheKey(svc, mon, dvc) updated = _to_ts(updated) config = _unjelly(config) - return ConfigRecord(key, uid, updated, config) + return CacheRecord(key, uid, updated, config) def _from_record(record): diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 2ea572feaa..47862591aa 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -30,7 +30,7 @@ from .app import initialize_environment from .app.args import get_subparser -from .cache import ConfigQuery, ConfigStatus +from .cache import CacheQuery, ConfigStatus class List_(object): @@ -91,38 +91,44 @@ def run(self): client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) if haswildcard: - query = ConfigQuery( + query = CacheQuery( service=self._service, monitor=self._monitor, device=self._devices[0], ) else: - query = ConfigQuery(service=self._service, monitor=self._monitor) + query = CacheQuery(service=self._service, monitor=self._monitor) results = store.get_status(*store.search(query)) if self._states: results = ( - (key, status) - for key, status in results + status + for status in results if isinstance(status, self._states) ) rows = [] maxd, maxs, maxm = 0, 0, 0 if len(self._devices) > 0: - data = (key for key in results if key[0].device in self._devices) + data = ( + status + for status in results + if status.key.device in self._devices + ) else: data = results - for key, status in sorted( - data, key=lambda x: (x[0].device, x[0].service) + for status in sorted( + data, key=lambda x: (x.key.device, x.key.service) ): if self._showuid: - uid = store.get_uid(key.device) + devid = status.uid else: - uid = key.device + devid = status.key.device status_text = _format_status(status) - maxd = max(maxd, len(uid)) + maxd = max(maxd, len(devid)) maxs = max(maxs, len(status_text)) - maxm = max(maxm, len(key.monitor)) - rows.append((uid, status_text, key.monitor, key.service)) + maxm = max(maxm, len(status.key.monitor)) + rows.append( + (devid, status_text, status.key.monitor, status.key.service) + ) if rows: print( "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( @@ -160,7 +166,7 @@ def run(self): def _format_status(status): if isinstance(status, ConfigStatus.Current): - return "last updated {}".format(_format_date(status.updated)) + return "current since {}".format(_format_date(status.updated)) elif isinstance(status, ConfigStatus.Retired): return "retired" elif isinstance(status, ConfigStatus.Expired): @@ -243,7 +249,7 @@ def run(self): def _query_cache(store, service, monitor, device): - query = ConfigQuery(service=service, monitor=monitor, device=device) + query = CacheQuery(service=service, monitor=monitor, device=device) results = store.search(query) first_key = next(results, None) if first_key is None: @@ -314,7 +320,7 @@ def run(self): initialize_environment(configs=self.configs, useZope=False) client = getRedisClient(url=getRedisUrl()) store = createObject("configcache-store", client) - query = ConfigQuery(service=self._service, monitor=self._monitor) + query = CacheQuery(service=self._service, monitor=self._monitor) results = store.get_status(*store.search(query)) method = self._no_devices if not self._devices else self._with_devices keys = method(results) @@ -327,13 +333,13 @@ def run(self): ) def _no_devices(self, results): - return tuple(ident.key for ident, state in results) + return tuple(status.key for status in results) def _with_devices(self, results): return tuple( - ident.key - for ident, state in results - if ident.key.device in self._devices + status.key + for status in results + if status.key.device in self._devices ) def _confirm_inputs(self): diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 8aff3f20eb..0bdbd9d024 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -24,7 +24,7 @@ from .app import Application from .app.args import get_subparser -from .cache import ConfigQuery, ConfigStatus +from .cache import CacheQuery, ConfigStatus from .debug import Debug as DebugCommand from .modelchange import InvalidationCause from .propertymap import DevicePropertyMap @@ -148,7 +148,7 @@ def _process(self, invalidation): ) return keys = list( - self.store.search(ConfigQuery(monitor=monitor, device=device.id)) + self.store.search(CacheQuery(monitor=monitor, device=device.id)) ) if not keys: self._new_device(device, monitor) @@ -184,8 +184,8 @@ def _updated_device(self, device, monitor, keys, invalidation): self.ctx.dmd.Devices ) statuses = tuple( - (ident.key, status) - for ident, status in self.store.get_status(*keys) + status + for status in self.store.get_status(*keys) if isinstance(status, ConfigStatus.Current) ) uid = device.getPrimaryId() @@ -193,9 +193,11 @@ def _updated_device(self, device, monitor, keys, invalidation): now = time.time() limit = now - minttl retired = set( - key for key, status in statuses if status.updated >= limit + status.key for status in statuses if status.updated >= limit + ) + expired = set( + status.key for status in statuses if status.key not in retired ) - expired = set(key for key, _ in statuses if key not in retired) retired = self.store.set_retired(*retired) now = time.time() expired = self.store.set_expired(*((key, now) for key in expired)) @@ -279,7 +281,7 @@ def _addNew(log, tool, timelimitmap, store, dispatcher): ) continue keys = tuple( - store.search(ConfigQuery(monitor=brain.collector, device=brain.id)) + store.search(CacheQuery(monitor=brain.collector, device=brain.id)) ) if not keys: timeout = timelimitmap.get(brain.uid) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 235cab1aed..f9c026c93f 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -104,15 +104,10 @@ def _retry_build(self): # with builder working on the same config. now = time() - 600 count = 0 - for ident, status in self.store.get_building(): - if ident.uid is None: - self.log.warn( - "No UID found for device device=%s", ident.key.device - ) - continue - duration = buildlimitmap.get(ident.uid) + for status in self.store.get_building(): + duration = buildlimitmap.get(status.uid) if status.started < (now - duration): - self.store.set_expired((ident.key, now)) + self.store.set_expired((status.key, now)) self.log.info( "expired configuration due to build timeout " "started=%s timeout=%s service=%s monitor=%s device=%s", @@ -120,9 +115,9 @@ def _retry_build(self): "%Y-%m-%d %H:%M:%S" ), duration, - ident.key.service, - ident.key.monitor, - ident.key.device, + status.key.service, + status.key.monitor, + status.key.device, ) count += 1 if count == 0: @@ -134,15 +129,10 @@ def _retry_pending(self): ) now = time() count = 0 - for ident, status in self.store.get_pending(): - if ident.uid is None: - self.log.warn( - "No UID found for device device=%s", ident.key.device - ) - continue - duration = pendinglimitmap.get(ident.uid) + for status in self.store.get_pending(): + duration = pendinglimitmap.get(status.uid) if status.submitted < (now - duration): - self.store.set_expired((ident.key, now)) + self.store.set_expired((status.key, now)) self.log.info( "expired pending configuration build due to timeout " "submitted=%s timeout=%s service=%s monitor=%s device=%s", @@ -150,27 +140,23 @@ def _retry_pending(self): "%Y-%m-%d %H:%M:%S" ), duration, - ident.key.service, - ident.key.monitor, - ident.key.device, + status.key.service, + status.key.monitor, + status.key.device, ) count += 1 if count == 0: self.log.debug("no pending configuration builds have timed out") def _expire_retired_configs(self): - retired = ( - (ident.key, status, ident.uid) - for ident, status in self.store.get_retired() - ) minttl_map = DevicePropertyMap.make_minimum_ttl_map( self.ctx.dmd.Devices ) now = time() expire = tuple( - key - for key, status, uid in retired - if status.updated < now - minttl_map.get(uid) + status.key + for status in self.store.get_retired() + if status.updated < now - minttl_map.get(status.uid) ) self.store.set_expired(*((key, now) for key in expire)) @@ -187,33 +173,33 @@ def _rebuild_older_configs(self): (self.store.get_expired(), self.store.get_older(min_age)) ) count = 0 - for ident, status in results: - if ident.uid is None: + for status in results: + if status.uid is None: self.log.warn( - "No UID found for device device=%s", ident.key.device + "No UID found for device device=%s", status.key.device ) continue - ttl = ttlmap.get(ident.uid) + ttl = ttlmap.get(status.uid) expiration_threshold = now - ttl if ( isinstance(status, ConfigStatus.Expired) or status.updated <= expiration_threshold ): - timeout = buildlimitmap.get(ident.uid) - self.store.set_pending((ident.key, time())) + timeout = buildlimitmap.get(status.uid) + self.store.set_pending((status.key, time())) self.dispatcher.dispatch( - ident.key.service, - ident.key.monitor, - ident.key.device, + status.key.service, + status.key.monitor, + status.key.device, timeout, ) if isinstance(status, ConfigStatus.Expired): self.log.info( "submitted job to rebuild expired config " "service=%s monitor=%s device=%s", - ident.key.service, - ident.key.monitor, - ident.key.device, + status.key.service, + status.key.monitor, + status.key.device, ) else: self.log.info( @@ -224,9 +210,9 @@ def _rebuild_older_configs(self): ), Constants.time_to_live_id, ttl, - ident.key.service, - ident.key.monitor, - ident.key.device, + status.key.service, + status.key.monitor, + status.key.device, ) count += 1 if count == 0: diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 929bdcd59d..78fe10f3c6 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -20,7 +20,7 @@ from Products.Jobber.task import requires, DMD, Abortable from Products.Jobber.zenjobs import app -from .cache import ConfigKey, ConfigQuery, ConfigRecord, ConfigStatus +from .cache import CacheKey, CacheQuery, CacheRecord, ConfigStatus from .constants import Constants from .propertymap import DevicePropertyMap @@ -53,18 +53,18 @@ def build_device_config( svcconfigclass = resolve(configclassname) svcname = configclassname.rsplit(".", 1)[0] store = _getStore() - key = ConfigKey(svcname, monitorname, deviceid) + key = CacheKey(svcname, monitorname, deviceid) # Check whether this is an old job, i.e. job pending timeout. # If it is an old job, skip it, manager already sent another one. - ident, status = next((store.get_status(key)), (None, None)) + status = next(store.get_status(key), None) if status is not None and submitted is not None: if isinstance(status, ConfigStatus.Pending): pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( self.dmd.Devices ) now = time() - duration = pendinglimitmap.get(ident.uid) + duration = pendinglimitmap.get(status.uid) if submitted < (now - duration): self.log.warn( "dropped this job in favor of newer job " @@ -80,7 +80,7 @@ def build_device_config( # Change the configuration's status from 'pending' to 'building' so # that configcache-manager doesn't prematurely timeout the build. - store.set_building((ConfigKey(svcname, monitorname, deviceid), time())) + store.set_building((CacheKey(svcname, monitorname, deviceid), time())) self.log.info( "building device configuration device=%s monitor=%s service=%s", deviceid, @@ -99,7 +99,7 @@ def build_device_config( ) key = next( store.search( - ConfigQuery( + CacheQuery( service=svcname, monitor=monitorname, device=deviceid ) ), @@ -118,7 +118,7 @@ def build_device_config( else: config = configs[0] uid = self.dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() - record = ConfigRecord.make( + record = CacheRecord.make( svcname, monitorname, deviceid, uid, time(), config ) store.add(record) diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py index f4919d9028..081362d0db 100644 --- a/Products/ZenCollector/configcache/tests/test_storage.py +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -16,14 +16,8 @@ from Products.ZenCollector.services.config import DeviceProxy from Products.Jobber.tests.utils import subTest, RedisLayer +from ..cache import CacheKey, CacheQuery, CacheRecord, ConfigStatus from ..cache.storage import ConfigStore -from ..cache import ( - ConfigId, - ConfigKey, - ConfigQuery, - ConfigRecord, - ConfigStatus, -) _fields = collections.namedtuple( @@ -47,11 +41,11 @@ def test_search(t): t.assertTupleEqual(tuple(t.store.search()), ()) def test_get_with_default_default(t): - key = ConfigKey("a", "b", "c") + key = CacheKey("a", "b", "c") t.assertIsNone(t.store.get(key)) def test_get_with_nondefault_default(t): - key = ConfigKey("a", "b", "c") + key = CacheKey("a", "b", "c") dflt = object() t.assertEqual(t.store.get(key, dflt), dflt) @@ -64,7 +58,7 @@ def test_get_status_no_keys(t): t.assertTupleEqual(tuple(result), ()) def test_get_status_unknown_key(t): - key = ConfigKey("a", "b", "c") + key = CacheKey("a", "b", "c") result = t.store.get_status(key) t.assertIsInstance(result, collections.Iterable) t.assertTupleEqual(tuple(result), ()) @@ -104,7 +98,7 @@ def setUp(t): t.store = ConfigStore(t.layer.redis) t.config1 = _make_config("test1", "_test1", "abc-test-01") t.config2 = _make_config("test2", "_test2", "abc-test-02") - t.record1 = ConfigRecord.make( + t.record1 = CacheRecord.make( t.fields[0].service, t.fields[0].monitor, t.fields[0].device, @@ -112,8 +106,7 @@ def setUp(t): t.fields[0].updated, t.config1, ) - t.cid1 = ConfigId(t.record1.key, t.record1.uid) - t.record2 = ConfigRecord.make( + t.record2 = CacheRecord.make( t.fields[1].service, t.fields[1].monitor, t.fields[1].device, @@ -121,7 +114,6 @@ def setUp(t): t.fields[1].updated, t.config2, ) - t.cid2 = ConfigId(t.record2.key, t.record2.uid) def tearDown(t): del t.store @@ -138,12 +130,12 @@ class ConfigStoreAddTest(_BaseTest): def test_add_new_config(t): t.store.add(t.record1) t.store.add(t.record2) - expected1 = ConfigKey( + expected1 = CacheKey( t.fields[0].service, t.fields[0].monitor, t.fields[0].device, ) - expected2 = ConfigKey( + expected2 = CacheKey( t.fields[1].service, t.fields[1].monitor, t.fields[1].device, @@ -154,11 +146,11 @@ def test_add_new_config(t): t.assertIn(expected2, result) result = t.store.get(t.record1.key) - t.assertIsInstance(result, ConfigRecord) + t.assertIsInstance(result, CacheRecord) t.assertEqual(t.record1, result) result = t.store.get(t.record2.key) - t.assertIsInstance(result, ConfigRecord) + t.assertIsInstance(result, CacheRecord) t.assertEqual(t.record2, result) @@ -177,7 +169,7 @@ def test_negative_search(t): ) for case in cases: with subTest(key=case): - result = tuple(t.store.search(ConfigQuery(**case))) + result = tuple(t.store.search(CacheQuery(**case))) t.assertTupleEqual((), result) def test_positive_search_single(t): @@ -197,7 +189,7 @@ def test_positive_search_single(t): ) for case in cases: with subTest(key=case): - result = tuple(t.store.search(ConfigQuery(**case))) + result = tuple(t.store.search(CacheQuery(**case))) t.assertTupleEqual((t.record1.key,), result) def test_positive_search_multiple(t): @@ -221,7 +213,7 @@ def test_positive_search_multiple(t): ) for args, count in cases: with subTest(key=args): - result = tuple(t.store.search(ConfigQuery(**args))) + result = tuple(t.store.search(CacheQuery(**args))) t.assertEqual(count, len(result)) @@ -234,15 +226,15 @@ def test_get_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.fields[0].updated, status.updated) result = tuple(t.store.get_status(t.record2.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid2, cid) + status = result[0] + t.assertEqual(t.record2.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.fields[1].updated, status.updated) @@ -264,16 +256,16 @@ def test_get_older_less_multiple(t): result = tuple(t.store.get_older(t.record2.updated - 1)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertEqual(t.record1.updated, status.updated) def test_get_older_equal_single(t): t.store.add(t.record1) result = tuple(t.store.get_older(t.record1.updated)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -283,22 +275,22 @@ def test_get_older_equal_multiple(t): result = tuple(t.store.get_older(t.record1.updated)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) result = sorted( - t.store.get_older(t.record2.updated), key=lambda x: x[1].updated + t.store.get_older(t.record2.updated), key=lambda x: x.updated ) t.assertEqual(2, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - cid, status = result[1] - t.assertEqual(t.cid2, cid) + status = result[1] + t.assertEqual(t.record2.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -306,8 +298,8 @@ def test_get_older_greater_single(t): t.store.add(t.record1) result = tuple(t.store.get_older(t.record1.updated + 1)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -317,23 +309,22 @@ def test_get_older_greater_multiple(t): result = tuple(t.store.get_older(t.record1.updated + 1)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) result = sorted( - t.store.get_older(t.record2.updated + 1), - key=lambda x: x[1].updated, + t.store.get_older(t.record2.updated + 1), key=lambda x: x.updated ) t.assertEqual(2, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - cid, status = result[1] - t.assertEqual(t.cid2, cid) + status = result[1] + t.assertEqual(t.record2.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -345,8 +336,8 @@ def test_get_newer_less_single(t): t.store.add(t.record1) result = tuple(t.store.get_newer(t.record1.updated - 1)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -355,16 +346,15 @@ def test_get_newer_less_multiple(t): t.store.add(t.record2) result = sorted( - t.store.get_newer(t.record1.updated - 1), - key=lambda x: x[1].updated, + t.store.get_newer(t.record1.updated - 1), key=lambda x: x.updated ) t.assertEqual(2, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) - cid, status = result[1] - t.assertEqual(t.cid2, cid) + status = result[1] + t.assertEqual(t.record2.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -379,8 +369,8 @@ def test_get_newer_equal_multiple(t): result = tuple(t.store.get_newer(t.record1.updated)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid2, cid) + status = result[0] + t.assertEqual(t.record2.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -394,8 +384,8 @@ def test_get_newer_greater_multiple(t): t.store.add(t.record2) result = tuple(t.store.get_newer(t.record1.updated + 1)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid2, cid) + status = result[0] + t.assertEqual(t.record2.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record2.updated, status.updated) @@ -426,8 +416,8 @@ def test_retired_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(status.updated, t.record1.updated) @@ -437,8 +427,8 @@ def test_get_retired(t): result = tuple(t.store.get_retired()) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(status.updated, t.record1.updated) @@ -474,8 +464,8 @@ def test_expired_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Expired) def test_get_expired(t): @@ -485,8 +475,8 @@ def test_get_expired(t): result = tuple(t.store.get_expired()) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Expired) def test_expired_is_not_older(t): @@ -533,8 +523,8 @@ def test_pending_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -545,8 +535,8 @@ def test_get_pending(t): result = tuple(t.store.get_pending()) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -598,8 +588,8 @@ def test_building_status(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -610,8 +600,8 @@ def test_get_building(t): result = tuple(t.store.get_building()) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -639,8 +629,8 @@ def test_retired_to_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Expired) retired_keys = tuple(t.store.get_retired()) @@ -655,8 +645,8 @@ def test_expired_to_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(t.record1.updated, status.updated) @@ -675,8 +665,8 @@ def test_current_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -690,8 +680,8 @@ def test_retired_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -705,8 +695,8 @@ def test_expired_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Pending) expired_keys = tuple(t.store.get_expired()) @@ -729,8 +719,8 @@ def test_pending_to_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Expired) t.assertEqual(ts, status.expired) @@ -744,8 +734,8 @@ def test_pending_to_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(t.record1.updated, status.updated) @@ -764,8 +754,8 @@ def test_current_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -788,8 +778,8 @@ def test_retired_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -813,8 +803,8 @@ def test_expired_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -839,8 +829,8 @@ def test_pending_to_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Building) t.assertEqual(started, status.started) @@ -864,8 +854,8 @@ def test_building_to_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Pending) t.assertEqual(submitted, status.submitted) @@ -880,8 +870,8 @@ def test_building_to_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Expired) t.assertEqual(expired, status.expired) @@ -895,8 +885,8 @@ def test_building_to_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Retired) t.assertEqual(t.record1.updated, status.updated) @@ -916,8 +906,8 @@ def test_add_overwrites_retired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -932,8 +922,8 @@ def test_add_overwrites_expired(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -953,8 +943,8 @@ def test_add_overwrites_pending(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -978,8 +968,8 @@ def test_add_overwrites_building(t): result = tuple(t.store.get_status(t.record1.key)) t.assertEqual(1, len(result)) - cid, status = result[0] - t.assertEqual(t.cid1, cid) + status = result[0] + t.assertEqual(t.record1.key, status.key) t.assertIsInstance(status, ConfigStatus.Current) t.assertEqual(t.record1.updated, status.updated) @@ -993,7 +983,7 @@ def test_add_monitor_change(t): t.store.add(t.record1) newmonitor = "b2" updated = t.record1.updated + 1000 - newrecord = ConfigRecord.make( + newrecord = CacheRecord.make( t.record1.service, newmonitor, t.record1.device, @@ -1020,7 +1010,7 @@ def setUp(t): t.store = ConfigStore(t.layer.redis) t.config1 = _make_config("qadevice", "qadevice", "abc-test-01") t.config2 = _make_config("qadevice", "qadevice", "abc-test-01") - t.record1 = ConfigRecord.make( + t.record1 = CacheRecord.make( "snmp", "localhost", t.device_name, @@ -1028,7 +1018,7 @@ def setUp(t): 123456.23, t.config1, ) - t.record2 = ConfigRecord.make( + t.record2 = CacheRecord.make( "ping", "localhost", t.device_name, @@ -1052,7 +1042,7 @@ def test_uid(t): records = tuple( t.store.get(key) - for key in t.store.search(ConfigQuery(device=t.device_name)) + for key in t.store.search(CacheQuery(device=t.device_name)) ) t.assertEqual(2, len(records)) @@ -1067,7 +1057,7 @@ def test_uid_after_one_removal(t): records = tuple( t.store.get(key) - for key in t.store.search(ConfigQuery(device=t.device_name)) + for key in t.store.search(CacheQuery(device=t.device_name)) ) t.assertEqual(1, len(records)) @@ -1080,7 +1070,7 @@ def test_uid_after_removing_all(t): records = tuple( t.store.get(key) - for key in t.store.search(ConfigQuery(device=t.device_name)) + for key in t.store.search(CacheQuery(device=t.device_name)) ) t.assertEqual(0, len(records)) diff --git a/Products/ZenCollector/services/ConfigCache.py b/Products/ZenCollector/services/ConfigCache.py index c7cea8b200..899c2d4bb5 100644 --- a/Products/ZenCollector/services/ConfigCache.py +++ b/Products/ZenCollector/services/ConfigCache.py @@ -11,7 +11,7 @@ from zope.component import createObject -from Products.ZenCollector.configcache.cache import ConfigQuery +from Products.ZenCollector.configcache.cache import CacheQuery from Products.ZenHub.errors import translateError from Products.ZenHub.HubService import HubService from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl @@ -107,11 +107,11 @@ def remote_getDeviceConfigs( # 'updated_keys' references newer configs found in 'previous' updated_keys = ( - key - for key, _ in self._store.get_newer( + status.key + for status in self._store.get_newer( when, service=servicename, monitor=self.instance ) - if key.device in previous + if status.key.device in previous ) # 'removed' references devices found in 'previous' @@ -136,7 +136,7 @@ def _keys(self, servicename): @type servicename: str @rtype: Iterator[str] """ - query = ConfigQuery(monitor=self.instance, service=servicename) + query = CacheQuery(monitor=self.instance, service=servicename) self.log.info("[ConfigCache] using query %s", query) return self._store.search(query) @@ -146,7 +146,7 @@ def _filter(self, keys, predicate): of the `options` parameter. @param keys: Cache config keys - @type keys: Iterable[ConfigKey] + @type keys: Iterable[CacheKey] @param predicate: Function that determines whether to keep the device @type options: Function(Device) -> Boolean @rtype: Iterator[str] From 75d870fdabc4498a0a966f2f6034b1e18f58b569 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 28 Feb 2024 11:31:45 -0600 Subject: [PATCH 085/147] fix: ensure minimumTTL has precedence over TTL. When looking for old device configs to rebuild, make sure the minimum TTL setting is applied. ZEN-34731 --- Products/ZenCollector/configcache/manager.py | 212 +++++++++---------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index f9c026c93f..a8d018cee1 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -13,7 +13,6 @@ import logging from datetime import datetime -from itertools import chain from time import time from zope.component import createObject @@ -88,148 +87,149 @@ def run(self): try: self.ctx.session.sync() gc.collect() - self._retry_build() - self._retry_pending() - self._expire_retired_configs() - self._rebuild_older_configs() + timedout = tuple(self._get_build_timeouts()) + if not timedout: + self.log.debug("no configuration builds have timed out") + else: + self._expire_configs(timedout, "build") + timedout = tuple(self._get_pending_timeouts()) + if not timedout: + self.log.debug( + "no pending configuration builds have timed out" + ) + else: + self._expire_configs(timedout, "pending") + statuses = self._get_configs_to_rebuild() + if statuses: + self._rebuild_configs(statuses) except Exception as ex: self.log.exception("unexpected error %s", ex) self.ctx.controller.wait(self.interval) - def _retry_build(self): + def _get_build_timeouts(self): buildlimitmap = DevicePropertyMap.make_build_timeout_map( self.ctx.dmd.Devices ) # Test against a time 10 minutes earlier to minimize interfering # with builder working on the same config. now = time() - 600 - count = 0 for status in self.store.get_building(): - duration = buildlimitmap.get(status.uid) - if status.started < (now - duration): - self.store.set_expired((status.key, now)) - self.log.info( - "expired configuration due to build timeout " - "started=%s timeout=%s service=%s monitor=%s device=%s", - datetime.fromtimestamp(status.started).strftime( - "%Y-%m-%d %H:%M:%S" - ), - duration, - status.key.service, - status.key.monitor, - status.key.device, + limit = buildlimitmap.get(status.uid) + if status.started < (now - limit): + yield ( + status, + "started", + status.started, + Constants.build_timeout_id, + limit, ) - count += 1 - if count == 0: - self.log.debug("no configuration builds have timed out") - def _retry_pending(self): + def _get_pending_timeouts(self): pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( self.ctx.dmd.Devices ) now = time() - count = 0 for status in self.store.get_pending(): - duration = pendinglimitmap.get(status.uid) - if status.submitted < (now - duration): - self.store.set_expired((status.key, now)) - self.log.info( - "expired pending configuration build due to timeout " - "submitted=%s timeout=%s service=%s monitor=%s device=%s", - datetime.fromtimestamp(status.submitted).strftime( - "%Y-%m-%d %H:%M:%S" - ), - duration, - status.key.service, - status.key.monitor, - status.key.device, + limit = pendinglimitmap.get(status.uid) + if status.submitted < (now - limit): + yield ( + status, + "submitted", + status.submitted, + Constants.pending_timeout_id, + limit, ) - count += 1 - if count == 0: - self.log.debug("no pending configuration builds have timed out") - def _expire_retired_configs(self): + def _expire_configs(self, data, kind): + now = time() + self.store.set_expired( + *((status.key, now) for status, _, _, _, _ in data) + ) + for status, valId, val, limitId, limitValue in data: + self.log.info( + "expired configuration due to %s timeout " + "%s=%s %s=%s service=%s monitor=%s device=%s", + kind, + valId, + datetime.fromtimestamp(val).strftime("%Y-%m-%d %H:%M:%S"), + limitId, + limitValue, + status.key.service, + status.key.monitor, + status.key.device, + ) + + def _get_configs_to_rebuild(self): minttl_map = DevicePropertyMap.make_minimum_ttl_map( self.ctx.dmd.Devices ) + ttl_map = DevicePropertyMap.make_ttl_map(self.ctx.dmd.Devices) now = time() - expire = tuple( - status.key + + # Retrieve the 'retired' configs + ready_to_rebuild = list( + status for status in self.store.get_retired() if status.updated < now - minttl_map.get(status.uid) ) - self.store.set_expired(*((key, now) for key in expire)) - def _rebuild_older_configs(self): + # Append the 'expired' configs + ready_to_rebuild.extend(self.store.get_expired()) + + # Append the 'older' configs. + min_age = now - ttl_map.smallest_value() + for status in self.store.get_older(min_age): + # Select the min ttl if the ttl is a smaller value + limit = max(minttl_map.get(status.uid), ttl_map.get(status.uid)) + expiration_threshold = now - limit + if status.updated <= expiration_threshold: + ready_to_rebuild.append(status) + + return ready_to_rebuild + + def _rebuild_configs(self, statuses): buildlimitmap = DevicePropertyMap.make_build_timeout_map( self.ctx.dmd.Devices ) - ttlmap = DevicePropertyMap.make_ttl_map(self.ctx.dmd.Devices) - min_ttl = ttlmap.smallest_value() - self.log.debug("minimum age limit is %s", _formatted_interval(min_ttl)) - now = time() - min_age = now - min_ttl - results = chain.from_iterable( - (self.store.get_expired(), self.store.get_older(min_age)) - ) count = 0 - for status in results: - if status.uid is None: - self.log.warn( - "No UID found for device device=%s", status.key.device + for status in statuses: + timeout = buildlimitmap.get(status.uid) + self.store.set_pending((status.key, time())) + self.dispatcher.dispatch( + status.key.service, + status.key.monitor, + status.key.device, + timeout, + ) + if isinstance(status, ConfigStatus.Expired): + self.log.info( + "submitted job to rebuild expired config " + "service=%s monitor=%s device=%s", + status.key.service, + status.key.monitor, + status.key.device, ) - continue - ttl = ttlmap.get(status.uid) - expiration_threshold = now - ttl - if ( - isinstance(status, ConfigStatus.Expired) - or status.updated <= expiration_threshold - ): - timeout = buildlimitmap.get(status.uid) - self.store.set_pending((status.key, time())) - self.dispatcher.dispatch( + elif isinstance(status, ConfigStatus.Retired): + self.log.info( + "submitted job to rebuild retired config " + "service=%s monitor=%s device=%s", status.key.service, status.key.monitor, status.key.device, + ) + else: + self.log.info( + "submitted job to rebuild old config " + "updated=%s %s=%s service=%s monitor=%s device=%s", + datetime.fromtimestamp(status.updated).strftime( + "%Y-%m-%d %H:%M:%S" + ), + Constants.time_to_live_id, timeout, + status.key.service, + status.key.monitor, + status.key.device, ) - if isinstance(status, ConfigStatus.Expired): - self.log.info( - "submitted job to rebuild expired config " - "service=%s monitor=%s device=%s", - status.key.service, - status.key.monitor, - status.key.device, - ) - else: - self.log.info( - "submitted job to rebuild old config " - "updated=%s %s=%s service=%s monitor=%s device=%s", - datetime.fromtimestamp(status.updated).strftime( - "%Y-%m-%d %H:%M:%S" - ), - Constants.time_to_live_id, - ttl, - status.key.service, - status.key.monitor, - status.key.device, - ) - count += 1 + count += 1 if count == 0: self.log.debug("found no expired or old configurations to rebuild") - - -def _formatted_interval(total_seconds): - minutes, seconds = divmod(total_seconds, 60) - hours, minutes = divmod(minutes, 60) - days, hours = divmod(hours, 24) - text = "" - if seconds: - text = "{:02} seconds".format(seconds) - if minutes: - text = "{:02} minutes {}".format(minutes, text).strip() - if hours: - text = "{:02} hours {}".format(hours, text).strip() - if days: - text = "{} days {}".format(days, text).strip() - return text From c84b75a379ba533eed963470429e1c47a8577444 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 29 Feb 2024 15:16:43 -0600 Subject: [PATCH 086/147] Replace 'monitor' with 'collector' in output. The typical term for a monitor is collector, so use collector when writing to stdout/stderr and log files. Also fixed wildcard handling with the 'expire' command. --- Products/ZenCollector/configcache/cli.py | 82 ++++++++++++------- .../ZenCollector/configcache/invalidator.py | 18 ++-- Products/ZenCollector/configcache/manager.py | 8 +- Products/ZenCollector/configcache/task.py | 10 +-- Products/ZenHub/zenhubclient.py | 2 +- 5 files changed, 70 insertions(+), 50 deletions(-) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 47862591aa..04fd9f2cd8 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -66,7 +66,7 @@ def add_arguments(parser, subparsers): subp.set_defaults(factory=List_) def __init__(self, args): - self._monitor = "*{}*".format(args.monitor).replace("***", "*") + self._monitor = "*{}*".format(args.collector).replace("***", "*") self._service = "*{}*".format(args.service).replace("***", "*") self._showuid = args.show_uid self._devices = getattr(args, "device", []) @@ -134,7 +134,7 @@ def run(self): "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( "DEVICE", "STATUS", - "MONITOR", + "COLLECTOR", "SERVICE", maxd=maxd, maxs=maxs, @@ -207,13 +207,13 @@ def add_arguments(parser, subparsers): "service", nargs=1, help="name of the configuration service" ) subp.add_argument( - "monitor", nargs=1, help="name of the performance monitor" + "collector", nargs=1, help="name of the performance collector" ) subp.add_argument("device", nargs=1, help="name of the device") subp.set_defaults(factory=Show) def __init__(self, args): - self._monitor = args.monitor[0] + self._monitor = args.collector[0] self._service = args.service[0] self._device = args.device[0] if _is_output_redirected(): @@ -309,11 +309,22 @@ def add_arguments(parser, subparsers): subp.set_defaults(factory=Expire) def __init__(self, args): - self._monitor = "*{}*".format(args.monitor).replace("***", "*") - self._service = "*{}*".format(args.service).replace("***", "*") + self._monitor = args.collector + self._service = args.service self._devices = getattr(args, "device", []) def run(self): + haswildcard = any("*" in d for d in self._devices) + if haswildcard: + if len(self._devices) > 1: + print( + "Only one DEVICE argument supported when a " + "wildcard is used.", + file=sys.stderr, + ) + return + else: + self._devices = self._devices[0].replace("*", "") if not self._confirm_inputs(): print("exit") return @@ -323,7 +334,7 @@ def run(self): query = CacheQuery(service=self._service, monitor=self._monitor) results = store.get_status(*store.search(query)) method = self._no_devices if not self._devices else self._with_devices - keys = method(results) + keys = method(results, wildcard=haswildcard) now = time.time() store.set_expired(*((key, now) for key in keys)) count = len(keys) @@ -332,16 +343,25 @@ def run(self): % (count, "" if count == 1 else "s") ) - def _no_devices(self, results): + def _no_devices(self, results, wildcard=False): return tuple(status.key for status in results) - def _with_devices(self, results): + def _with_devices(self, results, wildcard=False): + if wildcard: + predicate = self._check_wildcard + else: + predicate = self._check_list + return tuple( - status.key - for status in results - if status.key.device in self._devices + status.key for status in results if predicate(status.key.device) ) + def _check_wildcard(self, device): + return self._devices in device + + def _check_list(self, device): + return device in self._devices + def _confirm_inputs(self): if self._devices: return True @@ -349,50 +369,50 @@ def _confirm_inputs(self): mesg = "Recreate all device configurations" elif "*" not in self._monitor and self._service == "*": mesg = ( - "Recreate all device configurations monitored by the " + "Recreate all configurations for devices monitored by the " "'%s' collector" % (self._monitor,) ) elif "*" in self._monitor and self._service == "*": mesg = ( - "Recreate all device configurations monitored by all " + "Recreate all configurations for devices monitored by all " "collectors matching '%s'" % (self._monitor,) ) elif self._monitor == "*" and "*" not in self._service: mesg = ( "Recreate all device configurations created by the '%s' " - "configuration service" % (self._service.split(".")[-1],) + "service" % (self._service.split(".")[-1],) ) elif self._monitor == "*" and "*" in self._service: mesg = ( "Recreate all device configurations created by all " - "configuration services matching '%s'" % (self._service,) + "services matching '%s'" % (self._service,) ) elif "*" in self._monitor and "*" not in self._service: mesg = ( - "Recreate all device configurations created by the " - "'%s' configuration service and monitored by all " - "collectors matching '%s'" % (self._service, self._monitor) + "Recreate all configurations created by the '%s' " + "service for devices monitored by all collectors " + "matching '%s'" % (self._service, self._monitor) ) elif "*" not in self._monitor and "*" in self._service: mesg = ( - "Recreate all device configurations monitored by the '%s' " - "collector and created by all configuration services " - "matching '%s'" % (self._monitor, self._service) + "Recreate all configurations for devices monitored by the " + "'%s' collector and created by all services matching '%s'" + % (self._monitor, self._service) ) elif "*" not in self._monitor and "*" not in self._service: mesg = ( - "Recreate all device configurations monitored by the '%s' " - "collector and created by the '%s' configuration service" + "Recreate all configurations for devices monitored by the " + "'%s' collector and created by the '%s' service" % (self._monitor, self._service) ) elif "*" in self._monitor and "*" in self._service: mesg = ( - "Recreate all device configurations monitored by all " - "collectors matching '%s' and created by all configuration " - "services matching '%s'" % (self._monitor, self._service) + "Recreate all configurations device monitored by all " + "collectors matching '%s' and created by all services " + "matching '%s'" % (self._monitor, self._service) ) else: - mesg = "monitor '%s' service '%s'" % ( + mesg = "collector '%s' service '%s'" % ( self._monitor, self._service, ) @@ -461,11 +481,11 @@ def _get_common_parser(): _common_parser = argparse.ArgumentParser(add_help=False) _common_parser.add_argument( "-m", - "--monitor", + "--collector", type=str, default="*", - help="Name of the performance monitor. Supports simple '*' " - "wildcard comparisons. A lone '*' selects all monitors.", + help="Name of the performance collector. Supports simple '*' " + "wildcard comparisons. A lone '*' selects all collectors.", ) _common_parser.add_argument( "-s", diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 0bdbd9d024..114dcf973d 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -141,7 +141,7 @@ def _process(self, invalidation): monitor = device.getPerformanceServerName() if monitor is None: self.log.warn( - "ignoring invalidated device having undefined monitor " + "ignoring invalidated device having undefined collector " "device=%s reason=%s", device, reason, @@ -159,7 +159,7 @@ def _process(self, invalidation): else: self.log.warn( "ignored unexpected reason " - "reason=%s device=%s monitor=%s device-oid=%r", + "reason=%s device=%s collector=%s device-oid=%r", reason, device, monitor, @@ -174,7 +174,7 @@ def _new_device(self, device, monitor): timeout = timelimitmap.get(uid) self.dispatcher.dispatch_all(monitor, device.id, timeout) self.log.info( - "submitted build jobs for new device uid=%s monitor=%s", + "submitted build jobs for new device uid=%s collector=%s", uid, monitor, ) @@ -204,7 +204,7 @@ def _updated_device(self, device, monitor, keys, invalidation): for key in retired: self.log.info( "retired configuration of changed device " - "device=%s monitor=%s service=%s device-oid=%r", + "device=%s collector=%s service=%s device-oid=%r", key.device, key.monitor, key.service, @@ -213,7 +213,7 @@ def _updated_device(self, device, monitor, keys, invalidation): for key in expired: self.log.info( "expired configuration of changed device " - "device=%s monitor=%s service=%s device-oid=%r", + "device=%s collector=%s service=%s device-oid=%r", key.device, key.monitor, key.service, @@ -225,7 +225,7 @@ def _removed_device(self, keys, invalidation): for key in keys: self.log.info( "removed configuration of deleted device " - "device=%s monitor=%s service=%s device-oid=%r", + "device=%s collector=%s service=%s device-oid=%r", key.device, key.monitor, key.service, @@ -255,7 +255,7 @@ def _removeDeleted(log, tool, store): for key in devices_not_found: log.info( "removed configuration for deleted device " - "device=%s monitor=%s service=%s", + "device=%s collector=%s service=%s", key.device, key.monitor, key.service, @@ -275,7 +275,7 @@ def _addNew(log, tool, timelimitmap, store, dispatcher): for brain in catalog_results: if brain.collector is None: log.warn( - "ignoring device having undefined monitor device=%s uid=%s", + "ignoring device having undefined collector device=%s uid=%s", brain.id, brain.uid, ) @@ -288,7 +288,7 @@ def _addNew(log, tool, timelimitmap, store, dispatcher): dispatcher.dispatch_all(brain.collector, brain.id, timeout) log.info( "submitted build jobs for device without any configurations " - "uid=%s monitor=%s", + "uid=%s collector=%s", brain.uid, brain.collector, ) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index a8d018cee1..e08809d5a4 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -148,7 +148,7 @@ def _expire_configs(self, data, kind): for status, valId, val, limitId, limitValue in data: self.log.info( "expired configuration due to %s timeout " - "%s=%s %s=%s service=%s monitor=%s device=%s", + "%s=%s %s=%s service=%s collector=%s device=%s", kind, valId, datetime.fromtimestamp(val).strftime("%Y-%m-%d %H:%M:%S"), @@ -204,7 +204,7 @@ def _rebuild_configs(self, statuses): if isinstance(status, ConfigStatus.Expired): self.log.info( "submitted job to rebuild expired config " - "service=%s monitor=%s device=%s", + "service=%s collector=%s device=%s", status.key.service, status.key.monitor, status.key.device, @@ -212,7 +212,7 @@ def _rebuild_configs(self, statuses): elif isinstance(status, ConfigStatus.Retired): self.log.info( "submitted job to rebuild retired config " - "service=%s monitor=%s device=%s", + "service=%s collector=%s device=%s", status.key.service, status.key.monitor, status.key.device, @@ -220,7 +220,7 @@ def _rebuild_configs(self, statuses): else: self.log.info( "submitted job to rebuild old config " - "updated=%s %s=%s service=%s monitor=%s device=%s", + "updated=%s %s=%s service=%s collector=%s device=%s", datetime.fromtimestamp(status.updated).strftime( "%Y-%m-%d %H:%M:%S" ), diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 78fe10f3c6..7d3832d03c 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -68,7 +68,7 @@ def build_device_config( if submitted < (now - duration): self.log.warn( "dropped this job in favor of newer job " - "device=%s monitor=%s service=%s submitted=%f %s=%s", + "device=%s collector=%s service=%s submitted=%f %s=%s", deviceid, monitorname, svcname, @@ -82,7 +82,7 @@ def build_device_config( # that configcache-manager doesn't prematurely timeout the build. store.set_building((CacheKey(svcname, monitorname, deviceid), time())) self.log.info( - "building device configuration device=%s monitor=%s service=%s", + "building device configuration device=%s collector=%s service=%s", deviceid, monitorname, svcname, @@ -92,7 +92,7 @@ def build_device_config( configs = service.remote_getDeviceConfigs((deviceid,)) if not configs: self.log.info( - "no configuration built device=%s monitor=%s service=%s", + "no configuration built device=%s collector=%s service=%s", deviceid, monitorname, svcname, @@ -110,7 +110,7 @@ def build_device_config( store.remove(key) self.log.info( "removed previously built configuration " - "device=%s monitor=%s service=%s", + "device=%s collector=%s service=%s", key.device, key.monitor, key.service, @@ -124,7 +124,7 @@ def build_device_config( store.add(record) self.log.info( "added/replaced config " - "updated=%s device=%s monitor=%s service=%s", + "updated=%s device=%s collector=%s service=%s", datetime.fromtimestamp(record.updated).isoformat(), deviceid, monitorname, diff --git a/Products/ZenHub/zenhubclient.py b/Products/ZenHub/zenhubclient.py index 74bfcd103b..8f3829649f 100644 --- a/Products/ZenHub/zenhubclient.py +++ b/Products/ZenHub/zenhubclient.py @@ -175,7 +175,7 @@ def get_service(self, name, monitor, listener, options): self.__services[name] = service_ref log.debug( "retrieved remote reference to ZenHub service " - "name=%s monitor=%s service=%r", + "name=%s collector=%s service=%r", name, monitor, service_ref, From 521a8304ff52007a24ee8038d93aba1044441dbc Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 1 Mar 2024 15:26:39 -0600 Subject: [PATCH 087/147] fix: correctly support building configs for new devices Set the 'pending' status when dispatching config build jobs for new devices. Clear statuses on cache keys that have no configurations. ZEN-34739 --- .../ZenCollector/configcache/cache/model.py | 40 +- .../ZenCollector/configcache/cache/storage.py | 273 +++--- Products/ZenCollector/configcache/cli.py | 6 +- .../ZenCollector/configcache/invalidator.py | 97 +- Products/ZenCollector/configcache/manager.py | 13 +- .../ZenCollector/configcache/propertymap.py | 18 +- Products/ZenCollector/configcache/task.py | 20 +- .../tests/test_propertymap_makers.py | 9 +- .../configcache/tests/test_storage.py | 871 ++++++++++-------- 9 files changed, 716 insertions(+), 631 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py index e3e98c4060..4cafb636d3 100644 --- a/Products/ZenCollector/configcache/cache/model.py +++ b/Products/ZenCollector/configcache/cache/model.py @@ -11,33 +11,33 @@ import attr -from attr.validators import instance_of +from attr.validators import instance_of, optional from Products.ZenCollector.services.config import DeviceProxy @attr.s(frozen=True, slots=True) class CacheQuery(object): - service = attr.ib(validator=[instance_of(str)], default="*") - monitor = attr.ib(validator=[instance_of(str)], default="*") - device = attr.ib(validator=[instance_of(str)], default="*") + service = attr.ib(validator=instance_of(str), default="*") + monitor = attr.ib(validator=instance_of(str), default="*") + device = attr.ib(validator=instance_of(str), default="*") @attr.s(frozen=True, slots=True) class CacheKey(object): - service = attr.ib(validator=[instance_of(str)]) - monitor = attr.ib(validator=[instance_of(str)]) - device = attr.ib(validator=[instance_of(str)]) + service = attr.ib(validator=instance_of(str)) + monitor = attr.ib(validator=instance_of(str)) + device = attr.ib(validator=instance_of(str)) @attr.s(slots=True) class CacheRecord(object): key = attr.ib( - validator=[instance_of(CacheKey)], on_setattr=attr.setters.NO_OP + validator=instance_of(CacheKey), on_setattr=attr.setters.NO_OP ) - uid = attr.ib(validator=[instance_of(str)], on_setattr=attr.setters.NO_OP) - updated = attr.ib(validator=[instance_of(float)]) - config = attr.ib(validator=[instance_of(DeviceProxy)]) + uid = attr.ib(validator=instance_of(str), on_setattr=attr.setters.NO_OP) + updated = attr.ib(validator=instance_of(float)) + config = attr.ib(validator=instance_of(DeviceProxy)) @classmethod def make(cls, svc, mon, dev, uid, updated, config): @@ -60,8 +60,12 @@ def device(self): class _Status(object): """Base class for status classes.""" - key = attr.ib(validator=[instance_of(CacheKey)]) - uid = attr.ib(validator=[instance_of(str)]) + key = attr.ib(validator=instance_of(CacheKey)) + uid = attr.ib(validator=optional(instance_of(str))) + + @property + def has_config(self): + return self.uid is not None class _ConfigStatus(object): @@ -73,31 +77,31 @@ class _ConfigStatus(object): class Current(_Status): """The configuration is current.""" - updated = attr.ib(validator=[instance_of(float)]) + updated = attr.ib(validator=instance_of(float)) @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") class Retired(_Status): """The cofiguration is retired, but not yet expired.""" - updated = attr.ib(validator=[instance_of(float)]) + retired = attr.ib(validator=instance_of(float)) @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") class Expired(_Status): """The configuration has expired.""" - expired = attr.ib(validator=[instance_of(float)]) + expired = attr.ib(validator=instance_of(float)) @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") class Pending(_Status): """The configuration is waiting for a rebuild.""" - submitted = attr.ib(validator=[instance_of(float)]) + submitted = attr.ib(validator=instance_of(float)) @attr.s(slots=True, frozen=True, repr_ns="ConfigStatus") class Building(_Status): """The configuration is rebuilding.""" - started = attr.ib(validator=[instance_of(float)]) + started = attr.ib(validator=instance_of(float)) def __contains__(self, value): return isinstance( diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index cd5d6e20df..8c144bcbba 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -61,10 +61,9 @@ import ast import logging -import itertools from functools import partial -from itertools import islice +from itertools import chain, islice import attr @@ -77,7 +76,7 @@ from .table import DeviceUIDTable, DeviceConfigTable, ConfigMetadataTable _app = "configcache" -log = logging.getLogger("zen.modelchange.stores") +log = logging.getLogger("zen.configcache.storage") class ConfigStoreFactory(Factory): @@ -135,19 +134,13 @@ def search(self, query=CacheQuery()): """ if not isinstance(query, CacheQuery): raise TypeError("'{!r} is not a CacheQuery".format(query)) - return ( - CacheKey(svc, mon, dvc) - for svc, mon, dvc in self.__config.scan( - self.__client, **attr.asdict(query) - ) - ) + return self._query(**attr.asdict(query)) def add(self, record): """ @type record: CacheRecord """ svc, mon, dvc, uid, updated, config = _from_record(record) - orphaned_keys = tuple( key for key in self.search(CacheQuery(service=svc, device=dvc)) @@ -189,6 +182,18 @@ def get_uid(self, device): """ return self.__uids.get(self.__client, device) + def get_updated(self, key): + """ + Return the timestamp of when the config was built. + + @type key: CacheKey + """ + return _to_ts( + self.__age.score( + self.__client, key.service, key.monitor, key.device + ) + ) + def get(self, key, default=None): """ @type key: CacheKey @@ -222,191 +227,141 @@ def remove(self, *keys): self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) pipe.execute() - devices = [] - for dvc in set(k.device for k in keys): - configs = tuple(self.__config.scan(self.__client, device=dvc)) - if not configs: - devices.append(dvc) - if devices: - self.__uids.delete(self.__client, *devices) - def set_retired(self, *keys): - """ - Marks the indicated configuration as retired. + devices = set(key.device for key in keys) + remaining = set( + key.device + for key in chain.from_iterable( + self._query(device=dvc) for dvc in devices + ) + ) + deleted = devices - remaining + if deleted: + self.__uids.delete(self.__client, *deleted) - The keys that were retired are returned to the caller. The returned - keys may match or be a subset of the keys that were passed in. + def _query(self, service="*", monitor="*", device="*"): + return ( + CacheKey(svc, mon, dvc) + for svc, mon, dvc in self.__config.scan( + self.__client, service=service, monitor=monitor, device=device + ) + ) - If a config is already retired, it will not be among the keys - that are returned. + def clear_status(self, *keys): + """ + Removes retired, expired, pending, and building statuses. + + If a config is present, the status becomes current. If no config + is present, then there is no status. @type keys: Sequence[CacheKey] - @rtype: Sequence[CacheKey] """ - if len(keys) == 0: - return () + with self.__client.pipeline() as pipe: + for key in keys: + svc, mon, dvc = key.service, key.monitor, key.device + self.__retired.delete(pipe, svc, mon, dvc) + self.__expired.delete(pipe, svc, mon, dvc) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.delete(pipe, svc, mon, dvc) + pipe.execute() - not_retired = tuple( - self._filter_existing(self.__retired, keys, lambda x: x) - ) - if len(not_retired) == 0: - return () - - watch_keys = self._get_watch_keys(not_retired) - scores = ( - ( - key, - self.__age.score( - self.__client, key.service, key.monitor, key.device - ), - ) - for key in not_retired - ) - targets = ( - (key.service, key.monitor, key.device, score) - for key, score in scores - ) + def set_retired(self, *pairs): + """ + Marks the indicated configuration(s) as retired. - def _impl(pipe): + @type keys: Sequence[(CacheKey, float)] + """ + + def _impl(rows, pipe): pipe.multi() - for svc, mon, dvc, score in targets: + for svc, mon, dvc, ts in rows: + score = _to_score(ts) self.__retired.add(pipe, svc, mon, dvc, score) self.__expired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) - self.__client.transaction(_impl, *watch_keys) - return not_retired + self._set_status(pairs, self.__retired, _impl) def set_expired(self, *pairs): """ - Marks the indicated configuration as expired. - - The keys that were expired are returned to the caller. The returned - keys may match or be a subset of the keys that were passed in. - - If a config is already expired, it will not be among the keys - that are returned. + Marks the indicated configuration(s) as expired. @type keys: Sequence[(CacheKey, float)] - @rtype: Sequence[CacheKey] """ - if len(pairs) == 0: - return () - not_expired = tuple( - self._filter_existing(self.__expired, pairs, lambda x: x[0]) - ) - if len(not_expired) == 0: - return () - - watch_keys = self._get_watch_keys(key for key, _ in not_expired) - targets = ( - (key.service, key.monitor, key.device, ts) - for key, ts in not_expired - ) - - def _impl(pipe): + def _impl(rows, pipe): pipe.multi() - for svc, mon, dvc, ts in targets: + for svc, mon, dvc, ts in rows: score = _to_score(ts) self.__retired.delete(pipe, svc, mon, dvc) self.__expired.add(pipe, svc, mon, dvc, score) self.__pending.delete(pipe, svc, mon, dvc) self.__building.delete(pipe, svc, mon, dvc) - self.__client.transaction(_impl, *watch_keys) - return tuple(key for key, _ in not_expired) + self._set_status(pairs, self.__expired, _impl) def set_pending(self, *pairs): """ - Marks a configuration as waiting for a new configuration. - - The keys that were marked pending are returned to the caller. - The returned keys may match or be a subset of the keys that - were passed in. - - If a config is already marked pending, it will not be among the - keys that are returned. + Marks configuration(s) as waiting for a new configuration. @type pending: Sequence[(CacheKey, float)] - @rtype: Sequence[CacheKey] """ - if len(pairs) == 0: - return () - - not_pending = tuple( - self._filter_existing(self.__pending, pairs, lambda x: x[0]) - ) - if len(not_pending) == 0: - return () - - watch_keys = self._get_watch_keys(key for key, _ in not_pending) - targets = ( - (key.service, key.monitor, key.device, ts) - for key, ts in not_pending - ) - def _impl(pipe): + def _impl(rows, pipe): pipe.multi() - for svc, mon, dvc, ts in targets: + for svc, mon, dvc, ts in rows: score = _to_score(ts) self.__retired.delete(pipe, svc, mon, dvc) self.__expired.delete(pipe, svc, mon, dvc) self.__pending.add(pipe, svc, mon, dvc, score) self.__building.delete(pipe, svc, mon, dvc) - self.__client.transaction(_impl, *watch_keys) - return tuple(key for key, _ in not_pending) + self._set_status(pairs, self.__pending, _impl) def set_building(self, *pairs): """ - Marks a configuration as building a new configuration. - - The keys that were marked building are returned to the caller. - The returned keys may match or be a subset of the keys that - were passed in. - - If a config is already marked building, it will not be among the - keys that are returned. + Marks configuration(s) as building a new configuration. @type pairs: Sequence[(CacheKey, float)] - @rtype: Sequence[CacheKey] """ - if len(pairs) == 0: - return () - not_building = tuple( - self._filter_existing(self.__building, pairs, lambda x: x[0]) - ) - if len(not_building) == 0: - return () - - watch_keys = self._get_watch_keys(key for key, _ in not_building) - targets = ( - (key.service, key.monitor, key.device, ts) - for key, ts in not_building - ) - - def _impl(pipe): + def _impl(rows, pipe): pipe.multi() - for svc, mon, dvc, ts in targets: + for svc, mon, dvc, ts in rows: score = _to_score(ts) self.__retired.delete(pipe, svc, mon, dvc) self.__expired.delete(pipe, svc, mon, dvc) self.__pending.delete(pipe, svc, mon, dvc) self.__building.add(pipe, svc, mon, dvc, score) - self.__client.transaction(_impl, *watch_keys) - return tuple(key for key, _ in not_building) + self._set_status(pairs, self.__building, _impl) - def _filter_existing(self, table, items, getkey): - for item in items: - key = getkey(item) - if not table.exists( - self.__client, key.service, key.monitor, key.device - ): - yield item + def _set_status(self, pairs, table, fn): + if len(pairs) == 0: + return + + watch_keys = self._get_watch_keys(key for key, _ in pairs) + rows = ( + (key.service, key.monitor, key.device, ts) for key, ts in pairs + ) + + callback = partial(fn, rows) + self.__client.transaction(callback, *watch_keys) + + def _get_watch_keys(self, keys): + return set( + chain.from_iterable( + ( + self.__age.make_key(key.service, key.monitor), + self.__retired.make_key(key.service, key.monitor), + self.__expired.make_key(key.service, key.monitor), + self.__pending.make_key(key.service, key.monitor), + self.__building.make_key(key.service, key.monitor), + ) + for key in keys + ) + ) def get_status(self, *keys): """ @@ -417,10 +372,23 @@ def get_status(self, *keys): for key in keys: scores = self._get_scores(key) uid = self.__uids.get(self.__client, key.device) - status = self._get_status(scores, key, uid) + status = self._get_status_from_scores(scores, key, uid) if status is not None: yield status + def _get_status_from_scores(self, scores, key, uid): + age, retired, expired, pending, building = scores + if building is not None: + return ConfigStatus.Building(key, uid, _to_ts(building)) + elif pending is not None: + return ConfigStatus.Pending(key, uid, _to_ts(pending)) + elif expired is not None: + return ConfigStatus.Expired(key, uid, _to_ts(expired)) + elif retired is not None: + return ConfigStatus.Retired(key, uid, _to_ts(retired)) + elif age is not None: + return ConfigStatus.Current(key, uid, _to_ts(age)) + def get_building(self, service="*", monitor="*"): """ Return an iterator producing ConfigStatus.Building objects. @@ -525,33 +493,6 @@ def _get_scores(self, key): self.__building.score(pipe, service, monitor, device), return pipe.execute() - def _get_status(self, scores, key, uid): - age, retired, expired, pending, building = scores - if building is not None: - return ConfigStatus.Building(key, uid, _to_ts(building)) - elif pending is not None: - return ConfigStatus.Pending(key, uid, _to_ts(pending)) - elif expired is not None: - return ConfigStatus.Expired(key, uid, _to_ts(expired)) - elif retired is not None: - return ConfigStatus.Retired(key, uid, _to_ts(retired)) - elif age is not None: - return ConfigStatus.Current(key, uid, _to_ts(age)) - - def _get_watch_keys(self, keys): - return set( - itertools.chain.from_iterable( - ( - self.__age.make_key(key.service, key.monitor), - self.__retired.make_key(key.service, key.monitor), - self.__expired.make_key(key.service, key.monitor), - self.__pending.make_key(key.service, key.monitor), - self.__building.make_key(key.service, key.monitor), - ) - for key in keys - ) - ) - def _range(client, metadata, svc, mon, minv=None, maxv=None): pairs = metadata.get_pairs(client, svc, mon) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py index 04fd9f2cd8..afa5d516e7 100644 --- a/Products/ZenCollector/configcache/cli.py +++ b/Products/ZenCollector/configcache/cli.py @@ -168,11 +168,11 @@ def _format_status(status): if isinstance(status, ConfigStatus.Current): return "current since {}".format(_format_date(status.updated)) elif isinstance(status, ConfigStatus.Retired): - return "retired" + return "retired since {}".format(_format_date(status.retired)) elif isinstance(status, ConfigStatus.Expired): - return "expired" + return "expired since {}".format(_format_date(status.expired)) elif isinstance(status, ConfigStatus.Pending): - return "build request submitted {}".format( + return "waiting to build since {}".format( _format_date(status.submitted) ) elif isinstance(status, ConfigStatus.Building): diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 114dcf973d..8008852d86 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -24,7 +24,7 @@ from .app import Application from .app.args import get_subparser -from .cache import CacheQuery, ConfigStatus +from .cache import CacheKey, CacheQuery, ConfigStatus from .debug import Debug as DebugCommand from .modelchange import InvalidationCause from .propertymap import DevicePropertyMap @@ -107,16 +107,7 @@ def run(self): result = poller.poll() if result: self.log.debug("found %d relevant invalidations", len(result)) - for invalidation in result: - try: - self._process(invalidation) - except AttributeError: - self.log.info( - "invalidation device=%s reason=%s", - invalidation.device, - invalidation.reason, - ) - self.log.exception("failed while processing invalidation") + self._process_all(result) self.ctx.controller.wait(self.interval) def _synchronize(self): @@ -135,27 +126,48 @@ def _synchronize(self): if len(new_devices) == 0: self.log.info("no missing configurations found") - def _process(self, invalidation): + def _process_all(self, invalidations): + buildlimit_map = DevicePropertyMap.make_build_timeout_map( + self.ctx.dmd.Devices + ) + minttl_map = DevicePropertyMap.make_minimum_ttl_map( + self.ctx.dmd.Devices + ) + for invalidation in invalidations: + uid = invalidation.device.getPrimaryId() + buildlimit = buildlimit_map.get(uid) + minttl = minttl_map.get(uid) + try: + self._process(invalidation, buildlimit, minttl) + except AttributeError: + self.log.info( + "invalidation device=%s reason=%s", + invalidation.device, + invalidation.reason, + ) + self.log.exception("failed while processing invalidation") + + def _process(self, invalidation, buildlimit, minttl): device = invalidation.device reason = invalidation.reason monitor = device.getPerformanceServerName() if monitor is None: self.log.warn( "ignoring invalidated device having undefined collector " - "device=%s reason=%s", + "device=%s reason=%s", device, reason, ) return - keys = list( + keys = tuple( self.store.search(CacheQuery(monitor=monitor, device=device.id)) ) if not keys: - self._new_device(device, monitor) + self._new_device(device, monitor, buildlimit) elif reason is InvalidationCause.Updated: - self._updated_device(device, monitor, keys, invalidation) + self._updated_device(device, monitor, keys, minttl) elif reason is InvalidationCause.Removed: - self._removed_device(keys, invalidation) + self._removed_device(keys) else: self.log.warn( "ignored unexpected reason " @@ -166,30 +178,38 @@ def _process(self, invalidation): invalidation.oid, ) - def _new_device(self, device, monitor): - timelimitmap = DevicePropertyMap.make_build_timeout_map( - self.ctx.dmd.Devices + def _new_device(self, device, monitor, buildlimit): + # Don't dispatch jobs if there're any statuses. + keys = tuple( + CacheKey(svcname, monitor, device.id) + for svcname in self.dispatcher.service_names ) - uid = device.getPrimaryId() - timeout = timelimitmap.get(uid) - self.dispatcher.dispatch_all(monitor, device.id, timeout) + for key in keys: + status = self.store.get_status(key) + if status is not None: + self.log.debug( + "build jobs already submitted for new device " + "device=%s collector=%s", + device, + monitor, + ) + return + now = time.time() + for key in keys: + self.store.set_pending((key, now)) + self.dispatcher.dispatch_all(monitor, device.id, buildlimit) self.log.info( - "submitted build jobs for new device uid=%s collector=%s", - uid, + "submitted build jobs for new device device=%s collector=%s", + device, monitor, ) - def _updated_device(self, device, monitor, keys, invalidation): - minagelimitmap = DevicePropertyMap.make_minimum_ttl_map( - self.ctx.dmd.Devices - ) + def _updated_device(self, device, monitor, keys, minttl): statuses = tuple( status for status in self.store.get_status(*keys) if isinstance(status, ConfigStatus.Current) ) - uid = device.getPrimaryId() - minttl = minagelimitmap.get(uid) now = time.time() limit = now - minttl retired = set( @@ -198,38 +218,35 @@ def _updated_device(self, device, monitor, keys, invalidation): expired = set( status.key for status in statuses if status.key not in retired ) - retired = self.store.set_retired(*retired) now = time.time() - expired = self.store.set_expired(*((key, now) for key in expired)) + self.store.set_retired(*((key, now) for key in retired)) + self.store.set_expired(*((key, now) for key in expired)) for key in retired: self.log.info( "retired configuration of changed device " - "device=%s collector=%s service=%s device-oid=%r", + "device=%s collector=%s service=%s", key.device, key.monitor, key.service, - invalidation.oid, ) for key in expired: self.log.info( "expired configuration of changed device " - "device=%s collector=%s service=%s device-oid=%r", + "device=%s collector=%s service=%s", key.device, key.monitor, key.service, - invalidation.oid, ) - def _removed_device(self, keys, invalidation): + def _removed_device(self, keys): self.store.remove(*keys) for key in keys: self.log.info( "removed configuration of deleted device " - "device=%s collector=%s service=%s device-oid=%r", + "device=%s collector=%s service=%s", key.device, key.monitor, key.service, - invalidation.oid, ) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index e08809d5a4..822a1ffb7d 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -166,12 +166,13 @@ def _get_configs_to_rebuild(self): ttl_map = DevicePropertyMap.make_ttl_map(self.ctx.dmd.Devices) now = time() + ready_to_rebuild = [] + # Retrieve the 'retired' configs - ready_to_rebuild = list( - status - for status in self.store.get_retired() - if status.updated < now - minttl_map.get(status.uid) - ) + for status in self.store.get_retired(): + built = self.store.get_updated(status.key) + if built is None or built < now - minttl_map.get(status.uid): + ready_to_rebuild.append(status) # Append the 'expired' configs ready_to_rebuild.extend(self.store.get_expired()) @@ -224,7 +225,7 @@ def _rebuild_configs(self, statuses): datetime.fromtimestamp(status.updated).strftime( "%Y-%m-%d %H:%M:%S" ), - Constants.time_to_live_id, + Constants.build_timeout_id, timeout, status.key.service, status.key.monitor, diff --git a/Products/ZenCollector/configcache/propertymap.py b/Products/ZenCollector/configcache/propertymap.py index 6567d0330a..cf8be317dc 100644 --- a/Products/ZenCollector/configcache/propertymap.py +++ b/Products/ZenCollector/configcache/propertymap.py @@ -44,7 +44,7 @@ def make_minimum_ttl_map(cls, obj): obj, Constants.minimum_time_to_live_id, Constants.minimum_time_to_live_value, - _getZDeviceConfigMinimumTTL, + _getZProperty, ), Constants.minimum_time_to_live_value, ) @@ -138,19 +138,3 @@ def _getZProperty(obj, propname, default): if value is None: return default return value - - -def _getZDeviceConfigMinimumTTL(obj, propname, default): - """ - Compares zDeviceConfigTTL and zDeviceConfigMinimumTTL and - returns the lesser of the two. - """ - ttl = _getZProperty( - obj, Constants.time_to_live_id, Constants.time_to_live_value - ) - minttl = _getZProperty( - obj, - Constants.minimum_time_to_live_id, - Constants.minimum_time_to_live_value, - ) - return minttl if minttl < ttl else ttl diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 7d3832d03c..6acd9b2b41 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -80,7 +80,7 @@ def build_device_config( # Change the configuration's status from 'pending' to 'building' so # that configcache-manager doesn't prematurely timeout the build. - store.set_building((CacheKey(svcname, monitorname, deviceid), time())) + store.set_building((key, time())) self.log.info( "building device configuration device=%s collector=%s service=%s", deviceid, @@ -89,15 +89,16 @@ def build_device_config( ) service = svcconfigclass(self.dmd, monitorname) - configs = service.remote_getDeviceConfigs((deviceid,)) - if not configs: + result = service.remote_getDeviceConfigs((deviceid,)) + config = result[0] if result else None + if config is None: self.log.info( "no configuration built device=%s collector=%s service=%s", deviceid, monitorname, svcname, ) - key = next( + oldkey = next( store.search( CacheQuery( service=svcname, monitor=monitorname, device=deviceid @@ -105,9 +106,9 @@ def build_device_config( ), None, ) - if key is not None: + if oldkey is not None: # No result means device was deleted or moved to another monitor. - store.remove(key) + store.remove(oldkey) self.log.info( "removed previously built configuration " "device=%s collector=%s service=%s", @@ -115,8 +116,9 @@ def build_device_config( key.monitor, key.service, ) + # Ensure any status on this key is removed + store.clear_status(key) else: - config = configs[0] uid = self.dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() record = CacheRecord.make( svcname, monitorname, deviceid, uid, time(), config @@ -149,6 +151,10 @@ def __init__(self, configClasses): for cls in configClasses } + @property + def service_names(self): + return self._classnames.keys() + def dispatch_all(self, monitorid, deviceid, timeout): """ Submit a task to build a device configuration from each diff --git a/Products/ZenCollector/configcache/tests/test_propertymap_makers.py b/Products/ZenCollector/configcache/tests/test_propertymap_makers.py index f72f0d0fa0..6dfb6ddbbb 100644 --- a/Products/ZenCollector/configcache/tests/test_propertymap_makers.py +++ b/Products/ZenCollector/configcache/tests/test_propertymap_makers.py @@ -113,17 +113,16 @@ def test_make_min_ttl_map(t): actual = minttlmap.get(pathid) t.assertEqual(expected, actual) - def test_bad_min_ttl_value(t): - bad_minttl_value = Constants.time_to_live_value + 100 + def test_large_min_ttl_value(t): + minttl_value = Constants.time_to_live_value + 100 t.cmd_dev.setZenProperty( - Constants.minimum_time_to_live_id, - bad_minttl_value, + Constants.minimum_time_to_live_id, minttl_value ) minttlmap = DevicePropertyMap.make_minimum_ttl_map(t.dmd.Devices) pathid = t.cmd_dev.getPrimaryId() - expected = Constants.time_to_live_value + expected = Constants.time_to_live_value + 100 actual = minttlmap.get(pathid) t.assertEqual(expected, actual) diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py index 081362d0db..9acbab3e58 100644 --- a/Products/ZenCollector/configcache/tests/test_storage.py +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -83,6 +83,52 @@ def test_search_badarg(t): t.store.search("blargh") +class NoConfigTest(TestCase): + """Test statuses when no config is present.""" + + layer = RedisLayer + + key = CacheKey("a", "b", "c") + now = 12345.0 + + def setUp(t): + t.store = ConfigStore(t.layer.redis) + + def tearDown(t): + del t.store + + def test_current_status(t): + t.assertIsNone(next(t.store.get_status(t.key), None)) + + def test_search_with_status(t): + t.store.set_pending((t.key, t.now)) + t.assertEqual(0, len(tuple(t.store.search()))) + + def test_retired(t): + expected = ConfigStatus.Retired(t.key, None, t.now) + t.store.set_retired((t.key, t.now)) + status = next(t.store.get_status(t.key), None) + t.assertEqual(expected, status) + + def test_expired(t): + expected = ConfigStatus.Expired(t.key, None, t.now) + t.store.set_expired((t.key, t.now)) + status = next(t.store.get_status(t.key), None) + t.assertEqual(expected, status) + + def test_pending(t): + expected = ConfigStatus.Pending(t.key, None, t.now) + t.store.set_pending((t.key, t.now)) + status = next(t.store.get_status(t.key), None) + t.assertEqual(expected, status) + + def test_building(t): + expected = ConfigStatus.Building(t.key, None, t.now) + t.store.set_building((t.key, t.now)) + status = next(t.store.get_status(t.key), None) + t.assertEqual(expected, status) + + class _BaseTest(TestCase): # Base class to share setup code @@ -390,505 +436,592 @@ def test_get_newer_greater_multiple(t): t.assertEqual(t.record2.updated, status.updated) -class TestRetiredStatus(_BaseTest): +class SetStatusOnceTest(_BaseTest): """ - Test APIs regarding the ConfigStatus.Retired status. + Test the behavior when a set_ method is called once. """ - def test_set_retired(t): - t.store.add(t.record1) - expected = (t.record1.key,) + def test_retired_once(t): + ts = t.record1.updated + 100 + expected = ConfigStatus.Retired(t.record1.key, None, ts) + t.store.set_retired((t.record1.key, ts)) + + actual = next(t.store.get_retired(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + def test_expired_once(t): + ts = t.record1.updated + 100 + expected = ConfigStatus.Expired(t.record1.key, None, ts) + t.store.set_expired((t.record1.key, ts)) + + actual = next(t.store.get_expired(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + def test_pending_once(t): + ts = t.record1.updated + 100 + expected = ConfigStatus.Pending(t.record1.key, None, ts) + t.store.set_pending((t.record1.key, ts)) + + actual = next(t.store.get_pending(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + def test_building_once(t): + ts = t.record1.updated + 100 + expected = ConfigStatus.Building(t.record1.key, None, ts) + t.store.set_building((t.record1.key, ts)) + + actual = next(t.store.get_building(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) + + +class SetStatusTwiceTest(_BaseTest): + """ + Test the behavior when a set_ method is called twice + with different timestamp values. + """ - actual = t.store.set_retired(t.record1.key) - t.assertTupleEqual(expected, actual) + def test_retired_twice(t): + ts1 = t.record1.updated + 100 + ts2 = t.record1.updated + 200 + expected = ConfigStatus.Retired(t.record1.key, None, ts2) + t.store.set_retired((t.record1.key, ts1)) + t.store.set_retired((t.record1.key, ts2)) + + actual = next(t.store.get_retired(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + def test_expired_twice(t): + ts1 = t.record1.updated + 100 + ts2 = t.record1.updated + 200 + expected = ConfigStatus.Expired(t.record1.key, None, ts2) + t.store.set_expired((t.record1.key, ts1)) + t.store.set_expired((t.record1.key, ts2)) + + actual = next(t.store.get_expired(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + def test_pending_twice(t): + ts1 = t.record1.updated + 100 + ts2 = t.record1.updated + 200 + expected = ConfigStatus.Pending(t.record1.key, None, ts2) + t.store.set_pending((t.record1.key, ts1)) + t.store.set_pending((t.record1.key, ts2)) + + actual = next(t.store.get_pending(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + def test_building_twice(t): + ts1 = t.record1.updated + 100 + ts2 = t.record1.updated + 200 + expected = ConfigStatus.Building(t.record1.key, None, ts2) + t.store.set_building((t.record1.key, ts1)) + t.store.set_building((t.record1.key, ts2)) + + actual = next(t.store.get_building(), None) + t.assertEqual(expected, actual) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) + + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) + + +class TestCurrentOnlyMethods(_BaseTest): + """ + Verify that the get_older and get_newer methods work for Current status. + """ - def test_set_retired_twice(t): + def test_older_with_current(t): t.store.add(t.record1) - expected = () - t.store.set_retired(t.record1.key) - actual = t.store.set_retired(t.record1.key) - t.assertTupleEqual(expected, actual) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Current) + + older = next(t.store.get_older(t.record1.updated), None) + t.assertEqual(status, older) - def test_retired_status(t): + def test_older_with_retired(t): t.store.add(t.record1) - t.store.set_retired(t.record1.key) + ts = t.record1.updated + 500 + t.store.set_retired((t.record1.key, ts)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) + status = next(t.store.get_status(t.record1.key), None) t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(status.updated, t.record1.updated) - def test_get_retired(t): - t.store.add(t.record1) - t.store.set_retired(t.record1.key) + older = next(t.store.get_older(t.record1.updated), None) + t.assertIsNone(older) - result = tuple(t.store.get_retired()) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(status.updated, t.record1.updated) + def test_older_with_expired(t): + t.store.add(t.record1) + ts = t.record1.updated + 500 + t.store.set_expired((t.record1.key, ts)) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Expired) -class TestExpiredStatus(_BaseTest): - """ - Test APIs regarding the ConfigStatus.Expired status. - """ + older = next(t.store.get_older(t.record1.updated), None) + t.assertIsNone(older) - def test_set_expired(t): + def test_older_with_pending(t): t.store.add(t.record1) ts = t.record1.updated + 500 + t.store.set_pending((t.record1.key, ts)) - expected = (t.record1.key,) - actual = t.store.set_expired((t.record1.key, ts)) - t.assertTupleEqual(expected, actual) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Pending) + + older = next(t.store.get_older(t.record1.updated), None) + t.assertIsNone(older) - def test_set_expired_twice(t): + def test_older_with_building(t): t.store.add(t.record1) ts = t.record1.updated + 500 + t.store.set_building((t.record1.key, ts)) - expected = () + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Building) - t.store.set_expired((t.record1.key, ts)) - actual = t.store.set_expired((t.record1.key, ts)) - t.assertTupleEqual(expected, actual) + older = next(t.store.get_older(t.record1.updated), None) + t.assertIsNone(older) - def test_expired_status(t): + def test_newer_with_current(t): + t.store.add(t.record1) + + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Current) + + newer = next(t.store.get_newer(t.record1.updated - 1), None) + t.assertEqual(status, newer) + + def test_newer_with_retired(t): t.store.add(t.record1) ts = t.record1.updated + 500 - t.store.set_expired((t.record1.key, ts)) + t.store.set_retired((t.record1.key, ts)) - result = tuple(t.store.get_status(t.record1.key)) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Expired) + newer = next(t.store.get_newer(t.record1.updated - 1), None) + t.assertIsNone(newer) - def test_get_expired(t): + def test_newer_with_expired(t): t.store.add(t.record1) ts = t.record1.updated + 500 t.store.set_expired((t.record1.key, ts)) - result = tuple(t.store.get_expired()) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) + status = next(t.store.get_status(t.record1.key), None) t.assertIsInstance(status, ConfigStatus.Expired) - def test_expired_is_not_older(t): + newer = next(t.store.get_newer(t.record1.updated - 1), None) + t.assertIsNone(newer) + + def test_newer_with_pending(t): t.store.add(t.record1) ts = t.record1.updated + 500 - t.store.set_expired((t.record1.key, ts)) - - result = tuple(t.store.get_older(t.record1.updated)) - t.assertEqual(0, len(result)) + t.store.set_pending((t.record1.key, ts)) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Pending) -class TestPendingStatus(_BaseTest): - """ - Test APIs regarding the ConfigStatus.Pending status. - """ + newer = next(t.store.get_newer(t.record1.updated - 1), None) + t.assertIsNone(newer) - def test_set_pending(t): + def test_newer_with_building(t): t.store.add(t.record1) - submitted = t.record1.updated + 500 - expected = (t.record1.key,) - - actual = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual(expected, actual) + ts = t.record1.updated + 500 + t.store.set_building((t.record1.key, ts)) - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Building) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + newer = next(t.store.get_newer(t.record1.updated - 1), None) + t.assertIsNone(newer) - def test_set_pending_twice(t): - t.store.add(t.record1) - submitted = t.record1.updated + 500 - expected = () - t.store.set_pending((t.record1.key, submitted)) - actual = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual(expected, actual) +class GetStatusTest(_BaseTest): + """ + Verify that get_status returns all the statuses. + """ - def test_pending_status(t): + def test_current(t): t.store.add(t.record1) - submitted = t.record1.updated + 500 - t.store.set_pending((t.record1.key, submitted)) - - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Pending) - t.assertEqual(submitted, status.submitted) + expected = ConfigStatus.Current( + t.record1.key, t.record1.uid, t.record1.updated + ) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) - def test_get_pending(t): + def test_retired(t): t.store.add(t.record1) - submitted = t.record1.updated + 500 - t.store.set_pending((t.record1.key, submitted)) + ts = t.record1.updated + 100 + t.store.set_retired((t.record1.key, ts)) + expected = ConfigStatus.Retired(t.record1.key, t.record1.uid, ts) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) - result = tuple(t.store.get_pending()) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Pending) - t.assertEqual(submitted, status.submitted) + def test_expired(t): + t.store.add(t.record1) + ts = t.record1.updated + 200 + t.store.set_expired((t.record1.key, ts)) + expected = ConfigStatus.Expired(t.record1.key, t.record1.uid, ts) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) - def test_pending_is_not_older(t): + def test_pending(t): t.store.add(t.record1) - submitted = t.record1.updated + 500 - t.store.set_pending((t.record1.key, submitted)) + ts = t.record1.updated + 300 + t.store.set_pending((t.record1.key, ts)) + expected = ConfigStatus.Pending(t.record1.key, t.record1.uid, ts) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) - result = tuple(t.store.get_older(t.record1.updated)) - t.assertEqual(0, len(result)) + def test_building(t): + t.store.add(t.record1) + ts = t.record1.updated + 400 + t.store.set_building((t.record1.key, ts)) + expected = ConfigStatus.Building(t.record1.key, t.record1.uid, ts) + actual = next(t.store.get_status(t.record1.key), None) + t.assertEqual(expected, actual) -class TestBuildingStatus(_BaseTest): +class TestClearStatus(_BaseTest): """ - Test APIs regarding the ConfigStatus.Building status. + Test clearing the status. """ - def test_set_building(t): + def test_clear_from_current(t): t.store.add(t.record1) - started = t.record1.updated + 500 - expected = (t.record1.key,) - - t.store.set_pending((t.record1.key, started - 100)) - actual = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual(expected, actual) + t.store.clear_status(t.record1.key) - pending_keys = tuple(t.store.get_pending()) - t.assertTupleEqual((), pending_keys) - - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Current) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + t.assertIsNone(next(t.store.get_retired(), None)) + t.assertIsNone(next(t.store.get_expired(), None)) + t.assertIsNone(next(t.store.get_pending(), None)) + t.assertIsNone(next(t.store.get_building(), None)) - def test_set_building_twice(t): + def test_clear_from_expired_to_current(t): t.store.add(t.record1) - started = t.record1.updated + 500 - expected = () + ts = t.record1.updated + 100 + t.store.set_expired((t.record1.key, ts)) - t.store.set_building((t.record1.key, started)) - actual = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual(expected, actual) + t.store.clear_status(t.record1.key) - def test_building_status(t): - t.store.add(t.record1) - started = t.record1.updated + 500 - t.store.set_building((t.record1.key, started)) + status = next(t.store.get_status(t.record1.key), None) + t.assertIsInstance(status, ConfigStatus.Current) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + t.assertIsNone(next(t.store.get_retired(), None)) + t.assertIsNone(next(t.store.get_expired(), None)) + t.assertIsNone(next(t.store.get_pending(), None)) + t.assertIsNone(next(t.store.get_building(), None)) - def test_get_building(t): - t.store.add(t.record1) - started = t.record1.updated + 500 - t.store.set_building((t.record1.key, started)) + def test_clear_from_retired(t): + retired = t.record1.updated + 100 + t.store.set_retired((t.record1.key, retired)) - result = tuple(t.store.get_building()) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + t.store.clear_status(t.record1.key) - def test_building_is_not_older(t): - t.store.add(t.record1) - started = t.record1.updated + 500 - t.store.set_building((t.record1.key, started)) + t.assertIsNone(next(t.store.get_status(t.record1.key), None)) + t.assertIsNone(next(t.store.get_retired(), None)) + t.assertIsNone(next(t.store.get_expired(), None)) + t.assertIsNone(next(t.store.get_pending(), None)) + t.assertIsNone(next(t.store.get_building(), None)) - result = tuple(t.store.get_older(t.record1.updated)) - t.assertEqual(0, len(result)) + def test_clear_from_expired(t): + ts = t.record1.updated + 100 + t.store.set_expired((t.record1.key, ts)) + t.store.clear_status(t.record1.key) -class TestExpiredTransitions(_BaseTest): - """ - Test transitions to and from ConfigStatus.Expired. - """ + t.assertIsNone(next(t.store.get_status(t.record1.key), None)) + t.assertIsNone(next(t.store.get_retired(), None)) + t.assertIsNone(next(t.store.get_expired(), None)) + t.assertIsNone(next(t.store.get_pending(), None)) + t.assertIsNone(next(t.store.get_building(), None)) - def test_retired_to_expired(t): - t.store.add(t.record1) - t.store.set_retired(t.record1.key) - ts = t.record1.updated + 300 + def test_clear_from_pending(t): + ts = t.record1.updated + 100 + t.store.set_pending((t.record1.key, ts)) - expired_keys = t.store.set_expired((t.record1.key, ts)) - t.assertTupleEqual((t.record1.key,), expired_keys) + t.store.clear_status(t.record1.key) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Expired) + t.assertIsNone(next(t.store.get_status(t.record1.key), None)) + t.assertIsNone(next(t.store.get_retired(), None)) + t.assertIsNone(next(t.store.get_expired(), None)) + t.assertIsNone(next(t.store.get_pending(), None)) + t.assertIsNone(next(t.store.get_building(), None)) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + def test_clear_from_building(t): + ts = t.record1.updated + 100 + t.store.set_building((t.record1.key, ts)) - def test_expired_to_retired(t): - t.store.add(t.record1) - ts = t.record1.updated + 300 - t.store.set_expired((t.record1.key, ts)) - retired_keys = t.store.set_retired(t.record1.key) - t.assertTupleEqual((t.record1.key,), retired_keys) + t.store.clear_status(t.record1.key) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(t.record1.updated, status.updated) + t.assertIsNone(next(t.store.get_status(t.record1.key), None)) + t.assertIsNone(next(t.store.get_retired(), None)) + t.assertIsNone(next(t.store.get_expired(), None)) + t.assertIsNone(next(t.store.get_pending(), None)) + t.assertIsNone(next(t.store.get_building(), None)) -class TestPendingTransitions(_BaseTest): +class TestStatusChangesFromRetired(_BaseTest): """ - Test transitions to and from ConfigStatus.Pending. + Test changing the status of a config. """ - def test_current_to_pending(t): - t.store.add(t.record1) - submitted = t.record1.updated + 500 - - pending_keys = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual((t.record1.key,), pending_keys) + def test_retired_to_expired(t): + retired = t.record1.updated + 100 + t.store.set_retired((t.record1.key, retired)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Pending) - t.assertEqual(submitted, status.submitted) + expired = t.record1.updated + 300 + t.store.set_expired((t.record1.key, expired)) - def test_retired_to_pending(t): - t.store.add(t.record1) - t.store.set_retired(t.record1.key) - submitted = t.record1.updated + 500 + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) - pending_keys = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual((t.record1.key,), pending_keys) + expected = ConfigStatus.Expired(t.record1.key, None, expired) + actual = next(t.store.get_expired(), None) + t.assertEqual(expected, actual) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Pending) - t.assertEqual(submitted, status.submitted) + def test_retired_to_pending(t): + retired = t.record1.updated + 100 + t.store.set_retired((t.record1.key, retired)) - def test_expired_to_pending(t): - t.store.add(t.record1) - ts = t.record1.updated + 300 - submitted = t.record1.updated + 500 - t.store.set_expired((t.record1.key, ts)) - pending_keys = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual((t.record1.key,), pending_keys) + pending = t.record1.updated + 300 + t.store.set_pending((t.record1.key, pending)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Pending) + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) + expected = ConfigStatus.Pending(t.record1.key, None, pending) + actual = next(t.store.get_pending(), None) + t.assertEqual(expected, actual) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + def test_retired_to_building(t): + retired = t.record1.updated + 100 + t.store.set_retired((t.record1.key, retired)) - building_keys = tuple(t.store.get_building()) - t.assertTupleEqual((), building_keys) + building = t.record1.updated + 300 + t.store.set_building((t.record1.key, building)) - def test_pending_to_expired(t): - t.store.add(t.record1) - ts = t.record1.updated + 300 - submitted = t.record1.updated + 500 - t.store.set_pending((t.record1.key, submitted)) + actual = next(t.store.get_retired(), None) + t.assertIsNone(actual) - expired_keys = t.store.set_expired((t.record1.key, ts)) - t.assertTupleEqual((t.record1.key,), expired_keys) + expected = ConfigStatus.Building(t.record1.key, None, building) + actual = next(t.store.get_building(), None) + t.assertEqual(expected, actual) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Expired) - t.assertEqual(ts, status.expired) - def test_pending_to_retired(t): - t.store.add(t.record1) - submitted = t.record1.updated + 500 - t.store.set_pending((t.record1.key, submitted)) +class TestStatusChangesFromExpired(_BaseTest): + """ + Test changing the status of a config. + """ - retired_keys = t.store.set_retired(t.record1.key) - t.assertTupleEqual((t.record1.key,), retired_keys) + def test_expired_to_retired(t): + expired = t.record1.updated + 100 + t.store.set_expired((t.record1.key, expired)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(t.record1.updated, status.updated) + retired = t.record1.updated + 300 + t.store.set_retired((t.record1.key, retired)) + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) -class TestBuildingTransitions(_BaseTest): - """ - Test transitions to and from ConfigStatus.Building. - """ + expected = ConfigStatus.Retired(t.record1.key, None, retired) + actual = next(t.store.get_retired(), None) + t.assertEqual(expected, actual) - def test_current_to_building(t): - t.store.add(t.record1) - started = t.record1.updated + 500 + def test_expired_to_pending(t): + expired = t.record1.updated + 100 + t.store.set_expired((t.record1.key, expired)) - building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((t.record1.key,), building_keys) + pending = t.record1.updated + 300 + t.store.set_pending((t.record1.key, pending)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) - pending_keys = tuple(t.store.get_pending()) - t.assertTupleEqual((), pending_keys) + expected = ConfigStatus.Pending(t.record1.key, None, pending) + actual = next(t.store.get_pending(), None) + t.assertEqual(expected, actual) - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) + def test_expired_to_building(t): + expired = t.record1.updated + 100 + t.store.set_expired((t.record1.key, expired)) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + building = t.record1.updated + 300 + t.store.set_building((t.record1.key, building)) - def test_retired_to_building(t): - t.store.add(t.record1) - t.store.set_retired(t.record1.key) - started = t.record1.updated + 500 + actual = next(t.store.get_expired(), None) + t.assertIsNone(actual) - building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((t.record1.key,), building_keys) + expected = ConfigStatus.Building(t.record1.key, None, building) + actual = next(t.store.get_building(), None) + t.assertEqual(expected, actual) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) - pending_keys = tuple(t.store.get_pending()) - t.assertTupleEqual((), pending_keys) +class TestStatusChangesFromPending(_BaseTest): + """ + Test changing the status of a config. + """ - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) + def test_pending_to_retired(t): + pending = t.record1.updated + 100 + t.store.set_pending((t.record1.key, pending)) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + retired = t.record1.updated + 300 + t.store.set_retired((t.record1.key, retired)) - def test_expired_to_building(t): - t.store.add(t.record1) - ts = t.record1.updated + 300 - started = t.record1.updated + 500 - t.store.set_expired((t.record1.key, ts)) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) - building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((t.record1.key,), building_keys) + expected = ConfigStatus.Retired(t.record1.key, None, retired) + actual = next(t.store.get_retired(), None) + t.assertEqual(expected, actual) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + def test_pending_to_expired(t): + pending = t.record1.updated + 100 + t.store.set_pending((t.record1.key, pending)) - pending_keys = tuple(t.store.get_pending()) - t.assertTupleEqual((), pending_keys) + expired = t.record1.updated + 300 + t.store.set_expired((t.record1.key, expired)) - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) + expected = ConfigStatus.Expired(t.record1.key, None, expired) + actual = next(t.store.get_expired(), None) + t.assertEqual(expected, actual) def test_pending_to_building(t): - t.store.add(t.record1) - ts = t.record1.updated + 300 - started = t.record1.updated + 500 - t.store.set_expired((t.record1.key, ts)) - t.store.set_pending((t.record1.key, started - 100)) + pending = t.record1.updated + 100 + t.store.set_pending((t.record1.key, pending)) - building_keys = t.store.set_building((t.record1.key, started)) - t.assertTupleEqual((t.record1.key,), building_keys) + building = t.record1.updated + 300 + t.store.set_building((t.record1.key, building)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Building) - t.assertEqual(started, status.started) + actual = next(t.store.get_pending(), None) + t.assertIsNone(actual) - pending_keys = tuple(t.store.get_pending()) - t.assertTupleEqual((), pending_keys) + expected = ConfigStatus.Building(t.record1.key, None, building) + actual = next(t.store.get_building(), None) + t.assertEqual(expected, actual) - expired_keys = tuple(t.store.get_expired()) - t.assertTupleEqual((), expired_keys) - retired_keys = tuple(t.store.get_retired()) - t.assertTupleEqual((), retired_keys) +class TestStatusChangesFromBuilding(_BaseTest): + """ + Test changing the status of a config. + """ - def test_building_to_pending(t): - t.store.add(t.record1) - submitted = t.record1.updated + 300 - started = t.record1.updated + 500 - t.store.set_building((t.record1.key, started)) + def test_building_to_retired(t): + building = t.record1.updated + 100 + t.store.set_building((t.record1.key, building)) - pending_keys = t.store.set_pending((t.record1.key, submitted)) - t.assertTupleEqual((t.record1.key,), pending_keys) + retired = t.record1.updated + 300 + t.store.set_retired((t.record1.key, retired)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Pending) - t.assertEqual(submitted, status.submitted) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + expected = ConfigStatus.Retired(t.record1.key, None, retired) + actual = next(t.store.get_retired(), None) + t.assertEqual(expected, actual) def test_building_to_expired(t): - t.store.add(t.record1) + building = t.record1.updated + 100 + t.store.set_building((t.record1.key, building)) + expired = t.record1.updated + 300 - started = t.record1.updated + 500 - t.store.set_building((t.record1.key, started)) + t.store.set_expired((t.record1.key, expired)) - expired_keys = t.store.set_expired((t.record1.key, expired)) - t.assertTupleEqual((t.record1.key,), expired_keys) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Expired) - t.assertEqual(expired, status.expired) + expected = ConfigStatus.Expired(t.record1.key, None, expired) + actual = next(t.store.get_expired(), None) + t.assertEqual(expected, actual) - def test_building_to_retired(t): - t.store.add(t.record1) - started = t.record1.updated + 500 - t.store.set_building((t.record1.key, started)) + def test_building_to_pending(t): + building = t.record1.updated + 100 + t.store.set_building((t.record1.key, building)) - retired_keys = t.store.set_retired(t.record1.key) - t.assertTupleEqual((t.record1.key,), retired_keys) + pending = t.record1.updated + 300 + t.store.set_pending((t.record1.key, pending)) - result = tuple(t.store.get_status(t.record1.key)) - t.assertEqual(1, len(result)) - status = result[0] - t.assertEqual(t.record1.key, status.key) - t.assertIsInstance(status, ConfigStatus.Retired) - t.assertEqual(t.record1.updated, status.updated) + actual = next(t.store.get_building(), None) + t.assertIsNone(actual) + + expected = ConfigStatus.Pending(t.record1.key, None, pending) + actual = next(t.store.get_pending(), None) + t.assertEqual(expected, actual) class TestAddTransitions(_BaseTest): @@ -898,7 +1031,8 @@ class TestAddTransitions(_BaseTest): def test_add_overwrites_retired(t): t.store.add(t.record1) - t.store.set_retired(t.record1.key) + retired = t.record1.updated + 100 + t.store.set_retired((t.record1.key, retired)) t.store.add(t.record1) retired_keys = tuple(t.store.get_retired()) @@ -1055,11 +1189,13 @@ def test_uid_after_one_removal(t): t.store.add(t.record2) t.store.remove(t.record1.key) + actual = t.store.get_uid(t.device_name) + t.assertEqual(t.device_uid, actual) + records = tuple( t.store.get(key) for key in t.store.search(CacheQuery(device=t.device_name)) ) - t.assertEqual(1, len(records)) t.assertEqual(t.device_uid, records[0].uid) @@ -1072,7 +1208,6 @@ def test_uid_after_removing_all(t): t.store.get(key) for key in t.store.search(CacheQuery(device=t.device_name)) ) - t.assertEqual(0, len(records)) t.assertIsNone(t.store.get_uid(t.device_name)) @@ -1085,11 +1220,9 @@ def _make_config(_id, configId, guid): return config -# _compare_configs used to monkeypatch DeviceProxy -# to make equivalent instances equal. - - def _compare_configs(self, cfg): + # _compare_configs used to monkeypatch DeviceProxy + # to make equivalent instances equal. return all( ( self.id == cfg.id, From dabd81b50b9ef97ba586ab70e4e4224ed6dbbc66 Mon Sep 17 00:00:00 2001 From: Josh Wilmes Date: Fri, 16 Feb 2024 15:16:27 -0600 Subject: [PATCH 088/147] Move initialize_zenoss_env into worker_process_init signal handler, so that it runs in the worker process, and not in the main zenjobs one. --- Products/Jobber/model.py | 19 ++++++++++++++- Products/Jobber/signals.zcml | 7 ++++++ Products/Jobber/task/utils.py | 3 ++- .../Jobber/tests/test_jobstore_updates.py | 12 +++++----- Products/Jobber/worker.py | 23 ++++++++++++++++++- Products/Jobber/zenjobs.py | 6 +++++ bin/zenjobs | 17 -------------- 7 files changed, 61 insertions(+), 26 deletions(-) diff --git a/Products/Jobber/model.py b/Products/Jobber/model.py index 6318c30d00..bfba76bc6d 100644 --- a/Products/Jobber/model.py +++ b/Products/Jobber/model.py @@ -27,7 +27,6 @@ from .storage import Fields from .task.utils import job_log_has_errors from .utils.log import inject_logger -from .zenjobs import app mlog = logging.getLogger("zen.zenjobs.model") @@ -121,6 +120,8 @@ def job_name(self): @property def job_type(self): + from .zenjobs import app + task = app.tasks.get(self.name) if task is None: return self.name if self.name else "" @@ -153,6 +154,8 @@ def wait(self): @property def result(self): + from .zenjobs import app + return app.tasks[self.name].AsyncResult(self.jobid) def __eq__(self, other): @@ -288,6 +291,8 @@ def from_signal(cls, body, headers, properties): @classmethod def _build(cls, jobid, taskname, args, kwargs, headers, properties): + from .zenjobs import app + task = app.tasks[taskname] fields = {} description = properties.pop("description", None) @@ -313,6 +318,8 @@ def save_jobrecord(log, body=None, headers=None, properties=None, **ignored): :param dict headers: Headers to accompany message sent to Celery worker :param dict properties: Additional task and custom key/value pairs """ + from .zenjobs import app + if not body: # If body is empty (or None), no job to save. log.info("no body, so no job") @@ -387,6 +394,8 @@ def stage_jobrecord(log, storage, sig): :param sig: The job data :type sig: celery.canvas.Signature """ + from .zenjobs import app + task = app.tasks.get(sig.task) # Tasks with ignored results cannot be tracked, @@ -412,6 +421,8 @@ def commit_jobrecord(log, storage, sig): :param sig: The job data :type sig: celery.canvas.Signature """ + from .zenjobs import app + task = app.tasks.get(sig.task) # Tasks with ignored results cannot be tracked, @@ -493,6 +504,8 @@ def job_end(log, task_id, task=None, **ignored): @inject_logger(log=mlog) @_catch_exception def job_success(log, result, sender=None, **ignored): + from .zenjobs import app + if sender is not None and sender.ignore_result: return task_id = sender.request.id @@ -509,6 +522,8 @@ def job_success(log, result, sender=None, **ignored): @inject_logger(log=mlog) @_catch_exception def job_failure(log, task_id, exception=None, sender=None, **ignored): + from .zenjobs import app + if sender is not None and sender.ignore_result: return status = app.backend.get_status(task_id) @@ -539,6 +554,8 @@ def job_failure(log, task_id, exception=None, sender=None, **ignored): @inject_logger(log=mlog) @_catch_exception def job_retry(log, request, reason=None, sender=None, **ignored): + from .zenjobs import app + if sender is not None and sender.ignore_result: return jobstore = getUtility(IJobStore, "redis") diff --git a/Products/Jobber/signals.zcml b/Products/Jobber/signals.zcml index 716ac4e83b..7a62de78f5 100644 --- a/Products/Jobber/signals.zcml +++ b/Products/Jobber/signals.zcml @@ -1,6 +1,8 @@ + + + + Date: Tue, 5 Mar 2024 09:24:40 -0600 Subject: [PATCH 089/147] fix: DevicePropertyMap now returns default when given None. ZEN-34745 --- .../ZenCollector/configcache/propertymap.py | 4 +++ .../configcache/tests/test_propertymap.py | 30 ++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Products/ZenCollector/configcache/propertymap.py b/Products/ZenCollector/configcache/propertymap.py index cf8be317dc..689e3ed8c4 100644 --- a/Products/ZenCollector/configcache/propertymap.py +++ b/Products/ZenCollector/configcache/propertymap.py @@ -90,6 +90,10 @@ def smallest_value(self): return self.__default def get(self, request_uid): + # Be graceful on accepted input values. None is equivalent + # to no match so return the default value. + if request_uid is None: + return self.__default # Split the request into its parts req_parts = request_uid.split("/")[1:] # Find all the path parts that match the request diff --git a/Products/ZenCollector/configcache/tests/test_propertymap.py b/Products/ZenCollector/configcache/tests/test_propertymap.py index 9d6455455d..14442a69e3 100644 --- a/Products/ZenCollector/configcache/tests/test_propertymap.py +++ b/Products/ZenCollector/configcache/tests/test_propertymap.py @@ -18,16 +18,16 @@ class EmptyDevicePropertyMapTest(TestCase): """Test an empty DevicePropertyMap object.""" def setUp(t): - t.bmm = DevicePropertyMap({}, None) + t.dpm = DevicePropertyMap({}, None) def tearDown(t): - del t.bmm + del t.dpm def test_get(t): - t.assertIsNone(t.bmm.get("/zport/dmd/Devices")) + t.assertIsNone(t.dpm.get("/zport/dmd/Devices")) def test_smallest_value(t): - t.assertIsNone(t.bmm.smallest_value()) + t.assertIsNone(t.dpm.smallest_value()) class DevicePropertyMapTest(TestCase): @@ -44,29 +44,37 @@ class DevicePropertyMapTest(TestCase): _default = 15 def setUp(t): - t.bmm = DevicePropertyMap(t.mapping, t._default) + t.dpm = DevicePropertyMap(t.mapping, t._default) def tearDown(t): - del t.bmm + del t.dpm def test_minimal_match(t): - value = t.bmm.get("/zport/dmd/Devices/Server-stuff/devices/dev2") + value = t.dpm.get("/zport/dmd/Devices/Server-stuff/devices/dev2") t.assertEqual(10, value) def test_get_exact_match(t): - value = t.bmm.get( + value = t.dpm.get( "/zport/dmd/Devices/Server/SSH/Linux/devices/my-device" ) t.assertEqual(12, value) def test_get_best_match(t): - value = t.bmm.get("/zport/dmd/Devices/Server/Linux/devices/dev3") + value = t.dpm.get("/zport/dmd/Devices/Server/Linux/devices/dev3") t.assertEqual(11, value) def test_no_match(t): - value = t.bmm.get("/Devices") + value = t.dpm.get("/Devices") + t.assertEqual(t._default, value) + + def test_empty_string(t): + value = t.dpm.get("") t.assertEqual(t._default, value) def test_smallest_value(t): - value = t.bmm.smallest_value() + value = t.dpm.smallest_value() t.assertEqual(10, value) + + def test_uid_is_None(t): + value = t.dpm.get(None) + t.assertEqual(t._default, value) From b612fc92ebc9b456fd7b724717f91baaff0258ce Mon Sep 17 00:00:00 2001 From: Josh Wilmes Date: Tue, 5 Mar 2024 10:20:10 -0600 Subject: [PATCH 090/147] Update references to self._prefs in hub services to use self.conf instead. --- Products/ZenHub/services/ProcessConfig.py | 2 +- Products/ZenHub/services/ZenStatusConfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Products/ZenHub/services/ProcessConfig.py b/Products/ZenHub/services/ProcessConfig.py index f6296c9e8c..c9151bd84f 100644 --- a/Products/ZenHub/services/ProcessConfig.py +++ b/Products/ZenHub/services/ProcessConfig.py @@ -83,7 +83,7 @@ def _createDeviceProxy(self, device): return None proxy = CollectorConfigService._createDeviceProxy(self, device) - proxy.configCycleInterval = self._prefs.processCycleInterval + proxy.configCycleInterval = self.conf.processCycleInterval proxy.name = device.id proxy.lastmodeltime = device.getLastChangeString() diff --git a/Products/ZenHub/services/ZenStatusConfig.py b/Products/ZenHub/services/ZenStatusConfig.py index a48e20525b..8d85e405d2 100644 --- a/Products/ZenHub/services/ZenStatusConfig.py +++ b/Products/ZenHub/services/ZenStatusConfig.py @@ -72,7 +72,7 @@ def _filterDevice(self, device): def _createDeviceProxy(self, device): proxy = CollectorConfigService._createDeviceProxy(self, device) - proxy.configCycleInterval = self._prefs.statusCycleInterval + proxy.configCycleInterval = self.conf.statusCycleInterval # add each component proxy.components = [] From de7229b22ee21cfdce708e93567384feebd73575 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 5 Mar 2024 11:00:15 -0600 Subject: [PATCH 091/147] fix: use get_status method correctly. The get_status method returns a generator which must be iterated to retrieve the values. ZEN-34747 --- Products/ZenCollector/configcache/invalidator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 8008852d86..cbfde8d799 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -185,12 +185,12 @@ def _new_device(self, device, monitor, buildlimit): for svcname in self.dispatcher.service_names ) for key in keys: - status = self.store.get_status(key) + status = next(self.store.get_status(key), None) if status is not None: self.log.debug( "build jobs already submitted for new device " "device=%s collector=%s", - device, + device.id, monitor, ) return @@ -200,7 +200,7 @@ def _new_device(self, device, monitor, buildlimit): self.dispatcher.dispatch_all(monitor, device.id, buildlimit) self.log.info( "submitted build jobs for new device device=%s collector=%s", - device, + device.id, monitor, ) From 1c22ee8eb3aaecb2d675d6b6ccc144f00b073a0a Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 6 Mar 2024 12:22:34 -0600 Subject: [PATCH 092/147] fix: configure logging for stdout in zencyberark Also renamed the '--debug' option to '--verbose'. ZEN-34750 --- Products/ZenCollector/zencyberark.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Products/ZenCollector/zencyberark.py b/Products/ZenCollector/zencyberark.py index c64e9cb246..dbcc488624 100644 --- a/Products/ZenCollector/zencyberark.py +++ b/Products/ZenCollector/zencyberark.py @@ -13,6 +13,7 @@ import httplib import json import logging +import sys from twisted.internet.defer import inlineCallbacks, maybeDeferred from twisted.internet.task import react @@ -20,14 +21,20 @@ from .cyberark import get_cyberark -def configure_logging(debug=False): - log = logging.getLogger() - log.setLevel(logging.INFO if not debug else logging.DEBUG) +def configure_logging(verbose=False): + logging._handlers.clear() + formatter = logging.Formatter("%(levelname)s: %(message)s") + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(formatter) + + root = logging.getLogger() + root.setLevel(logging.INFO if not verbose else logging.DEBUG) + root.addHandler(handler) return logging.getLogger("zencyberark") def check(args): - log = configure_logging(args.debug) + log = configure_logging(args.verbose) log.info("Checking config") get_cyberark() return 0 @@ -35,7 +42,7 @@ def check(args): @inlineCallbacks def get(args): - log = configure_logging(args.debug) + log = configure_logging(args.verbose) cyberark = get_cyberark() client = cyberark._manager._client @@ -92,7 +99,13 @@ def parse_args(): description="Model Catalog hacking tool", formatter_class=argparse.RawTextHelpFormatter, ) - parser.add_argument("-d", "--debug", action="store_true", default=False) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="display more information", + ) subparsers = parser.add_subparsers(help="sub-command help") check_cmd = subparsers.add_parser( From bc72d3119d0772c5c23898d397eb95f98aedeb2f Mon Sep 17 00:00:00 2001 From: Josh Wilmes Date: Wed, 6 Mar 2024 15:18:22 -0600 Subject: [PATCH 093/147] Raise celery worker process alive timeout from the default of 4 seconds to 5 minutes, to allow plenty of time to load the zenoss environment. --- Products/Jobber/zenjobs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Products/Jobber/zenjobs.py b/Products/Jobber/zenjobs.py index 2ace101da3..fbba06b771 100644 --- a/Products/Jobber/zenjobs.py +++ b/Products/Jobber/zenjobs.py @@ -35,3 +35,14 @@ config_source="Products.Jobber.config:Celery", task_cls="Products.Jobber.task:ZenTask", ) + +# Allow considerably more time for the worker_process_init signal +# to complete (rather than the default of 4 seconds). This is required +# because loading the zenoss environment / zenpacks can take a while. + +# celery 3.1.26 (remove once we update celery) +from celery.concurrency import asynpool +asynpool.PROC_ALIVE_TIMEOUT = 300 + +# celery 4.4.0+ +# app.conf.worker_proc_alive_timeout = 300 \ No newline at end of file From 357658b7b2d1033ee232f75f5cd36c98d05477d2 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 7 Mar 2024 11:33:51 -0600 Subject: [PATCH 094/147] fix: addLogsFromConfigFile no longer clears out loggers ZEN-34747 --- Products/ZenUtils/configlog.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Products/ZenUtils/configlog.py b/Products/ZenUtils/configlog.py index ccc2ca0c40..2999b8544b 100644 --- a/Products/ZenUtils/configlog.py +++ b/Products/ZenUtils/configlog.py @@ -109,11 +109,8 @@ def addLogsFromConfigFile(fname, configDefaults=None): log.exception('Problem loading log configuration file: %s', fname) return False - # critical section - logging._acquireLock() try: - logging._handlers.clear() - del logging._handlerList[:] + logging._acquireLock() # Handlers add themselves to logging._handlers handlers = logging.config._install_handlers(cp, formatters) _zen_install_loggers(cp, handlers) From eb491085043113af28430908cb888ae930f4b6a5 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 7 Mar 2024 13:43:24 -0600 Subject: [PATCH 095/147] fix: add absolute import support to CheckRelations.py Also added argument checking to checkrel.py ZEN-34580 --- Products/ZenRelations/checkrel.py | 12 ++++-- Products/ZenUtils/CheckRelations.py | 66 +++++++++++++++-------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/Products/ZenRelations/checkrel.py b/Products/ZenRelations/checkrel.py index a661e163b8..2ecfa61de7 100644 --- a/Products/ZenRelations/checkrel.py +++ b/Products/ZenRelations/checkrel.py @@ -7,6 +7,8 @@ # ############################################################################## +from __future__ import print_function + import logging import sys @@ -39,7 +41,7 @@ def checkRelationshipSchema(cls, baseModule): for relname, rel in cls._relations: try: remoteClass = importClass(rel.remoteClass, None) - except AttributeError as e: + except AttributeError: logging.critical( "RemoteClass '%s' from '%s.%s' not found", rel.remoteClass, @@ -49,7 +51,7 @@ def checkRelationshipSchema(cls, baseModule): continue try: rschema = lookupSchema(remoteClass, rel.remoteName) - except ZenSchemaError as e: + except ZenSchemaError: logging.critical( "Inverse def '%s' for '%s.%s' not found on '%s'", rel.remoteName, @@ -100,7 +102,11 @@ def checkRelationshipSchema(cls, baseModule): baseModule = None if len(sys.argv) > 1: - baseModule = sys.argv[1] + baseModule = sys.argv[1].strip() + +if not baseModule: + print("An argument is required", file=sys.stderr) + sys.exit(1) classList = importClasses( basemodule=baseModule, skipnames=("ZentinelPortal", "ZDeviceLoader") diff --git a/Products/ZenUtils/CheckRelations.py b/Products/ZenUtils/CheckRelations.py index e7e7991f1e..98f6d95445 100644 --- a/Products/ZenUtils/CheckRelations.py +++ b/Products/ZenUtils/CheckRelations.py @@ -1,28 +1,19 @@ ############################################################################## -# +# # Copyright (C) Zenoss, Inc. 2007, all rights reserved. -# +# # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. -# +# ############################################################################## - -__doc__="""CmdBase - -Add data base access functions for command line programs - -$Id: CheckRelations.py,v 1.2 2004/10/19 22:28:59 edahl Exp $""" - -__version__ = "$Revision: 1.2 $"[11:-2] - +from __future__ import absolute_import import transaction - from Products.ZenUtils.Utils import getAllConfmonObjects +from Products.ZenUtils.ZenScriptBase import ZenScriptBase -from ZenScriptBase import ZenScriptBase class CheckRelations(ZenScriptBase): @@ -32,10 +23,13 @@ def rebuild(self): self.log.info("Checking relations...") for object in getAllConfmonObjects(self.dmd): ccount += 1 - self.log.debug("checking relations on object %s", object.getPrimaryDmdId()) + self.log.debug( + "checking relations on object %s", object.getPrimaryDmdId() + ) object.checkRelations(repair=repair) ch = object._p_changed - if not ch: object._p_deactivate() + if not ch: + object._p_deactivate() if ccount >= self.options.savepoint: transaction.savepoint() ccount = 0 @@ -43,24 +37,34 @@ def rebuild(self): self.log.info("not commiting any changes") else: trans = transaction.get() - trans.note('CheckRelations cleaned relations' ) + trans.note("CheckRelations cleaned relations") trans.commit() - def buildOptions(self): - ZenScriptBase.buildOptions(self) - - self.parser.add_option('-r', '--repair', - dest='repair', action="store_true", - help='repair all inconsistant relations') - - self.parser.add_option('-x', '--savepoint', - dest='savepoint', default=500, type="int", - help='how many lines should be loaded before savepoint') - - self.parser.add_option('-n', '--nocommit', - dest='nocommit', action="store_true", - help='Do not store changes to the Dmd (for debugging)') + super(CheckRelations, self).buildOptions() + + self.parser.add_option( + "-r", + "--repair", + dest="repair", + action="store_true", + help="repair all inconsistant relations", + ) + self.parser.add_option( + "-x", + "--savepoint", + dest="savepoint", + default=500, + type="int", + help="how many lines should be loaded before savepoint", + ) + self.parser.add_option( + "-n", + "--nocommit", + dest="nocommit", + action="store_true", + help="Do not store changes to the Dmd (for debugging)", + ) if __name__ == "__main__": From d6b07448211e6ff4c096940b76dcae43101a6189 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 8 Mar 2024 15:16:59 -0600 Subject: [PATCH 096/147] fix: generate a config regardless of missing z-props ZEN-34728 --- Products/ZenHub/services/CommandPerformanceConfig.py | 10 ++++------ Products/ZenRRD/zencommand.py | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Products/ZenHub/services/CommandPerformanceConfig.py b/Products/ZenHub/services/CommandPerformanceConfig.py index 06fe0f0047..75a828d0b4 100644 --- a/Products/ZenHub/services/CommandPerformanceConfig.py +++ b/Products/ZenHub/services/CommandPerformanceConfig.py @@ -7,13 +7,13 @@ # ############################################################################## -from __future__ import print_function - """CommandPerformanceConfig Provides configuration to zencommand clients. """ +from __future__ import print_function + import logging import traceback @@ -250,10 +250,8 @@ def _createDeviceProxy(self, device): comp, device, perfServer, commands, proxy.thresholds ) - if commands: - proxy.datasources = list(commands) - return proxy - return None + proxy.datasources = list(commands) + return proxy def _sendCmdEvent( self, diff --git a/Products/ZenRRD/zencommand.py b/Products/ZenRRD/zencommand.py index fb6fe49dfd..f5fd9e7fd3 100755 --- a/Products/ZenRRD/zencommand.py +++ b/Products/ZenRRD/zencommand.py @@ -686,7 +686,8 @@ def _parseResults(self, resultList): for success, (command, result) in resultList: parse = self._parse_result if success else self._parse_error datasources = self._commandMap.get(command) - parsed_results.extend(parse(datasources, result)) + if datasources: + parsed_results.extend(parse(datasources, result)) return parsed_results def _timeout_error_result(self, datasource): From 55a78136dc601dd7fe197e22c0a4d3fc15235e90 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 8 Mar 2024 15:24:55 -0600 Subject: [PATCH 097/147] fix: improve detection of devices that change their device class ZEN-34728 --- .../ZenCollector/configcache/cache/storage.py | 4 +- .../ZenCollector/configcache/invalidator.py | 74 +++++++++++++++---- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 8c144bcbba..b83d01f402 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -147,7 +147,7 @@ def add(self, record): if key.monitor != mon ) watch_keys = self._get_watch_keys(orphaned_keys + (record.key,)) - add_uid = not self.__uids.exists(self.__client, dvc) + stored_uid = self.__uids.get(self.__client, dvc) def _add_impl(pipe): pipe.multi() @@ -163,7 +163,7 @@ def _add_impl(pipe): self.__expired.delete(pipe, *parts) self.__pending.delete(pipe, *parts) self.__building.delete(pipe, *parts) - if add_uid: + if stored_uid != uid: self.__uids.set(pipe, dvc, uid) self.__config.set(pipe, svc, mon, dvc, config) self.__age.add(pipe, svc, mon, dvc, updated) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index cbfde8d799..0ed0379619 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -120,11 +120,13 @@ def _synchronize(self): timelimitmap = DevicePropertyMap.make_build_timeout_map( self.ctx.dmd.Devices ) - new_devices = _addNew( + new_devices, changed_devices = _addNewOrChangedDevices( self.log, tool, timelimitmap, self.store, self.dispatcher ) if len(new_devices) == 0: self.log.info("no missing configurations found") + if len(changed_devices) == 0: + self.log.info("no devices with a different device class found") def _process_all(self, invalidations): buildlimit_map = DevicePropertyMap.make_build_timeout_map( @@ -138,7 +140,7 @@ def _process_all(self, invalidations): buildlimit = buildlimit_map.get(uid) minttl = minttl_map.get(uid) try: - self._process(invalidation, buildlimit, minttl) + self._process(invalidation, uid, buildlimit, minttl) except AttributeError: self.log.info( "invalidation device=%s reason=%s", @@ -147,7 +149,7 @@ def _process_all(self, invalidations): ) self.log.exception("failed while processing invalidation") - def _process(self, invalidation, buildlimit, minttl): + def _process(self, invalidation, uid, buildlimit, minttl): device = invalidation.device reason = invalidation.reason monitor = device.getPerformanceServerName() @@ -165,7 +167,12 @@ def _process(self, invalidation, buildlimit, minttl): if not keys: self._new_device(device, monitor, buildlimit) elif reason is InvalidationCause.Updated: - self._updated_device(device, monitor, keys, minttl) + # Check for device class change + stored_uid = self.store.get_uid(device.id) + if uid != stored_uid: + self._changed_device_class(device, monitor, buildlimit) + else: + self._updated_device(device, monitor, keys, minttl) elif reason is InvalidationCause.Removed: self._removed_device(keys) else: @@ -204,6 +211,23 @@ def _new_device(self, device, monitor, buildlimit): monitor, ) + def _changed_device_class(self, device, monitor, keys, buildlimit): + # Don't dispatch jobs if there're any statuses. + keys = tuple( + CacheKey(svcname, monitor, device.id) + for svcname in self.dispatcher.service_names + ) + now = time.time() + for key in keys: + self.store.set_pending((key, now)) + self.dispatcher.dispatch_all(monitor, device.id, buildlimit) + self.log.info( + "submitted build jobs for device with new device class " + "device=%s collector=%s", + device.id, + monitor, + ) + def _updated_device(self, device, monitor, keys, minttl): statuses = tuple( status @@ -280,8 +304,9 @@ def _removeDeleted(log, tool, store): return len(devices_not_found) -def _addNew(log, tool, timelimitmap, store, dispatcher): +def _addNewOrChangedDevices(log, tool, timelimitmap, store, dispatcher): # Add new devices to the config and metadata store. + # Also look for device that have changed their device class. # Query the catalog for all devices catalog_results = tool.cursor_search( types=("Products.ZenModel.Device.Device",), @@ -289,6 +314,8 @@ def _addNew(log, tool, timelimitmap, store, dispatcher): fields=_solr_fields, ).results new_devices = [] + changed_devices = [] + jobs_args = [] for brain in catalog_results: if brain.collector is None: log.warn( @@ -302,12 +329,33 @@ def _addNew(log, tool, timelimitmap, store, dispatcher): ) if not keys: timeout = timelimitmap.get(brain.uid) - dispatcher.dispatch_all(brain.collector, brain.id, timeout) - log.info( - "submitted build jobs for device without any configurations " - "uid=%s collector=%s", - brain.uid, - brain.collector, - ) + jobs_args.append((brain, timeout)) new_devices.append(brain.id) - return new_devices + else: + current_uid = store.get_uid(brain.id) + if current_uid != brain.uid: + timeout = timelimitmap.get(brain.uid) + jobs_args.append((brain, timeout)) + changed_devices.append(brain.id) + + now = time.time() + for brain, timeout in jobs_args: + keys = tuple( + CacheKey(svcname, brain.collector, brain.id) + for svcname in dispatcher.service_names + ) + for key in keys: + store.set_pending((key, now)) + dispatcher.dispatch_all(brain.collector, brain.id, timeout) + log.info( + "submitted build jobs for device %s " + "uid=%s collector=%s", + ( + "without any configurations" + if brain.id in new_devices + else "with a new device class" + ), + brain.uid, + brain.collector, + ) + return (new_devices, changed_devices) From 9019f5c4b192433f4f543df0bc5ce735f2ead30e Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 11 Mar 2024 11:25:13 -0500 Subject: [PATCH 098/147] fix: import zenpack entrypoints before starting celery ZenJobs has to load the 'zenoss.zenpacks' entrypoints to ensure that celery is able to locate jobs defined in the zenpacks. Added a `get_app` function to avoid littering nested imports for the zenjobs/celery app all over the codebase. ZEN-34751 --- Products/Jobber/bin.py | 31 ++++++++++++++++++++ Products/Jobber/config.py | 3 -- Products/Jobber/model.py | 55 +++++++++++++----------------------- Products/Jobber/signals.zcml | 5 ++++ Products/Jobber/utils/app.py | 16 +++++++++++ Products/Jobber/worker.py | 44 +++++++++++++++++------------ Products/Jobber/zenjobs.py | 11 ++------ bin/zenjobs | 26 ----------------- setup.py | 1 + 9 files changed, 100 insertions(+), 92 deletions(-) create mode 100644 Products/Jobber/bin.py create mode 100644 Products/Jobber/utils/app.py delete mode 100755 bin/zenjobs diff --git a/Products/Jobber/bin.py b/Products/Jobber/bin.py new file mode 100644 index 0000000000..dca0c15a8f --- /dev/null +++ b/Products/Jobber/bin.py @@ -0,0 +1,31 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + + +def main(): + import sys + + import Products.Jobber + + from celery.bin.celery import main + from Products.ZenUtils.Utils import load_config + from Products.ZenUtils.zenpackload import load_zenpacks + + # The Zenoss environment requires that the 'zenoss.zenpacks' entrypoints + # be explicitely loaded because celery doesn't know to do that. + # Not loading those entrypoints means that celery will be unaware of + # any celery 'task' definitions in the ZenPacks. + load_zenpacks() + + load_config("signals.zcml", Products.Jobber) + + # All calls to celery need the 'app instance' for zenjobs. + sys.argv[1:] = ["-A", "Products.Jobber.zenjobs"] + sys.argv[1:] + + sys.exit(main()) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 9a2515ceb6..5dcc872939 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -126,9 +126,6 @@ class Celery(object): # Beat (scheduler) configuration CELERYBEAT_MAX_LOOP_INTERVAL = ZenJobs.get("scheduler-max-loop-interval") - CELERYBEAT_LOG_FILE = os.path.join( - ZenJobs.get("logpath"), "zenjobs-scheduler.log" - ) CELERYBEAT_REDIRECT_STDOUTS = True CELERYBEAT_REDIRECT_STDOUTS_LEVEL = "INFO" diff --git a/Products/Jobber/model.py b/Products/Jobber/model.py index bfba76bc6d..7916f90007 100644 --- a/Products/Jobber/model.py +++ b/Products/Jobber/model.py @@ -26,9 +26,10 @@ from .interfaces import IJobStore, IJobRecord from .storage import Fields from .task.utils import job_log_has_errors +from .utils.app import get_app from .utils.log import inject_logger -mlog = logging.getLogger("zen.zenjobs.model") +_mlog = logging.getLogger("zen.zenjobs.model") sortable_keys = list(set(Fields) - {"details"}) @@ -120,9 +121,7 @@ def job_name(self): @property def job_type(self): - from .zenjobs import app - - task = app.tasks.get(self.name) + task = get_app().tasks.get(self.name) if task is None: return self.name if self.name else "" try: @@ -154,9 +153,7 @@ def wait(self): @property def result(self): - from .zenjobs import app - - return app.tasks[self.name].AsyncResult(self.jobid) + return get_app().tasks[self.name].AsyncResult(self.jobid) def __eq__(self, other): if not isinstance(other, type(self)): @@ -291,9 +288,7 @@ def from_signal(cls, body, headers, properties): @classmethod def _build(cls, jobid, taskname, args, kwargs, headers, properties): - from .zenjobs import app - - task = app.tasks[taskname] + task = get_app().tasks[taskname] fields = {} description = properties.pop("description", None) if description: @@ -306,7 +301,7 @@ def _build(cls, jobid, taskname, args, kwargs, headers, properties): return cls.from_task(task, jobid, args, kwargs, **fields) -@inject_logger(log=mlog) +@inject_logger(log=_mlog) def save_jobrecord(log, body=None, headers=None, properties=None, **ignored): """Save the Zenoss specific job metadata to redis. @@ -318,8 +313,6 @@ def save_jobrecord(log, body=None, headers=None, properties=None, **ignored): :param dict headers: Headers to accompany message sent to Celery worker :param dict properties: Additional task and custom key/value pairs """ - from .zenjobs import app - if not body: # If body is empty (or None), no job to save. log.info("no body, so no job") @@ -330,7 +323,7 @@ def save_jobrecord(log, body=None, headers=None, properties=None, **ignored): log.info("no headers, bad signal?") return - task = app.tasks.get(body.get("task")) + task = get_app().tasks.get(body.get("task")) if task is None: log.warn("Ignoring unknown task: %s", body.get("task")) return @@ -387,16 +380,14 @@ def _save_record(log, storage, record): return False -@inject_logger(log=mlog) +@inject_logger(log=_mlog) def stage_jobrecord(log, storage, sig): """Save Zenoss job data to redis with status "STAGED". :param sig: The job data :type sig: celery.canvas.Signature """ - from .zenjobs import app - - task = app.tasks.get(sig.task) + task = get_app().tasks.get(sig.task) # Tasks with ignored results cannot be tracked, # so don't insert a record into Redis. @@ -414,16 +405,14 @@ def stage_jobrecord(log, storage, sig): _save_record(log, storage, record) -@inject_logger(log=mlog) +@inject_logger(log=_mlog) def commit_jobrecord(log, storage, sig): """Update STAGED job records to PENDING. :param sig: The job data :type sig: celery.canvas.Signature """ - from .zenjobs import app - - task = app.tasks.get(sig.task) + task = get_app().tasks.get(sig.task) # Tasks with ignored results cannot be tracked, # so there won't be a record to update. @@ -452,7 +441,7 @@ def wrapper(log, *args, **kw): return wrapper -@inject_logger(log=mlog) +@inject_logger(log=_mlog) @_catch_exception def job_start(log, task_id, task=None, **ignored): if task is not None and task.ignore_result: @@ -476,7 +465,7 @@ def job_start(log, task_id, task=None, **ignored): log.info("status=%s started=%s", status, tm) -@inject_logger(log=mlog) +@inject_logger(log=_mlog) @_catch_exception def job_end(log, task_id, task=None, **ignored): if task is not None and task.ignore_result: @@ -501,16 +490,14 @@ def job_end(log, task_id, task=None, **ignored): log.info("Job total duration is %0.3f seconds", finished - started) -@inject_logger(log=mlog) +@inject_logger(log=_mlog) @_catch_exception def job_success(log, result, sender=None, **ignored): - from .zenjobs import app - if sender is not None and sender.ignore_result: return task_id = sender.request.id jobstore = getUtility(IJobStore, "redis") - status = app.backend.get_status(task_id) + status = get_app().backend.get_status(task_id) if job_log_has_errors(task_id): log.warn("Error messages detected in job log.") status = states.FAILURE @@ -519,14 +506,12 @@ def job_success(log, result, sender=None, **ignored): log.info("status=%s finished=%s", status, tm) -@inject_logger(log=mlog) +@inject_logger(log=_mlog) @_catch_exception def job_failure(log, task_id, exception=None, sender=None, **ignored): - from .zenjobs import app - if sender is not None and sender.ignore_result: return - status = app.backend.get_status(task_id) + status = get_app().backend.get_status(task_id) jobstore = getUtility(IJobStore, "redis") if task_id not in jobstore: @@ -551,15 +536,13 @@ def job_failure(log, task_id, exception=None, sender=None, **ignored): jobstore.update(cbid, status=ABORTED, finished=tm) -@inject_logger(log=mlog) +@inject_logger(log=_mlog) @_catch_exception def job_retry(log, request, reason=None, sender=None, **ignored): - from .zenjobs import app - if sender is not None and sender.ignore_result: return jobstore = getUtility(IJobStore, "redis") task_id = request.id - status = app.backend.get_status(task_id) + status = get_app().backend.get_status(task_id) jobstore.update(task_id, status=status) log.info("status=%s", status) diff --git a/Products/Jobber/signals.zcml b/Products/Jobber/signals.zcml index 7a62de78f5..247ca57882 100644 --- a/Products/Jobber/signals.zcml +++ b/Products/Jobber/signals.zcml @@ -5,6 +5,11 @@ + + Date: Tue, 12 Mar 2024 16:09:40 -0500 Subject: [PATCH 099/147] fix: add support for loading tasks from jobs.zcml files. This change is to support ZenPacks defining jobs in separate jobs.zcml files so that celery tasks can be loaded without having to load all the zcml files. ZEN-34751 --- Products/Jobber/meta.py | 66 +++++++++++++++++++++++++++++++ Products/Jobber/meta.zcml | 9 +++-- Products/Jobber/metaconfigure.py | 26 ------------ Products/Jobber/metadirectives.py | 43 -------------------- Products/Jobber/signals.zcml | 5 +++ Products/Jobber/worker.py | 36 +++++++++++++++++ 6 files changed, 112 insertions(+), 73 deletions(-) create mode 100644 Products/Jobber/meta.py delete mode 100644 Products/Jobber/metaconfigure.py delete mode 100644 Products/Jobber/metadirectives.py diff --git a/Products/Jobber/meta.py b/Products/Jobber/meta.py new file mode 100644 index 0000000000..d92e860e6d --- /dev/null +++ b/Products/Jobber/meta.py @@ -0,0 +1,66 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2012-2024 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function, unicode_literals + +from celery import signals +from zope.configuration.exceptions import ConfigurationError +from zope.configuration.fields import GlobalObject +from zope.interface import Interface +from zope.schema import TextLine + + +class IJob(Interface): + """Registers a ZenJobs task.""" + + task = GlobalObject( + title="ZenJobs Task", + description="Path to a task class or function", + required=False, + ) + + class_ = task # old name for backward compatibility + + +def job(_context, **kw): + """Register the task with Celery, if necessary.""" + from Products.Jobber.zenjobs import app + + task = kw.get("task") + if task is None: + task = kw.get("class_") + if task is None: + raise ConfigurationError( + ("Missing parameter:", "'task' or 'class'") + ) + if task.name not in app.tasks: + app.tasks.register(task) + + +class ICelerySignal(Interface): + """Registers a Celery signal handler.""" + + name = TextLine( + title="Name", + description="The signal receiving a handler", + ) + + handler = TextLine( + title="Handler", + description="Classpath to the function handling the signal", + ) + + +def signal(_context, name, handler): + """Register a Celery signal handler.""" + signal = getattr(signals, name, None) + if signal is None: + raise AttributeError("Unknown signal name '%s'" % name) + handler_fn = _context.resolve(handler) + signal.connect(handler_fn) diff --git a/Products/Jobber/meta.zcml b/Products/Jobber/meta.zcml index 0e19b2b00f..e9e5dd1a12 100644 --- a/Products/Jobber/meta.zcml +++ b/Products/Jobber/meta.zcml @@ -1,19 +1,20 @@ + diff --git a/Products/Jobber/metaconfigure.py b/Products/Jobber/metaconfigure.py deleted file mode 100644 index 81670e2227..0000000000 --- a/Products/Jobber/metaconfigure.py +++ /dev/null @@ -1,26 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2012-2019 all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -from __future__ import absolute_import, print_function - -from celery import signals - - -def job(_context, class_, name=None): - """Hold place for unused job directive.""" - pass - - -def signal(_context, name, handler): - """Register a Celery signal handler.""" - signal = getattr(signals, name, None) - if signal is None: - raise AttributeError("Unknown signal name '%s'" % name) - handler_fn = _context.resolve(handler) - signal.connect(handler_fn) diff --git a/Products/Jobber/metadirectives.py b/Products/Jobber/metadirectives.py deleted file mode 100644 index a4eae6f39b..0000000000 --- a/Products/Jobber/metadirectives.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2012-2019 all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -from __future__ import absolute_import, unicode_literals - -from zope.configuration.fields import GlobalObject -from zope.interface import Interface -from zope.schema import TextLine - - -class IJob(Interface): - """Registers a ZenJobs Job class.""" - - class_ = GlobalObject( - title="Job Class", - description="The class of the job to register", - ) - - name = TextLine( - title="Name", - description="Optional name of the job", - required=False, - ) - - -class ICelerySignal(Interface): - """Registers a Celery signal handler.""" - - name = TextLine( - title="Name", - description="The signal receiving a handler", - ) - - handler = TextLine( - title="Handler", - description="Classpath to the function handling the signal", - ) diff --git a/Products/Jobber/signals.zcml b/Products/Jobber/signals.zcml index 247ca57882..a2a39485e5 100644 --- a/Products/Jobber/signals.zcml +++ b/Products/Jobber/signals.zcml @@ -5,6 +5,11 @@ + + Date: Sun, 17 Mar 2024 07:27:22 -0500 Subject: [PATCH 100/147] Fix _changed_device_class to receive the correct number of arguments. ZEN-34759 --- Products/ZenCollector/configcache/invalidator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 0ed0379619..34c730cd2c 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -211,7 +211,7 @@ def _new_device(self, device, monitor, buildlimit): monitor, ) - def _changed_device_class(self, device, monitor, keys, buildlimit): + def _changed_device_class(self, device, monitor, buildlimit): # Don't dispatch jobs if there're any statuses. keys = tuple( CacheKey(svcname, monitor, device.id) From 690157b75a0b961a62dc7fc29ee66c713107ee75 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Sun, 17 Mar 2024 08:18:45 -0500 Subject: [PATCH 101/147] Allow zencommand to get configs without defined credentials. Old behavior was to not create a config if SSH credentials were not defined, but this behavior does not work with the configcache system. So, allow zencommand to receive configs without SSH credentials and accept the errors that happen. ZEN-34758 --- .../ZenHub/services/CommandPerformanceConfig.py | 14 +++++++------- Products/ZenRRD/runner.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Products/ZenHub/services/CommandPerformanceConfig.py b/Products/ZenHub/services/CommandPerformanceConfig.py index 75a828d0b4..25d46e6cc9 100644 --- a/Products/ZenHub/services/CommandPerformanceConfig.py +++ b/Products/ZenHub/services/CommandPerformanceConfig.py @@ -134,14 +134,13 @@ def _getComponentConfig(self, comp, device, perfServer, cmds): if not ds.enabled: continue - # Ignore SSH datasources if no username set useSsh = getattr(ds, "usessh", False) if useSsh and not device.zCommandUsername: + # Send an event about no username set self._warnUsernameNotSet(device) - continue - - # clear any lingering no-username events - self._clearUsernameNotSet(device) + else: + # clear any lingering no-username events + self._clearUsernameNotSet(device) parserName = getattr(ds, "parser", "Auto") ploader = getParserLoader(self.dmd, parserName) @@ -250,8 +249,9 @@ def _createDeviceProxy(self, device): comp, device, perfServer, commands, proxy.thresholds ) - proxy.datasources = list(commands) - return proxy + if commands: + proxy.datasources = list(commands) + return proxy def _sendCmdEvent( self, diff --git a/Products/ZenRRD/runner.py b/Products/ZenRRD/runner.py index 400586729f..c008ab5be2 100644 --- a/Products/ZenRRD/runner.py +++ b/Products/ZenRRD/runner.py @@ -211,8 +211,8 @@ def __init__(self, proxy, client): self.manageIp = self.proxy.manageIp self.port = self.proxy.zCommandPort - _username = self.proxy.zCommandUsername - _password = self.proxy.zCommandPassword + _username = self.proxy.zCommandUsername or "" + _password = self.proxy.zCommandPassword or "" _loginTimeout = self.proxy.zCommandLoginTimeout _commandTimeout = self.proxy.zCommandCommandTimeout _keyPath = self.proxy.zKeyPath From 6c9405f0a005940946806cfd991ef4e564536ba4 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 18 Mar 2024 10:36:11 -0500 Subject: [PATCH 102/147] Add 'name' attribute back into clause. ZEN-34751 --- Products/Jobber/meta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Products/Jobber/meta.py b/Products/Jobber/meta.py index d92e860e6d..8135795f66 100644 --- a/Products/Jobber/meta.py +++ b/Products/Jobber/meta.py @@ -19,6 +19,8 @@ class IJob(Interface): """Registers a ZenJobs task.""" + name = TextLine(title="Name", description="Unused", required=False) + task = GlobalObject( title="ZenJobs Task", description="Path to a task class or function", From 477d822758ad08e0637e091477a4359cc77eeb67 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Sun, 17 Mar 2024 07:24:43 -0500 Subject: [PATCH 103/147] fix: revert to depending on jobs.zcml to register jobs The ZenPacks defining their own jobs must add a jobs.zcml file that identifies these jobs so that zenjobs can find and register them. ZEN-34751 --- Products/Jobber/bin.py | 7 ------- Products/Jobber/worker.py | 8 ++++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Products/Jobber/bin.py b/Products/Jobber/bin.py index dca0c15a8f..5c0a6cc20a 100644 --- a/Products/Jobber/bin.py +++ b/Products/Jobber/bin.py @@ -15,13 +15,6 @@ def main(): from celery.bin.celery import main from Products.ZenUtils.Utils import load_config - from Products.ZenUtils.zenpackload import load_zenpacks - - # The Zenoss environment requires that the 'zenoss.zenpacks' entrypoints - # be explicitely loaded because celery doesn't know to do that. - # Not loading those entrypoints means that celery will be unaware of - # any celery 'task' definitions in the ZenPacks. - load_zenpacks() load_config("signals.zcml", Products.Jobber) diff --git a/Products/Jobber/worker.py b/Products/Jobber/worker.py index 49a363ecf9..10ff9b6730 100644 --- a/Products/Jobber/worker.py +++ b/Products/Jobber/worker.py @@ -35,8 +35,16 @@ def initialize_zenoss_env(**kw): import Products.ZenWidgets from Products.ZenUtils.Utils import load_config_override + from Products.ZenUtils.zenpackload import load_zenpacks import_products() + + # The Zenoss environment requires that the 'zenoss.zenpacks' entrypoints + # be explicitely loaded because celery doesn't know to do that. + # Not loading those entrypoints means that celery will be unaware of + # any celery 'task' definitions in the ZenPacks. + load_zenpacks() + zcml.load_site() load_config_override("scriptmessaging.zcml", Products.ZenWidgets) From 5a63d8a7770700d098f5dd992d5cd8f6e1947288 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 8 Mar 2024 09:50:55 -0600 Subject: [PATCH 104/147] Add a 'remove' command to delete configs from the cache. The CLI code was also refactored into a sub-package. ZEN-34657 --- Products/ZenCollector/configcache/app/args.py | 6 +- Products/ZenCollector/configcache/app/init.py | 8 +- Products/ZenCollector/configcache/cli.py | 506 ------------------ .../ZenCollector/configcache/cli/__init__.py | 19 + Products/ZenCollector/configcache/cli/args.py | 88 +++ .../ZenCollector/configcache/cli/expire.py | 161 ++++++ .../configcache/{ => cli}/expire.zcml | 0 Products/ZenCollector/configcache/cli/list.py | 178 ++++++ .../configcache/{ => cli}/list.zcml | 0 .../ZenCollector/configcache/cli/remove.py | 159 ++++++ .../{show.zcml => cli/remove.zcml} | 0 Products/ZenCollector/configcache/cli/show.py | 135 +++++ .../ZenCollector/configcache/cli/show.zcml | 13 + .../ZenCollector/configcache/configcache.py | 3 +- .../ZenCollector/configcache/invalidator.py | 11 +- Products/ZenCollector/configcache/manager.py | 8 +- Products/ZenCollector/configcache/version.py | 2 +- 17 files changed, 777 insertions(+), 520 deletions(-) delete mode 100644 Products/ZenCollector/configcache/cli.py create mode 100644 Products/ZenCollector/configcache/cli/__init__.py create mode 100644 Products/ZenCollector/configcache/cli/args.py create mode 100644 Products/ZenCollector/configcache/cli/expire.py rename Products/ZenCollector/configcache/{ => cli}/expire.zcml (100%) create mode 100644 Products/ZenCollector/configcache/cli/list.py rename Products/ZenCollector/configcache/{ => cli}/list.zcml (100%) create mode 100644 Products/ZenCollector/configcache/cli/remove.py rename Products/ZenCollector/configcache/{show.zcml => cli/remove.zcml} (100%) create mode 100644 Products/ZenCollector/configcache/cli/show.py create mode 100644 Products/ZenCollector/configcache/cli/show.zcml diff --git a/Products/ZenCollector/configcache/app/args.py b/Products/ZenCollector/configcache/app/args.py index b16251c9ea..84a4bb6a57 100644 --- a/Products/ZenCollector/configcache/app/args.py +++ b/Products/ZenCollector/configcache/app/args.py @@ -24,15 +24,15 @@ def get_arg_parser(description, epilog=None): return parser -def get_subparser(subparsers, title, description=None, parent=None): +def get_subparser(subparsers, name, description=None, parent=None): subparser = subparsers.add_parser( - title, + name, description=description + ".", help=description, parents=[parent] if parent else [], formatter_class=ZenHelpFormatter, ) - _fix_optional_args_title(subparser, title.capitalize()) + _fix_optional_args_title(subparser, name.capitalize()) return subparser diff --git a/Products/ZenCollector/configcache/app/init.py b/Products/ZenCollector/configcache/app/init.py index d1f2f14ed9..263c6a7ad8 100644 --- a/Products/ZenCollector/configcache/app/init.py +++ b/Products/ZenCollector/configcache/app/init.py @@ -1,12 +1,16 @@ ############################################################################## # -# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. # # This content is made available according to terms specified in # License.zenoss under the directory where your Zenoss product is installed. # ############################################################################## +import sys + +import six + from zope.configuration import xmlconfig @@ -41,6 +45,8 @@ def _no_zope(configs, overrides): def _load_configs(ctx, configs): for filename, module in configs: + if isinstance(module, six.string_types): + module = sys.modules[module] xmlconfig.file(filename, package=module, context=ctx) diff --git a/Products/ZenCollector/configcache/cli.py b/Products/ZenCollector/configcache/cli.py deleted file mode 100644 index afa5d516e7..0000000000 --- a/Products/ZenCollector/configcache/cli.py +++ /dev/null @@ -1,506 +0,0 @@ -############################################################################## -# -# Copyright (C) Zenoss, Inc. 2023, all rights reserved. -# -# This content is made available according to terms specified in -# License.zenoss under the directory where your Zenoss product is installed. -# -############################################################################## - -from __future__ import absolute_import, print_function - -import argparse -import os -import sys -import time - -from datetime import datetime - -import six - -from IPython.lib import pretty -from twisted.spread.jelly import unjellyableRegistry -from zope.component import createObject - -import Products.ZenCollector.configcache as CONFIGCACHE_MODULE - -from Products.ZenCollector.services.config import DeviceProxy -from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl -from Products.ZenUtils.terminal_size import get_terminal_size - -from .app import initialize_environment -from .app.args import get_subparser -from .cache import CacheQuery, ConfigStatus - - -class List_(object): - - description = "List configurations" - - configs = (("list.zcml", CONFIGCACHE_MODULE),) - - @staticmethod - def add_arguments(parser, subparsers): - subp = get_subparser( - subparsers, - "list", - List_.description, - parent=_get_common_parser(), - ) - subp.add_argument( - "-u", - dest="show_uid", - default=False, - action="store_true", - help="Display ZODB path for device", - ) - subp.add_argument( - "-f", - dest="states", - action=MultiChoice, - choices=("current", "retired", "expired", "pending", "building"), - default=argparse.SUPPRESS, - help="Only list configurations having these states. One or " - "more states may be specified, separated by commas.", - ) - subp.set_defaults(factory=List_) - - def __init__(self, args): - self._monitor = "*{}*".format(args.collector).replace("***", "*") - self._service = "*{}*".format(args.service).replace("***", "*") - self._showuid = args.show_uid - self._devices = getattr(args, "device", []) - state_names = getattr(args, "states", ()) - if state_names: - states = set() - for name in state_names: - states.add(_name_state_lookup[name]) - self._states = tuple(states) - else: - self._states = () - - def run(self): - haswildcard = any("*" in d for d in self._devices) - if haswildcard and len(self._devices) > 1: - print( - "Only one DEVICE argument supported when a wildcard is used.", - file=sys.stderr, - ) - return - initialize_environment(configs=self.configs, useZope=False) - client = getRedisClient(url=getRedisUrl()) - store = createObject("configcache-store", client) - if haswildcard: - query = CacheQuery( - service=self._service, - monitor=self._monitor, - device=self._devices[0], - ) - else: - query = CacheQuery(service=self._service, monitor=self._monitor) - results = store.get_status(*store.search(query)) - if self._states: - results = ( - status - for status in results - if isinstance(status, self._states) - ) - rows = [] - maxd, maxs, maxm = 0, 0, 0 - if len(self._devices) > 0: - data = ( - status - for status in results - if status.key.device in self._devices - ) - else: - data = results - for status in sorted( - data, key=lambda x: (x.key.device, x.key.service) - ): - if self._showuid: - devid = status.uid - else: - devid = status.key.device - status_text = _format_status(status) - maxd = max(maxd, len(devid)) - maxs = max(maxs, len(status_text)) - maxm = max(maxm, len(status.key.monitor)) - rows.append( - (devid, status_text, status.key.monitor, status.key.service) - ) - if rows: - print( - "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( - "DEVICE", - "STATUS", - "COLLECTOR", - "SERVICE", - maxd=maxd, - maxs=maxs, - maxm=maxm, - ) - ) - for row in rows: - print( - "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( - row[0], - row[1], - row[2], - row[3], - maxd=maxd, - maxs=maxs, - maxm=maxm, - ) - ) - - -_name_state_lookup = { - "current": ConfigStatus.Current, - "retired": ConfigStatus.Retired, - "expired": ConfigStatus.Expired, - "pending": ConfigStatus.Pending, - "building": ConfigStatus.Building, -} - - -def _format_status(status): - if isinstance(status, ConfigStatus.Current): - return "current since {}".format(_format_date(status.updated)) - elif isinstance(status, ConfigStatus.Retired): - return "retired since {}".format(_format_date(status.retired)) - elif isinstance(status, ConfigStatus.Expired): - return "expired since {}".format(_format_date(status.expired)) - elif isinstance(status, ConfigStatus.Pending): - return "waiting to build since {}".format( - _format_date(status.submitted) - ) - elif isinstance(status, ConfigStatus.Building): - return "build started {}".format(_format_date(status.started)) - else: - return "????" - - -def _format_date(ts): - when = datetime.fromtimestamp(ts) - return when.strftime("%Y-%m-%d %H:%M:%S") - - -class Show(object): - - description = "Show a configuration" - - configs = (("show.zcml", CONFIGCACHE_MODULE),) - - @staticmethod - def add_arguments(parser, subparsers): - subp = get_subparser(subparsers, "show", Show.description) - termsize = get_terminal_size() - subp.add_argument( - "--width", - type=int, - default=termsize.columns, - help="Maxiumum number of columns to use in the output. " - "By default, this is the width of the terminal", - ) - subp.add_argument( - "service", nargs=1, help="name of the configuration service" - ) - subp.add_argument( - "collector", nargs=1, help="name of the performance collector" - ) - subp.add_argument("device", nargs=1, help="name of the device") - subp.set_defaults(factory=Show) - - def __init__(self, args): - self._monitor = args.collector[0] - self._service = args.service[0] - self._device = args.device[0] - if _is_output_redirected(): - # when stdout is redirected, default to 79 columns unless - # the --width option has a non-default value. - termsize = get_terminal_size() - if args.width != termsize.columns: - self._columns = args.width - else: - self._columns = 79 - else: - self._columns = args.width - - def run(self): - initialize_environment(configs=self.configs, useZope=False) - client = getRedisClient(url=getRedisUrl()) - store = createObject("configcache-store", client) - results, err = _query_cache( - store, - service="*{}*".format(self._service), - monitor="*{}*".format(self._monitor), - device="*{}*".format(self._device), - ) - if results: - for cls in set(unjellyableRegistry.values()): - if cls is DeviceProxy: - pretty.for_type(cls, _pp_DeviceProxy) - else: - pretty.for_type(cls, _pp_default) - pretty.pprint(results.config, max_width=self._columns) - else: - print(err, file=sys.stderr) - - -def _query_cache(store, service, monitor, device): - query = CacheQuery(service=service, monitor=monitor, device=device) - results = store.search(query) - first_key = next(results, None) - if first_key is None: - return (None, "configuration not found") - second_key = next(results, None) - if second_key is not None: - return (None, "more than one configuration matched arguments") - return (store.get(first_key), None) - - -def _pp_DeviceProxy(obj, p, cycle): - _printer( - obj, - p, - cycle, - lambda k, v: v if "password" not in k.lower() else "******", - ) - - -def _pp_default(obj, p, cycle): - _printer(obj, p, cycle, lambda k, v: v) - - -def _printer(obj, p, cycle, vprint): - clsname = obj.__class__.__name__ - if cycle: - p.text("<{}: ...>".format(clsname)) - else: - with p.group(2, "<{}: ".format(clsname), ">"): - attrs = ( - (k, v) - for k, v in sorted(obj.__dict__.items(), key=lambda x: x[0]) - if v not in (None, "", {}, []) - ) - for idx, (k, v) in enumerate(attrs): - if idx: - p.text(",") - p.breakable() - p.text("{}=".format(k)) - p.pretty(vprint(k, v)) - - -class Expire(object): - - description = "Mark configurations as expired" - - configs = (("expire.zcml", CONFIGCACHE_MODULE),) - - @staticmethod - def add_arguments(parser, subparsers): - subp = get_subparser( - subparsers, - "expire", - "Mark configurations as expired", - parent=_get_common_parser(), - ) - subp.set_defaults(factory=Expire) - - def __init__(self, args): - self._monitor = args.collector - self._service = args.service - self._devices = getattr(args, "device", []) - - def run(self): - haswildcard = any("*" in d for d in self._devices) - if haswildcard: - if len(self._devices) > 1: - print( - "Only one DEVICE argument supported when a " - "wildcard is used.", - file=sys.stderr, - ) - return - else: - self._devices = self._devices[0].replace("*", "") - if not self._confirm_inputs(): - print("exit") - return - initialize_environment(configs=self.configs, useZope=False) - client = getRedisClient(url=getRedisUrl()) - store = createObject("configcache-store", client) - query = CacheQuery(service=self._service, monitor=self._monitor) - results = store.get_status(*store.search(query)) - method = self._no_devices if not self._devices else self._with_devices - keys = method(results, wildcard=haswildcard) - now = time.time() - store.set_expired(*((key, now) for key in keys)) - count = len(keys) - print( - "expired %d device configuration%s" - % (count, "" if count == 1 else "s") - ) - - def _no_devices(self, results, wildcard=False): - return tuple(status.key for status in results) - - def _with_devices(self, results, wildcard=False): - if wildcard: - predicate = self._check_wildcard - else: - predicate = self._check_list - - return tuple( - status.key for status in results if predicate(status.key.device) - ) - - def _check_wildcard(self, device): - return self._devices in device - - def _check_list(self, device): - return device in self._devices - - def _confirm_inputs(self): - if self._devices: - return True - if (self._monitor, self._service) == ("*", "*"): - mesg = "Recreate all device configurations" - elif "*" not in self._monitor and self._service == "*": - mesg = ( - "Recreate all configurations for devices monitored by the " - "'%s' collector" % (self._monitor,) - ) - elif "*" in self._monitor and self._service == "*": - mesg = ( - "Recreate all configurations for devices monitored by all " - "collectors matching '%s'" % (self._monitor,) - ) - elif self._monitor == "*" and "*" not in self._service: - mesg = ( - "Recreate all device configurations created by the '%s' " - "service" % (self._service.split(".")[-1],) - ) - elif self._monitor == "*" and "*" in self._service: - mesg = ( - "Recreate all device configurations created by all " - "services matching '%s'" % (self._service,) - ) - elif "*" in self._monitor and "*" not in self._service: - mesg = ( - "Recreate all configurations created by the '%s' " - "service for devices monitored by all collectors " - "matching '%s'" % (self._service, self._monitor) - ) - elif "*" not in self._monitor and "*" in self._service: - mesg = ( - "Recreate all configurations for devices monitored by the " - "'%s' collector and created by all services matching '%s'" - % (self._monitor, self._service) - ) - elif "*" not in self._monitor and "*" not in self._service: - mesg = ( - "Recreate all configurations for devices monitored by the " - "'%s' collector and created by the '%s' service" - % (self._monitor, self._service) - ) - elif "*" in self._monitor and "*" in self._service: - mesg = ( - "Recreate all configurations device monitored by all " - "collectors matching '%s' and created by all services " - "matching '%s'" % (self._monitor, self._service) - ) - else: - mesg = "collector '%s' service '%s'" % ( - self._monitor, - self._service, - ) - return _confirm(mesg) - - -def _confirm(mesg): - response = None - while response not in ["y", "n", ""]: - response = six.moves.input( - "%s. Are you sure (y/N)? " % (mesg,) - ).lower() - return response == "y" - - -class MultiChoice(argparse.Action): - """Allow multiple values for a choice option.""" - - def __init__(self, option_strings, dest, **kwargs): - kwargs["type"] = self._split_listed_choices - super(MultiChoice, self).__init__(option_strings, dest, **kwargs) - - @property - def choices(self): - return self._choices_checker - - @choices.setter - def choices(self, values): - self._choices_checker = _ChoicesChecker(values) - - def _split_listed_choices(self, value): - if "," in value: - return tuple(value.split(",")) - return value - - def __call__(self, parser, namespace, values=None, option_string=None): - if isinstance(values, six.string_types): - values = (values,) - setattr(namespace, self.dest, values) - - -class _ChoicesChecker(object): - def __init__(self, values): - self._choices = values - - def __contains__(self, value): - if isinstance(value, (list, tuple)): - return all(v in self._choices for v in value) - else: - return value in self._choices - - def __iter__(self): - return iter(self._choices) - - -def _is_output_redirected(): - return os.fstat(0) != os.fstat(1) - - -_common_parser = None - - -def _get_common_parser(): - global _common_parser - if _common_parser is None: - _common_parser = argparse.ArgumentParser(add_help=False) - _common_parser.add_argument( - "-m", - "--collector", - type=str, - default="*", - help="Name of the performance collector. Supports simple '*' " - "wildcard comparisons. A lone '*' selects all collectors.", - ) - _common_parser.add_argument( - "-s", - "--service", - type=str, - default="*", - help="Name of the configuration service. Supports simple '*' " - "wildcard comparisons. A lone '*' selects all services.", - ) - _common_parser.add_argument( - "device", - nargs="*", - default=argparse.SUPPRESS, - help="Name of the device. Multiple devices may be specified. " - "Supports simple '*' wildcard comparisons. Not specifying a " - "device will select all devices.", - ) - return _common_parser diff --git a/Products/ZenCollector/configcache/cli/__init__.py b/Products/ZenCollector/configcache/cli/__init__.py new file mode 100644 index 0000000000..fb026aaa52 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/__init__.py @@ -0,0 +1,19 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + + +from .expire import Expire +from .list import List_ +from .remove import Remove +from .show import Show + + +__all__ = ("Expire", "List_", "Remove", "Show") diff --git a/Products/ZenCollector/configcache/cli/args.py b/Products/ZenCollector/configcache/cli/args.py new file mode 100644 index 0000000000..dcdcf82e6d --- /dev/null +++ b/Products/ZenCollector/configcache/cli/args.py @@ -0,0 +1,88 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import argparse + +import six + + +class MultiChoice(argparse.Action): + """Allow multiple values for a choice option.""" + + def __init__(self, option_strings, dest, **kwargs): + kwargs["type"] = self._split_listed_choices + super(MultiChoice, self).__init__(option_strings, dest, **kwargs) + + @property + def choices(self): + return self._choices_checker + + @choices.setter + def choices(self, values): + self._choices_checker = _ChoicesChecker(values) + + def _split_listed_choices(self, value): + if "," in value: + return tuple(value.split(",")) + return value + + def __call__(self, parser, namespace, values=None, option_string=None): + if isinstance(values, six.string_types): + values = (values,) + setattr(namespace, self.dest, values) + + +class _ChoicesChecker(object): + def __init__(self, values): + self._choices = values + + def __contains__(self, value): + if isinstance(value, (list, tuple)): + return all(v in self._choices for v in value) + else: + return value in self._choices + + def __iter__(self): + return iter(self._choices) + + +_common_parser = None + + +def get_common_parser(): + global _common_parser + if _common_parser is None: + _common_parser = argparse.ArgumentParser(add_help=False) + _common_parser.add_argument( + "-m", + "--collector", + type=str, + default="*", + help="Name of the performance collector. Supports simple '*' " + "wildcard comparisons. A lone '*' selects all collectors.", + ) + _common_parser.add_argument( + "-s", + "--service", + type=str, + default="*", + help="Name of the configuration service. Supports simple '*' " + "wildcard comparisons. A lone '*' selects all services.", + ) + _common_parser.add_argument( + "device", + nargs="*", + default=argparse.SUPPRESS, + help="Name of the device. Multiple devices may be specified. " + "Supports simple '*' wildcard comparisons. Not specifying a " + "device will select all devices.", + ) + return _common_parser diff --git a/Products/ZenCollector/configcache/cli/expire.py b/Products/ZenCollector/configcache/cli/expire.py new file mode 100644 index 0000000000..5335d85f45 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/expire.py @@ -0,0 +1,161 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import sys +import time + +import six + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from ..app import initialize_environment +from ..app.args import get_subparser +from ..cache import CacheQuery + +from .args import get_common_parser + + +class Expire(object): + + description = "Mark configurations as expired" + + configs = (("expire.zcml", __name__),) + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, + "expire", + description=Expire.description, + parent=get_common_parser(), + ) + subp.set_defaults(factory=Expire) + + def __init__(self, args): + self._monitor = args.collector + self._service = args.service + self._devices = getattr(args, "device", []) + + def run(self): + haswildcard = any("*" in d for d in self._devices) + if haswildcard: + if len(self._devices) > 1: + print( + "Only one DEVICE argument supported when a " + "wildcard is used.", + file=sys.stderr, + ) + return + else: + self._devices = self._devices[0].replace("*", "") + if not self._confirm_inputs(): + print("exit") + return + initialize_environment(configs=self.configs, useZope=False) + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + query = CacheQuery(service=self._service, monitor=self._monitor) + results = store.get_status(*store.search(query)) + method = self._no_devices if not self._devices else self._with_devices + keys = method(results, wildcard=haswildcard) + now = time.time() + store.set_expired(*((key, now) for key in keys)) + count = len(keys) + print( + "expired %d device configuration%s" + % (count, "" if count == 1 else "s") + ) + + def _no_devices(self, results, wildcard=False): + return tuple(status.key for status in results) + + def _with_devices(self, results, wildcard=False): + if wildcard: + predicate = self._check_wildcard + else: + predicate = self._check_list + + return tuple( + status.key for status in results if predicate(status.key.device) + ) + + def _check_wildcard(self, device): + return self._devices in device + + def _check_list(self, device): + return device in self._devices + + def _confirm_inputs(self): + if self._devices: + return True + if (self._monitor, self._service) == ("*", "*"): + mesg = "Recreate all device configurations" + elif "*" not in self._monitor and self._service == "*": + mesg = ( + "Recreate all configurations for devices monitored by the " + "'%s' collector" % (self._monitor,) + ) + elif "*" in self._monitor and self._service == "*": + mesg = ( + "Recreate all configurations for devices monitored by all " + "collectors matching '%s'" % (self._monitor,) + ) + elif self._monitor == "*" and "*" not in self._service: + mesg = ( + "Recreate all device configurations created by the '%s' " + "service" % (self._service.split(".")[-1],) + ) + elif self._monitor == "*" and "*" in self._service: + mesg = ( + "Recreate all device configurations created by all " + "services matching '%s'" % (self._service,) + ) + elif "*" in self._monitor and "*" not in self._service: + mesg = ( + "Recreate all configurations created by the '%s' " + "service for devices monitored by all collectors " + "matching '%s'" % (self._service, self._monitor) + ) + elif "*" not in self._monitor and "*" in self._service: + mesg = ( + "Recreate all configurations for devices monitored by the " + "'%s' collector and created by all services matching '%s'" + % (self._monitor, self._service) + ) + elif "*" not in self._monitor and "*" not in self._service: + mesg = ( + "Recreate all configurations for devices monitored by the " + "'%s' collector and created by the '%s' service" + % (self._monitor, self._service) + ) + elif "*" in self._monitor and "*" in self._service: + mesg = ( + "Recreate all configurations device monitored by all " + "collectors matching '%s' and created by all services " + "matching '%s'" % (self._monitor, self._service) + ) + else: + mesg = "collector '%s' service '%s'" % ( + self._monitor, + self._service, + ) + return _confirm(mesg) + + +def _confirm(mesg): + response = None + while response not in ["y", "n", ""]: + response = six.moves.input( + "%s. Are you sure (y/N)? " % (mesg,) + ).lower() + return response == "y" diff --git a/Products/ZenCollector/configcache/expire.zcml b/Products/ZenCollector/configcache/cli/expire.zcml similarity index 100% rename from Products/ZenCollector/configcache/expire.zcml rename to Products/ZenCollector/configcache/cli/expire.zcml diff --git a/Products/ZenCollector/configcache/cli/list.py b/Products/ZenCollector/configcache/cli/list.py new file mode 100644 index 0000000000..7d18019370 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/list.py @@ -0,0 +1,178 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import argparse +import sys + +from datetime import datetime + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from ..app import initialize_environment +from ..app.args import get_subparser +from ..cache import CacheQuery, ConfigStatus + +from .args import get_common_parser, MultiChoice + + +class List_(object): + + description = "List configurations" + + configs = (("list.zcml", __name__),) + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, + "list", + description=List_.description, + parent=get_common_parser(), + ) + subp.add_argument( + "-u", + dest="show_uid", + default=False, + action="store_true", + help="Display ZODB path for device", + ) + subp.add_argument( + "-f", + dest="states", + action=MultiChoice, + choices=("current", "retired", "expired", "pending", "building"), + default=argparse.SUPPRESS, + help="Only list configurations having these states. One or " + "more states may be specified, separated by commas.", + ) + subp.set_defaults(factory=List_) + + def __init__(self, args): + self._monitor = "*{}*".format(args.collector).replace("***", "*") + self._service = "*{}*".format(args.service).replace("***", "*") + self._showuid = args.show_uid + self._devices = getattr(args, "device", []) + state_names = getattr(args, "states", ()) + if state_names: + states = set() + for name in state_names: + states.add(_name_state_lookup[name]) + self._states = tuple(states) + else: + self._states = () + + def run(self): + haswildcard = any("*" in d for d in self._devices) + if haswildcard and len(self._devices) > 1: + print( + "Only one DEVICE argument supported when a wildcard is used.", + file=sys.stderr, + ) + return + initialize_environment(configs=self.configs, useZope=False) + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + if haswildcard: + query = CacheQuery( + service=self._service, + monitor=self._monitor, + device=self._devices[0], + ) + else: + query = CacheQuery(service=self._service, monitor=self._monitor) + results = store.get_status(*store.search(query)) + if self._states: + results = ( + status + for status in results + if isinstance(status, self._states) + ) + rows = [] + maxd, maxs, maxm = 0, 0, 0 + if len(self._devices) > 0: + data = ( + status + for status in results + if status.key.device in self._devices + ) + else: + data = results + for status in sorted( + data, key=lambda x: (x.key.device, x.key.service) + ): + if self._showuid: + devid = status.uid + else: + devid = status.key.device + status_text = _format_status(status) + maxd = max(maxd, len(devid)) + maxs = max(maxs, len(status_text)) + maxm = max(maxm, len(status.key.monitor)) + rows.append( + (devid, status_text, status.key.monitor, status.key.service) + ) + if rows: + print( + "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( + "DEVICE", + "STATUS", + "COLLECTOR", + "SERVICE", + maxd=maxd, + maxs=maxs, + maxm=maxm, + ) + ) + for row in rows: + print( + "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( + row[0], + row[1], + row[2], + row[3], + maxd=maxd, + maxs=maxs, + maxm=maxm, + ) + ) + + +_name_state_lookup = { + "current": ConfigStatus.Current, + "retired": ConfigStatus.Retired, + "expired": ConfigStatus.Expired, + "pending": ConfigStatus.Pending, + "building": ConfigStatus.Building, +} + + +def _format_status(status): + if isinstance(status, ConfigStatus.Current): + return "current since {}".format(_format_date(status.updated)) + elif isinstance(status, ConfigStatus.Retired): + return "retired since {}".format(_format_date(status.retired)) + elif isinstance(status, ConfigStatus.Expired): + return "expired since {}".format(_format_date(status.expired)) + elif isinstance(status, ConfigStatus.Pending): + return "waiting to build since {}".format( + _format_date(status.submitted) + ) + elif isinstance(status, ConfigStatus.Building): + return "build started {}".format(_format_date(status.started)) + else: + return "????" + + +def _format_date(ts): + when = datetime.fromtimestamp(ts) + return when.strftime("%Y-%m-%d %H:%M:%S") diff --git a/Products/ZenCollector/configcache/list.zcml b/Products/ZenCollector/configcache/cli/list.zcml similarity index 100% rename from Products/ZenCollector/configcache/list.zcml rename to Products/ZenCollector/configcache/cli/list.zcml diff --git a/Products/ZenCollector/configcache/cli/remove.py b/Products/ZenCollector/configcache/cli/remove.py new file mode 100644 index 0000000000..b1837c2d51 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/remove.py @@ -0,0 +1,159 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import sys + +import six + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from ..app import initialize_environment +from ..app.args import get_subparser +from ..cache import CacheQuery + +from .args import get_common_parser + + +class Remove(object): + + description = "Delete configurations from the cache" + + configs = (("remove.zcml", __name__),) + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, + "remove", + description=Remove.description, + parent=get_common_parser(), + ) + subp.set_defaults(factory=Remove) + + def __init__(self, args): + self._monitor = args.collector + self._service = args.service + self._devices = getattr(args, "device", []) + + def run(self): + haswildcard = any("*" in d for d in self._devices) + if haswildcard: + if len(self._devices) > 1: + print( + "Only one DEVICE argument supported when a " + "wildcard is used.", + file=sys.stderr, + ) + return + else: + self._devices = self._devices[0].replace("*", "") + if not self._confirm_inputs(): + print("exit") + return + initialize_environment(configs=self.configs, useZope=False) + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + query = CacheQuery(service=self._service, monitor=self._monitor) + results = store.get_status(*store.search(query)) + method = self._no_devices if not self._devices else self._with_devices + keys = method(results, wildcard=haswildcard) + store.remove(*keys) + count = len(keys) + print( + "deleted %d device configuration%s" + % (count, "" if count == 1 else "s") + ) + + def _no_devices(self, results, wildcard=False): + return tuple(status.key for status in results) + + def _with_devices(self, results, wildcard=False): + if wildcard: + predicate = self._check_wildcard + else: + predicate = self._check_list + + return tuple( + status.key for status in results if predicate(status.key.device) + ) + + def _check_wildcard(self, device): + return self._devices in device + + def _check_list(self, device): + return device in self._devices + + def _confirm_inputs(self): + if self._devices: + return True + if (self._monitor, self._service) == ("*", "*"): + mesg = "Delete all device configurations" + elif "*" not in self._monitor and self._service == "*": + mesg = ( + "Delete all configurations for devices monitored by the " + "'%s' collector" % (self._monitor,) + ) + elif "*" in self._monitor and self._service == "*": + mesg = ( + "Delete all configurations for devices monitored by all " + "collectors matching '%s'" % (self._monitor,) + ) + elif self._monitor == "*" and "*" not in self._service: + mesg = ( + "Delete all device configurations created by the '%s' " + "service" % (self._service.split(".")[-1],) + ) + elif self._monitor == "*" and "*" in self._service: + mesg = ( + "Delete all device configurations created by all " + "services matching '%s'" % (self._service,) + ) + elif "*" in self._monitor and "*" not in self._service: + mesg = ( + "Delete all configurations created by the '%s' " + "service for devices monitored by all collectors " + "matching '%s'" % (self._service, self._monitor) + ) + elif "*" not in self._monitor and "*" in self._service: + mesg = ( + "Delete all configurations for devices monitored by the " + "'%s' collector and created by all services matching '%s'" + % (self._monitor, self._service) + ) + elif "*" not in self._monitor and "*" not in self._service: + mesg = ( + "Delete all configurations for devices monitored by the " + "'%s' collector and created by the '%s' service" + % (self._monitor, self._service) + ) + elif "*" in self._monitor and "*" in self._service: + mesg = ( + "Delete all configurations device monitored by all " + "collectors matching '%s' and created by all services " + "matching '%s'" % (self._monitor, self._service) + ) + else: + mesg = "collector '%s' service '%s'" % ( + self._monitor, + self._service, + ) + return _confirm(mesg) + + +def _confirm(mesg): + response = None + while response not in ["y", "n", ""]: + response = six.moves.input( + "%s. Are you sure (y/N)? " % (mesg,) + ).lower() + return response == "y" diff --git a/Products/ZenCollector/configcache/show.zcml b/Products/ZenCollector/configcache/cli/remove.zcml similarity index 100% rename from Products/ZenCollector/configcache/show.zcml rename to Products/ZenCollector/configcache/cli/remove.zcml diff --git a/Products/ZenCollector/configcache/cli/show.py b/Products/ZenCollector/configcache/cli/show.py new file mode 100644 index 0000000000..d5d8d8031d --- /dev/null +++ b/Products/ZenCollector/configcache/cli/show.py @@ -0,0 +1,135 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import os +import sys + +from IPython.lib import pretty +from twisted.spread.jelly import unjellyableRegistry +from zope.component import createObject + +from Products.ZenCollector.services.config import DeviceProxy +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl +from Products.ZenUtils.terminal_size import get_terminal_size + +from ..app import initialize_environment +from ..app.args import get_subparser +from ..cache import CacheQuery + + +class Show(object): + + description = "Show a configuration" + + configs = (("show.zcml", __name__),) + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser(subparsers, "show", description=Show.description) + termsize = get_terminal_size() + subp.add_argument( + "--width", + type=int, + default=termsize.columns, + help="Maxiumum number of columns to use in the output. " + "By default, this is the width of the terminal", + ) + subp.add_argument( + "service", nargs=1, help="name of the configuration service" + ) + subp.add_argument( + "collector", nargs=1, help="name of the performance collector" + ) + subp.add_argument("device", nargs=1, help="name of the device") + subp.set_defaults(factory=Show) + + def __init__(self, args): + self._monitor = args.collector[0] + self._service = args.service[0] + self._device = args.device[0] + if _is_output_redirected(): + # when stdout is redirected, default to 79 columns unless + # the --width option has a non-default value. + termsize = get_terminal_size() + if args.width != termsize.columns: + self._columns = args.width + else: + self._columns = 79 + else: + self._columns = args.width + + def run(self): + initialize_environment(configs=self.configs, useZope=False) + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + results, err = _query_cache( + store, + service="*{}*".format(self._service), + monitor="*{}*".format(self._monitor), + device="*{}*".format(self._device), + ) + if results: + for cls in set(unjellyableRegistry.values()): + if cls is DeviceProxy: + pretty.for_type(cls, _pp_DeviceProxy) + else: + pretty.for_type(cls, _pp_default) + pretty.pprint(results.config, max_width=self._columns) + else: + print(err, file=sys.stderr) + + +def _query_cache(store, service, monitor, device): + query = CacheQuery(service=service, monitor=monitor, device=device) + results = store.search(query) + first_key = next(results, None) + if first_key is None: + return (None, "configuration not found") + second_key = next(results, None) + if second_key is not None: + return (None, "more than one configuration matched arguments") + return (store.get(first_key), None) + + +def _pp_DeviceProxy(obj, p, cycle): + _printer( + obj, + p, + cycle, + lambda k, v: v if "password" not in k.lower() else "******", + ) + + +def _pp_default(obj, p, cycle): + _printer(obj, p, cycle, lambda k, v: v) + + +def _printer(obj, p, cycle, vprint): + clsname = obj.__class__.__name__ + if cycle: + p.text("<{}: ...>".format(clsname)) + else: + with p.group(2, "<{}: ".format(clsname), ">"): + attrs = ( + (k, v) + for k, v in sorted(obj.__dict__.items(), key=lambda x: x[0]) + if v not in (None, "", {}, []) + ) + for idx, (k, v) in enumerate(attrs): + if idx: + p.text(",") + p.breakable() + p.text("{}=".format(k)) + p.pretty(vprint(k, v)) + + +def _is_output_redirected(): + return os.fstat(0) != os.fstat(1) diff --git a/Products/ZenCollector/configcache/cli/show.zcml b/Products/ZenCollector/configcache/cli/show.zcml new file mode 100644 index 0000000000..8ec2993701 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/show.zcml @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/Products/ZenCollector/configcache/configcache.py b/Products/ZenCollector/configcache/configcache.py index a77a8812f2..c8e7c7bce3 100644 --- a/Products/ZenCollector/configcache/configcache.py +++ b/Products/ZenCollector/configcache/configcache.py @@ -10,7 +10,7 @@ from __future__ import absolute_import, print_function from .app.args import get_arg_parser -from .cli import List_, Show, Expire +from .cli import Expire, List_, Remove, Show from .invalidator import Invalidator from .manager import Manager from .version import Version @@ -27,6 +27,7 @@ def main(argv=None): List_.add_arguments(parser, subparsers) Show.add_arguments(parser, subparsers) Expire.add_arguments(parser, subparsers) + Remove.add_arguments(parser, subparsers) args = parser.parse_args() args.factory(args).run() diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 34c730cd2c..3a44f370c3 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -46,12 +46,12 @@ class Invalidator(object): @staticmethod def add_arguments(parser, subparsers): subp = get_subparser( - subparsers, "invalidator", Invalidator.description + subparsers, "invalidator", description=Invalidator.description ) subsubparsers = subp.add_subparsers(title="Invalidator Commands") subp_run = get_subparser( - subsubparsers, "run", "Run the invalidator service" + subsubparsers, "run", description="Run the invalidator service" ) Application.add_all_arguments(subp_run) subp_run.add_argument( @@ -69,7 +69,9 @@ def add_arguments(parser, subparsers): subp_debug = get_subparser( subsubparsers, "debug", - "Signal the invalidator service to toggle debug logging", + description=( + "Signal the invalidator service to toggle debug logging" + ), ) Application.add_pidfile_arguments(subp_debug) subp_debug.set_defaults(factory=DebugCommand.from_args) @@ -348,8 +350,7 @@ def _addNewOrChangedDevices(log, tool, timelimitmap, store, dispatcher): store.set_pending((key, now)) dispatcher.dispatch_all(brain.collector, brain.id, timeout) log.info( - "submitted build jobs for device %s " - "uid=%s collector=%s", + "submitted build jobs for device %s " "uid=%s collector=%s", ( "without any configurations" if brain.id in new_devices diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 822a1ffb7d..ec3154f0a9 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -39,11 +39,13 @@ class Manager(object): @staticmethod def add_arguments(parser, subparsers): - subp = get_subparser(subparsers, "manager", Manager.description) + subp = get_subparser( + subparsers, "manager", description=Manager.description + ) subsubparsers = subp.add_subparsers(title="Manager Commands") subp_run = get_subparser( - subsubparsers, "run", "Run the manager service" + subsubparsers, "run", description="Run the manager service" ) Application.add_all_arguments(subp_run) subp_run.add_argument( @@ -61,7 +63,7 @@ def add_arguments(parser, subparsers): subp_debug = get_subparser( subsubparsers, "debug", - "Signal the manager service to toggle debug logging", + description="Signal the manager service to toggle debug logging", ) Application.add_pidfile_arguments(subp_debug) subp_debug.set_defaults(factory=DebugCommand.from_args) diff --git a/Products/ZenCollector/configcache/version.py b/Products/ZenCollector/configcache/version.py index 1604104a88..88ca314144 100644 --- a/Products/ZenCollector/configcache/version.py +++ b/Products/ZenCollector/configcache/version.py @@ -19,7 +19,7 @@ class Version(object): @staticmethod def add_arguments(parser, subparsers): subp_version = get_subparser( - subparsers, "version", Version.description + subparsers, "version", description=Version.description ) subp_version.set_defaults(factory=Version) From cc2f55326404032d10db3524aca6e320f89f68b8 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 21 Mar 2024 14:13:15 -0500 Subject: [PATCH 105/147] Replace __cmp__ with __eq__/__lt__/__le__ combos. Some __cmp__ implementations don't work with 'in' operators and __cmp__ is not supported in Python 3. ZEN-34657 --- Products/ZenEvents/Availability.py | 39 +++++++-- Products/ZenHub/services/PerformanceConfig.py | 32 +++++-- Products/ZenModel/ZenMenuItem.py | 30 +++++-- Products/ZenModel/migrate/Migrate.py | 84 +++++++++++++------ Products/ZenModel/tests/testMigrate.py | 25 +++++- Products/ZenUtils/Version.py | 82 +++++++++++++----- 6 files changed, 221 insertions(+), 71 deletions(-) diff --git a/Products/ZenEvents/Availability.py b/Products/ZenEvents/Availability.py index 01c8ff5ada..ffe6151846 100644 --- a/Products/ZenEvents/Availability.py +++ b/Products/ZenEvents/Availability.py @@ -113,9 +113,19 @@ def __float__(self): def __int__(self): return int(self.availability * 100) - def __cmp__(self, other): - return cmp((self.availability, self.device, self.component()), - (other.availability, other.device, other.component())) + def __eq__(self, other): + if not isinstance(other, Availability): + return False + this = (self.availability, self.device, self.component()) + that = (other.availability, other.device, other.component()) + return this == that + + def __lt__(self, other): + if not isinstance(other, Availability): + return NotImplemented + this = (self.availability, self.device, self.component()) + that = (other.availability, other.device, other.component()) + return this < that def getDevice(self, dmd): return dmd.Devices.findDevice(self.device) @@ -183,9 +193,26 @@ def tuple(self): def __hash__(self): return hash(self.tuple()) - def __cmp__(self, other): - return cmp(self.tuple(), other.tuple()) - + def __eq__(self, other): + if not isinstance(other, Report): + return False + if self is other: + return True + return self.tuple() == other.tuple() + + def __lt__(self, other): + if not isinstance(other, Report): + return NotImplemented + if self is other: + return False + return self.tuple() < other.tuple() + + def __le__(self, other): + if not isinstance(other, Report): + return NotImplemented + if self is other: + return True + return self.tuple() <= other.tuple() def run(self, dmd): """Run the report, returning an Availability object for each device""" diff --git a/Products/ZenHub/services/PerformanceConfig.py b/Products/ZenHub/services/PerformanceConfig.py index edbe1303e2..cc72bc8234 100644 --- a/Products/ZenHub/services/PerformanceConfig.py +++ b/Products/ZenHub/services/PerformanceConfig.py @@ -53,12 +53,32 @@ def __init__(self, device): setattr(self, propertyName, getattr(device, propertyName, None)) self.id = device.id - def __cmp__(self, other): - for propertyName in ATTRIBUTES: - c = cmp(getattr(self, propertyName), getattr(other, propertyName)) - if c != 0: - return c - return 0 + def __eq__(self, other): + if not isinstance(other, SnmpConnInfo): + return False + if self is other: + return True + return all( + getattr(self, name) == getattr(other, name) for name in ATTRIBUTES + ) + + def __lt__(self, other): + if not isinstance(other, SnmpConnInfo): + return NotImplemented + if self is other: + return False + return any( + getattr(self, name) < getattr(other, name) for name in ATTRIBUTES + ) + + def __le__(self, other): + if not isinstance(other, SnmpConnInfo): + return NotImplemented + if self is other: + return True + return not any( + getattr(self, name) > getattr(other, name) for name in ATTRIBUTES + ) def summary(self): result = "SNMP info for %s at %s:%s" % ( diff --git a/Products/ZenModel/ZenMenuItem.py b/Products/ZenModel/ZenMenuItem.py index 7c827bdcd8..7240aaa0d1 100644 --- a/Products/ZenModel/ZenMenuItem.py +++ b/Products/ZenModel/ZenMenuItem.py @@ -56,12 +56,26 @@ def getMenuItemOwner(self): parent = aq_parent(parent) return parent - def __cmp__(self, other): - if isinstance(other, ZenMenuItem): - if other and other.ordering: - return cmp(other.ordering, self.ordering) - else: - return cmp(0.0, self.ordering) - return cmp(id(self), id(other)) - + def __eq__(self, other): + if not isinstance(other, ZenMenuItem): + return False + if self is other: + return True + return self.ordering == other.ordering + + def __lt__(self, other): + if not isinstance(other, ZenMenuItem): + return NotImplemented + if self is other: + return False + return self.ordering < other.ordering + + def __le__(self, other): + if not isinstance(other, ZenMenuItem): + return NotImplemented + if self is other: + return True + return self.ordering <= other.ordering + + InitializeClass(ZenMenuItem) diff --git a/Products/ZenModel/migrate/Migrate.py b/Products/ZenModel/migrate/Migrate.py index 153e34d57c..ac32eae76a 100644 --- a/Products/ZenModel/migrate/Migrate.py +++ b/Products/ZenModel/migrate/Migrate.py @@ -7,19 +7,23 @@ # ############################################################################## - -__doc__='''Migrate +'''Migrate A small framework for data migration. ''' +from __future__ import print_function + +import re + import transaction from Products.ZenUtils.ZenScriptBase import ZenScriptBase from Products.ZenUtils.Version import Version as VersionBase from Products.ZenReports.ReportLoader import ReportLoader from Products.ZenUtils.Utils import zenPath from Products.ZenModel.ZVersion import VERSION +from Products.ZenUtils.terminal_size import get_terminal_size import sys from textwrap import wrap @@ -60,17 +64,42 @@ def __init__(self): "self insert ourselves in the list of all steps" allSteps.append(self) - def __cmp__(self, other): - result = cmp(self.version, other.version) - if result: - return result - # if we're in the other dependency list, we are "less" - if self in other.getDependencies(): - return -1 - # if other is in the out dependency list, we are "greater" - if other in self.getDependencies(): - return 1 - return 0 + def __eq__(self, other): + if not isinstance(other, Step): + return False + if self is other: + return True + return ( + self.version == other.version + and self.dependencies == other.dependencies + ) + + def __lt__(self, other): + if not isinstance(other, Step): + return NotImplemented + if self is other: + return False + if self.version > other.version: + return False + return self._equivalency(other) + + def __le__(self, other): + if not isinstance(other, Step): + return NotImplemented + if self is other: + return True + return self._equivalency(other) + + def _equivalency(self, other): + if self.version > other.version: + return False + if self.version == other.version: + if self in other.getDependencies(): + return True + if other in self.getDependencies(): + return False + return self.name() < other.name() + return True def getDependencies(self): if not self.dependencies: @@ -121,11 +150,7 @@ def __init__(self, noopts=0): ZenScriptBase.__init__(self, noopts=noopts, connect=False) self.connect() self.allSteps = allSteps[:] - # 2 phase sorting - # 1. sort by name - self.allSteps.sort(lambda x,y: cmp(x.name(), y.name())) - # 2. sort by dependencies - self.allSteps.sort() + self.allSteps.sort() # _must_ sort the dependencies # Log output to a file # self.setupLogging() does *NOT* do what we want. @@ -426,34 +451,39 @@ def orderedSteps(self): def list(self, inputSteps=None, execSteps=None): steps = inputSteps or self.allSteps nameWidth = max(list(len(x.name()) for x in steps)) - + maxwidth = min(get_terminal_size().columns, 200) + indentSize = 8 + 3 + nameWidth + def switch(inp): switcher = { 1: ((" Ver Name" + " "*(nameWidth-3) + "Status\n" "--------+" + "-"*nameWidth +"+-------"), "%-8s %-{}s %-8s".format(nameWidth+1)), - 0: ((" Ver Name" + " "*(nameWidth-3) + "Description\n" - "--------+" + "-"*nameWidth +"+-----------" + "-"*30), + 0: ((" Ver Name" + " "*(nameWidth-2) + "Description\n" + "--------+" + "-"*(nameWidth+1) +"+-----------" + + "-"*(maxwidth - indentSize - 3)), "%-8s %-{}s %s".format(nameWidth+1)) } return switcher.get(inp) header, outputTemplate = switch(1 if inputSteps else 0) - print header + print(header) def printState(tpl, version, name, doc=None, status=None): if status: - print tpl%(version, name, status) + print(tpl%(version, name, status)) else: - print tpl%(version, name, doc) + print(tpl%(version, name, doc)) + indent = ' ' * indentSize + docWidth = maxwidth for s in steps: doc = s.__doc__ if not doc: doc = sys.modules[s.__class__.__module__].__doc__ \ or 'Not Documented' - doc.strip() - indent = ' '*22 - doc = '\n'.join(wrap(doc, width=80, + doc.strip() + doc = re.sub("\s+", " ", doc) + doc = '\n'.join(wrap(doc, width=docWidth, initial_indent=indent, subsequent_indent=indent)) doc = doc.lstrip() diff --git a/Products/ZenModel/tests/testMigrate.py b/Products/ZenModel/tests/testMigrate.py index 43a8f7ffa9..74acb593b9 100644 --- a/Products/ZenModel/tests/testMigrate.py +++ b/Products/ZenModel/tests/testMigrate.py @@ -13,14 +13,17 @@ from Products.ZenModel.migrate.Migrate import Migration, Version, Step class MyTestStep(Step): - def __init__(self, major, minor, micro): + def __init__(self, major, minor, micro, name=None): self.version = Version(major, minor, micro) + self._name = name or "MyTestStep" def __cutover__(self): pass def __cleanup__(self): pass def name(self): - return 'MyTestStep_%s' % self.version.short() + return '%s_%s' % (self._name, self.version.short()) + def __repr__(self): + return self.name() step300 = MyTestStep(3, 0, 0) step30_70 = MyTestStep(3, 0, 70) @@ -153,6 +156,24 @@ def testDetermineSteps(self): m.options.steps = ['MyTestStep_1.1.0'] self.assertEquals(m.determineSteps(), m.allSteps[1:2]) + def testDependencies(t): + m = Migration(noopts=True) + s1 = MyTestStep(1, 0, 0, name="StepA") + s2 = MyTestStep(1, 0, 0, name="StepB") + s3 = MyTestStep(1, 1, 0, name="StepC") + s4 = MyTestStep(1, 1, 0, name="StepD") + s5 = MyTestStep(1, 2, 0, name="StepE") + s6 = MyTestStep(1, 2, 0, name="StepCe") + s5.dependencies = [s3] + s3.dependencies = [s2, s4] + s1.dependencies = [s2] + m.allSteps = [s1, s2, s3, s4, s5, s6] + m.allSteps.sort() + m.options.level = "1.0.0" + t.assertEquals(m.determineSteps(), [s2, s1, s4, s3, s6, s5]) + m.options.level = "1.1.0" + t.assertEquals(m.determineSteps(), [s4, s3, s6, s5]) + def test_suite(): from unittest import TestSuite, makeSuite diff --git a/Products/ZenUtils/Version.py b/Products/ZenUtils/Version.py index 3a11f51cd4..1780e3b945 100644 --- a/Products/ZenUtils/Version.py +++ b/Products/ZenUtils/Version.py @@ -7,13 +7,18 @@ # ############################################################################## - """ Zenoss versioning module. """ + +from __future__ import print_function + import re +import six + + def getVersionTupleFromString(versionString): """ A utility function for parsing dot-delimited stings as a version tuple. @@ -197,25 +202,37 @@ def incrMicro(self): def setComment(self, comment): self.comment = comment - def __cmp__(self, other): - """ - Comparse one verion to another. If the other version supplied is not a - Version instance, attempt coercion. - - The assumption here is that any non-Version object being compared to a - Version object represents a verion of the same product with the same - name but a different version number. - """ + def __eq__(self, other): + if self is other: + return True + other = self._common_compare(other) + if not isinstance(other, Version): + return NotImplemented + return self.tuple() == other.tuple() + + def __lt__(self, other): + if self is other: + return False + other = self._common_compare(other) + if not isinstance(other, Version): + return NotImplemented + return self.tuple() < other.tuple() + + def __le__(self, other): + if self is other: + return True + other = self._common_compare(other) + if not isinstance(other, Version): + return NotImplemented + return self.tuple() <= other.tuple() + + def _common_compare(self, other): + other = Version.make(self.name, other) if other is None: - return 1 - if isinstance(other, tuple): - version = '.'.join(str(x) for x in other) - other = Version.parse("%s %s" % (self.name, version)) - elif any(isinstance(other, x) for x in (str, int, float, long)): - other = Version.parse("%s %s" % (self.name, str(other))) + return NotImplemented if self.name != other.name: raise IncomparableVersions() - return cmp(self.tuple(), other.tuple()) + return other def _formatSVNRevision(self): svnrev = self.revision @@ -242,6 +259,20 @@ def __str__(self): self.micro, self._formatSVNRevision()) + @classmethod + def make(cls, name, obj): + if isinstance(obj, cls): + return obj + if isinstance(obj, (tuple, list)): + version = '.'.join(str(x) for x in obj) + return cls.parse("%s %s" % (name, version)) + if any( + isinstance(obj, x) + for x in six.string_types + six.integer_types + (float,) + ): + return cls.parse("%s %s" % (name, obj)) + + @classmethod def parse(cls, versionString): """ Parse the version info from a string. This method is usable without @@ -263,26 +294,34 @@ def parse(cls, versionString): >>> v = Version.parse('Zenoss') >>> repr(v) 'Version(Zenoss, 0, 0, 0,)' - >>> print v + >>> print(v) [Zenoss, version 0.0.0] >>> v = Version.parse('Zenoss 1') >>> repr(v) 'Version(Zenoss, 1, 0, 0,)' - >>> print v + >>> print(v) [Zenoss, version 1.0.0] >>> v = Version.parse('Zenoss 0.26.4') >>> repr(v) 'Version(Zenoss, 0, 26, 4,)' - >>> print v + >>> print(v) [Zenoss, version 0.26.4] + >>> Version.parse('Zenoss 1.1.0') <= Version('Zenoss', 1, 0, 0) + False + >>> Version.parse('Zenoss 1.1.0') >= Version('Zenoss', 1, 0, 0) + True + >>> Version.parse('Zenoss 1.1.0') <= Version('Zenoss', 1, 1, 0) + True + >>> Version.parse('Zenoss 1.1.0') >= Version('Zenoss', 1, 1, 0) + True >>> v = Version.parse('Zenoss 0.32.1 r13667') >>> repr(v) 'Version(Zenoss, 0, 32, 1, r13667)' - >>> print v + >>> print(v) [Zenoss, version 0.32.1 r13667] """ versionParts = versionString.strip().split() @@ -302,7 +341,6 @@ def parse(cls, versionString): revision = '' self = Version(name, major, minor, micro, revision) return self - parse = classmethod(parse) def _test(): From cf092feb1b9ae2dfc36ba28233783a7d4b45b0ca Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Sun, 24 Mar 2024 11:07:34 -0500 Subject: [PATCH 106/147] add '--config-file' option to zenjobs. Allow different zenjobs services to have separate config files. ZEN-34771 --- Products/Jobber/config.py | 148 ++++++++++++++++------------- Products/Jobber/jobs/job.py | 4 +- Products/Jobber/jobs/purge_logs.py | 4 +- Products/Jobber/log.py | 72 ++++++++------ Products/Jobber/model.py | 4 +- Products/Jobber/scheduler.py | 10 +- Products/Jobber/signals.zcml | 7 +- Products/Jobber/storage.py | 7 +- Products/Jobber/task/base.py | 8 +- Products/Jobber/task/dmd.py | 4 +- Products/Jobber/worker.py | 14 ++- Products/Jobber/zenjobs.py | 11 ++- 12 files changed, 175 insertions(+), 118 deletions(-) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 5dcc872939..1b9005c8e5 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -9,13 +9,13 @@ import os +import attr + from Products.ZenUtils.config import Config, ConfigLoader from Products.ZenUtils.GlobalConfig import getGlobalConfiguration from Products.ZenUtils.RedisUtils import DEFAULT_REDIS_URL from Products.ZenUtils.Utils import zenPath -__all__ = ("Celery", "ZenJobs") - _default_configs = { "logpath": "/opt/zenoss/log", @@ -52,13 +52,23 @@ } -def _getConfig(): +_configuration = None + + +def getConfig(filename=None): """Return a dict containing the configuration for zenjobs.""" + global _configuration + + if filename is None or _configuration is not None: + return _configuration + + if not os.path.exists(filename): + filename = zenPath("etc", filename) + conf = _default_configs.copy() conf.update(getGlobalConfiguration()) - app_conf_file = zenPath("etc", "zenjobs.conf") - app_config_loader = ConfigLoader(app_conf_file, Config) + app_config_loader = ConfigLoader(filename, Config) try: conf.update(app_config_loader()) except IOError as ex: @@ -72,72 +82,82 @@ def _getConfig(): continue conf[key] = cast(conf[key]) + _configuration = conf return conf -ZenJobs = _getConfig() - - # Broker settings -def _buildBrokerUrl(): - usr = ZenJobs.get("amqpuser") - pwd = ZenJobs.get("amqppassword") - host = ZenJobs.get("amqphost") - port = ZenJobs.get("amqpport") - vhost = ZenJobs.get("amqpvhost") +def buildBrokerUrl(cfg): + usr = cfg.get("amqpuser") + pwd = cfg.get("amqppassword") + host = cfg.get("amqphost") + port = cfg.get("amqpport") + vhost = cfg.get("amqpvhost") return "amqp://{usr}:{pwd}@{host}:{port}/{vhost}".format(**locals()) -class Celery(object): - """Celery configuration.""" +_celery_config = None - BROKER_URL = _buildBrokerUrl() - CELERY_ACCEPT_CONTENT = ["without-unicode"] - # List of modules to import when the Celery worker starts - CELERY_IMPORTS = ( - "Products.Jobber.jobs", - "Products.ZenCollector.configcache.task", - ) +@attr.s(slots=True) +class CeleryConfig(object): + """Celery configuration.""" - # Job/Task routing - CELERY_ROUTES = { - "configcache.build_device_config": {"queue": "configcache"} - } - - # Result backend (redis) - CELERY_RESULT_BACKEND = ZenJobs.get("redis-url") - CELERY_RESULT_SERIALIZER = "without-unicode" - CELERY_TASK_RESULT_EXPIRES = ZenJobs.get("zenjobs-job-expires") - - # Worker configuration - CELERYD_CONCURRENCY = ZenJobs.get("concurrent-jobs") - CELERYD_PREFETCH_MULTIPLIER = 1 - CELERYD_MAX_TASKS_PER_CHILD = ZenJobs.get("max-jobs-per-worker") - CELERYD_TASK_TIME_LIMIT = ZenJobs.get("job-hard-time-limit") - CELERYD_TASK_SOFT_TIME_LIMIT = ZenJobs.get("job-soft-time-limit") - - # Task settings - CELERY_ACKS_LATE = True - CELERY_IGNORE_RESULT = False - CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True - CELERY_TASK_SERIALIZER = "without-unicode" - CELERY_TRACK_STARTED = True - - # Beat (scheduler) configuration - CELERYBEAT_MAX_LOOP_INTERVAL = ZenJobs.get("scheduler-max-loop-interval") - CELERYBEAT_REDIRECT_STDOUTS = True - CELERYBEAT_REDIRECT_STDOUTS_LEVEL = "INFO" - - # Event settings - CELERY_SEND_EVENTS = True - CELERY_SEND_TASK_SENT_EVENT = True - - # Log settings - CELERYD_LOG_COLOR = False - - -# Timezone -_tz = os.environ.get("TZ") -if _tz: - Celery.CELERY_TIMEZONE = _tz + BROKER_URL = attr.ib() + CELERY_TIMEZONE = attr.ib() + CELERY_RESULT_BACKEND = attr.ib() + CELERY_TASK_RESULT_EXPIRES = attr.ib() + CELERYD_CONCURRENCY = attr.ib() + CELERYD_MAX_TASKS_PER_CHILD = attr.ib() + CELERYD_TASK_TIME_LIMIT = attr.ib() + CELERYD_TASK_SOFT_TIME_LIMIT = attr.ib() + CELERYBEAT_MAX_LOOP_INTERVAL = attr.ib() + + CELERY_ACCEPT_CONTENT = attr.ib(default=["without-unicode"]) + CELERY_IMPORTS = attr.ib( + default=[ + "Products.Jobber.jobs", + "Products.ZenCollector.configcache.task", + ] + ) + CELERY_ROUTES = attr.ib( + default={"configcache.build_device_config": {"queue": "configcache"}} + ) + CELERY_RESULT_SERIALIZER = attr.ib(default="without-unicode") + CELERYD_PREFETCH_MULTIPLIER = attr.ib(default=1) + CELERY_ACKS_LATE = attr.ib(default=True) + CELERY_IGNORE_RESULT = attr.ib(default=False) + CELERY_STORE_ERRORS_EVEN_IF_IGNORED = attr.ib(default=True) + CELERY_TASK_SERIALIZER = attr.ib(default="without-unicode") + CELERY_TRACK_STARTED = attr.ib(default=True) + CELERYBEAT_REDIRECT_STDOUTS = attr.ib(default=True) + CELERYBEAT_REDIRECT_STDOUTS_LEVEL = attr.ib(default="INFO") + CELERY_SEND_EVENTS = attr.ib(default=True) + CELERY_SEND_TASK_SENT_EVENT = attr.ib(default=True) + CELERYD_LOG_COLOR = attr.ib(default=False) + + @classmethod + def from_config(cls, cfg=None): + global _celery_config + + if cfg is None or _celery_config is not None: + return _celery_config + + args = { + "BROKER_URL": buildBrokerUrl(_configuration), + "CELERY_RESULT_BACKEND": cfg.get("redis-url"), + "CELERY_TASK_RESULT_EXPIRES": cfg.get("zenjobs-job-expires"), + "CELERYD_CONCURRENCY": cfg.get("concurrent-jobs"), + "CELERYD_MAX_TASKS_PER_CHILD": cfg.get("max-jobs-per-worker"), + "CELERYD_TASK_TIME_LIMIT": cfg.get("job-hard-time-limit"), + "CELERYD_TASK_SOFT_TIME_LIMIT": cfg.get("job-soft-time-limit"), + "CELERYBEAT_MAX_LOOP_INTERVAL": cfg.get( + "scheduler-max-loop-interval" + ), + } + tz = os.environ.get("TZ") + if tz: + args["CELERY_TIMEZONE"] = tz + + _celery_config = cls(**args) + return _celery_config diff --git a/Products/Jobber/jobs/job.py b/Products/Jobber/jobs/job.py index 2d55541e61..93fa8df1b7 100644 --- a/Products/Jobber/jobs/job.py +++ b/Products/Jobber/jobs/job.py @@ -9,7 +9,7 @@ from __future__ import absolute_import -from ..config import ZenJobs +from ..config import getConfig from ..exceptions import NoSuchJobException from ..task import Abortable, DMD, ZenTask from ..zenjobs import app @@ -81,7 +81,7 @@ def setProperties(self, **properties): self.dmd.JobManager.update(jobid, **details) def _get_config(self, key, default=_MARKER): - value = ZenJobs.get(key, default) + value = getConfig().get(key, default) if value is _MARKER: raise KeyError("Config option '{}' is not defined".format(key)) return value diff --git a/Products/Jobber/jobs/purge_logs.py b/Products/Jobber/jobs/purge_logs.py index 3391d0fe14..b9b30e89c3 100644 --- a/Products/Jobber/jobs/purge_logs.py +++ b/Products/Jobber/jobs/purge_logs.py @@ -11,7 +11,7 @@ import os -from ..config import ZenJobs +from ..config import getConfig from ..task import requires, Abortable from ..zenjobs import app @@ -29,7 +29,7 @@ def purge_logs(self): key.replace(backend.task_keyprefix, "") for key in backend.client.keys("%s*" % backend.task_keyprefix) ) - logpath = ZenJobs.get("job-log-path") + logpath = getConfig().get("job-log-path") logfiles = os.listdir(logpath) if not logfiles: self.log.info("No log files to remove") diff --git a/Products/Jobber/log.py b/Products/Jobber/log.py index 6605f0346b..a9be43fe34 100644 --- a/Products/Jobber/log.py +++ b/Products/Jobber/log.py @@ -24,7 +24,7 @@ from Products.ZenUtils.Utils import zenPath -from .config import ZenJobs +from .config import getConfig from .interfaces import IJobStore from .utils.algorithms import partition from .utils.log import ( @@ -37,8 +37,6 @@ TaskLogFileHandler, ) -_default_log_level = logging.getLevelName(ZenJobs.get("logseverity")) - _default_config = { "version": 1, "disable_existing_loggers": False, @@ -71,15 +69,9 @@ }, "handlers": {}, "loggers": { - "STDOUT": { - "level": _default_log_level, - }, - "zen": { - "level": _default_log_level, - }, - "celery": { - "level": _default_log_level, - }, + "STDOUT": {}, + "zen": {}, + "celery": {}, }, "root": { "handlers": [], @@ -88,12 +80,10 @@ _main_loggers = { "zen.zenjobs": { - "level": _default_log_level, "propagate": False, "handlers": ["main"], }, "zen.zenjobs.job": { - "level": _default_log_level, "propagate": False, }, } @@ -103,26 +93,40 @@ "formatter": "main", "class": "cloghandler.ConcurrentRotatingFileHandler", "filename": None, - "maxBytes": ZenJobs.get("maxlogsize") * 1024, - "backupCount": ZenJobs.get("maxbackuplogs"), "mode": "a", "filters": ["main"], } + _beat_handler = { "formatter": "beat", "class": "cloghandler.ConcurrentRotatingFileHandler", "filename": None, - "maxBytes": ZenJobs.get("maxlogsize") * 1024, - "backupCount": ZenJobs.get("maxbackuplogs"), "mode": "a", } -_logpath = ZenJobs.get("logpath") -_filenames = { - "zenjobs": os.path.join(_logpath, "zenjobs.log"), - "beat": os.path.join(_logpath, "zenjobs-scheduler.log"), - "configcache_builder": os.path.join(_logpath, "configcache-builder.log"), -} + +def _get_handler(handler): + cfg = dict(handler) + cfg.update( + { + "maxBytes": getConfig().get("maxlogsize") * 1024, + "backupCount": getConfig().get("maxbackuplogs"), + } + ) + return cfg + + +def _get_filenames(cfg): + logpath = cfg.get("logpath") + return { + "zenjobs": os.path.join(logpath, "zenjobs.log"), + "beat": os.path.join(logpath, "zenjobs-scheduler.log"), + "configcache_builder": os.path.join( + logpath, "configcache-builder.log" + ), + } + + _loglevel_confs = { "zenjobs": zenPath("etc", "zenjobs_log_levels.conf"), "beat": zenPath("etc", "zenjobs_log_levels.conf"), @@ -142,17 +146,29 @@ def _get_logger(name=None): def configure_logging(logfile, **kw): """Configure logging for zenjobs.""" + cfg = getConfig() + default_log_level = logging.getLevelName(cfg.get("logseverity")) + filenames = _get_filenames(cfg) + + _default_config["loggers"]["STDOUT"]["level"] = default_log_level + _default_config["loggers"]["zen"]["level"] = default_log_level + _default_config["loggers"]["celery"]["level"] = default_log_level + # NOTE: Cleverly used the `-f` command line argument to specify # which logging configuration to use. if logfile in ("zenjobs", "configcache_builder"): + _main_loggers["zen.zenjobs"]["level"] = default_log_level + _main_loggers["zen.zenjobs.job"]["level"] = default_log_level _default_config["loggers"].update(**_main_loggers) _default_config["root"]["handlers"].append("main") - _main_handler["filename"] = _filenames[logfile] - _default_config["handlers"]["main"] = _main_handler + handler = _get_handler(_main_handler) + handler["filename"] = filenames[logfile] + _default_config["handlers"]["main"] = handler elif logfile == "beat": _default_config["root"]["handlers"].append("beat") - _beat_handler["filename"] = _filenames[logfile] - _default_config["handlers"]["beat"] = _beat_handler + handler = _get_handler(_beat_handler) + handler["filename"] = filenames[logfile] + _default_config["handlers"]["beat"] = handler logging.config.dictConfig(_default_config) diff --git a/Products/Jobber/model.py b/Products/Jobber/model.py index 7916f90007..d6227fe727 100644 --- a/Products/Jobber/model.py +++ b/Products/Jobber/model.py @@ -22,7 +22,7 @@ from Products.Zuul.interfaces import IMarshaller, IInfo -from .config import ZenJobs +from .config import getConfig from .interfaces import IJobStore, IJobRecord from .storage import Fields from .task.utils import job_log_has_errors @@ -249,7 +249,7 @@ def from_task(cls, task, jobid, args, kwargs, **fields): summary=task.summary, description=description, logfile=os.path.join( - ZenJobs.get("job-log-path"), "%s.log" % jobid + getConfig().get("job-log-path"), "%s.log" % jobid ), ) if "status" in fields: diff --git a/Products/Jobber/scheduler.py b/Products/Jobber/scheduler.py index f3eeae9ce1..d60c86ff0d 100644 --- a/Products/Jobber/scheduler.py +++ b/Products/Jobber/scheduler.py @@ -21,7 +21,7 @@ from celery.beat import Scheduler from celery.schedules import crontab -from .config import ZenJobs, Celery +from .config import getConfig, CeleryConfig class ZenJobsScheduler(Scheduler): @@ -120,7 +120,7 @@ def schedule(self): @property def info(self): return " . schedule-file -> {}".format( - ZenJobs.get("scheduler-config-file"), + getConfig().get("scheduler-config-file"), ) def sync(self): @@ -142,7 +142,7 @@ def close(self): def load_schedule(): - configfile = ZenJobs.get("scheduler-config-file") + configfile = getConfig().get("scheduler-config-file") with open(configfile, "r") as f: raw = yaml.load(f, Loader=yaml.loader.SafeLoader) parsed_schedule = {} @@ -222,7 +222,9 @@ def _key(name): def _getClient(): """Create and return the ZenJobs JobStore client.""" - return redis.StrictRedis.from_url(Celery.CELERY_RESULT_BACKEND) + return redis.StrictRedis.from_url( + CeleryConfig.from_config().CELERY_RESULT_BACKEND + ) def handle_beat_init(*args, **kw): diff --git a/Products/Jobber/signals.zcml b/Products/Jobber/signals.zcml index a2a39485e5..90380c8484 100644 --- a/Products/Jobber/signals.zcml +++ b/Products/Jobber/signals.zcml @@ -1,7 +1,12 @@ - + + + diff --git a/Products/Jobber/storage.py b/Products/Jobber/storage.py index 6394e45a43..6d4adfa9d5 100644 --- a/Products/Jobber/storage.py +++ b/Products/Jobber/storage.py @@ -19,7 +19,7 @@ from Products.ZenUtils.RedisUtils import getRedisClient -from .config import Celery +from .config import CeleryConfig _keybase = "zenjobs:job:" _keypattern = _keybase + "*" @@ -31,8 +31,9 @@ def makeJobStore(): """Create and return the ZenJobs JobStore client.""" - client = getRedisClient(url=Celery.CELERY_RESULT_BACKEND) - return JobStore(client, expires=Celery.CELERY_TASK_RESULT_EXPIRES) + cfg = CeleryConfig.from_config() + client = getRedisClient(url=cfg.CELERY_RESULT_BACKEND) + return JobStore(client, expires=cfg.CELERY_TASK_RESULT_EXPIRES) class _Converter(object): diff --git a/Products/Jobber/task/base.py b/Products/Jobber/task/base.py index 666ba1af9b..3353f2469f 100644 --- a/Products/Jobber/task/base.py +++ b/Products/Jobber/task/base.py @@ -17,9 +17,11 @@ from celery import Task from celery.exceptions import Ignore, SoftTimeLimitExceeded -from .event import SendZenossEventMixin +from ..config import getConfig from ..utils.log import get_task_logger, get_logger +from .event import SendZenossEventMixin + _default_summary = "Task {0.__class__.__name__}" mlog = get_logger("zen.zenjobs.task.base") @@ -41,9 +43,7 @@ def __new__(cls, *args, **kwargs): summary = _default_summary.format(task) setattr(cls, "summary", summary) - from Products.Jobber.config import ZenJobs - - task.max_retries = ZenJobs.get("zodb-max-retries", 5) + task.max_retries = getConfig().get("zodb-max-retries", 5) return task diff --git a/Products/Jobber/task/dmd.py b/Products/Jobber/task/dmd.py index 3fd17c337f..7a26a2a43f 100644 --- a/Products/Jobber/task/dmd.py +++ b/Products/Jobber/task/dmd.py @@ -29,7 +29,7 @@ from Products.ZenRelations.ZenPropertyManager import setDescriptors from Products.ZenUtils.Utils import getObjByPath -from ..config import ZenJobs +from ..config import getConfig from ..utils.log import get_logger, get_task_logger, inject_logger mlog = get_logger("zen.zenjobs.task.dmd") @@ -71,7 +71,7 @@ def __retry_on_conflict(self, *args, **kw): except (ReadConflictError, ConflictError) as ex: transaction.abort() self.log.warn("Transaction aborted reason=%s", ex) - limit = ZenJobs.get("zodb-retry-interval-limit", 30) + limit = getConfig().get("zodb-retry-interval-limit", 30) duration = int(SystemRandom().uniform(1, limit)) self.log.info( "Reschedule task to execute after %s seconds.", diff --git a/Products/Jobber/worker.py b/Products/Jobber/worker.py index 10ff9b6730..5e4d538e79 100644 --- a/Products/Jobber/worker.py +++ b/Products/Jobber/worker.py @@ -20,12 +20,22 @@ from Zope2.App import zcml -from .config import ZenJobs +from .config import getConfig, CeleryConfig from .utils.app import get_app _mlog = logging.getLogger("zen.zenjobs.worker") +def apply_config_file(options, app, **kw): + # Note: **kw has 'signal' and 'sender' + cfgfile = options.get("config_file") + if not cfgfile: + cfgfile = "zenjobs.conf" + cfg = getConfig(cfgfile) + celerycfg = CeleryConfig.from_config(cfg) + app.config_from_object(celerycfg) + + def initialize_zenoss_env(**kw): start = time.time() @@ -93,7 +103,7 @@ def report_tasks(**kw): def setup_zodb(**kw): """Initialize a ZODB connection.""" - zodbcfg = ZenJobs.get("zodb-config-file") + zodbcfg = getConfig().get("zodb-config-file") url = "file://%s" % zodbcfg get_app().db = ZODB.config.databaseFromURL(url) _mlog.getChild("setup_zodb").info("ZODB connection initialized") diff --git a/Products/Jobber/zenjobs.py b/Products/Jobber/zenjobs.py index fa2f3593d0..a112f15529 100644 --- a/Products/Jobber/zenjobs.py +++ b/Products/Jobber/zenjobs.py @@ -10,6 +10,7 @@ from __future__ import absolute_import from celery import Celery +from celery.bin import Option from kombu.serialization import register from .serialization import without_unicode @@ -23,10 +24,11 @@ content_encoding="utf-8", ) -app = Celery( - "zenjobs", - config_source="Products.Jobber.config:Celery", - task_cls="Products.Jobber.task:ZenTask", +app = Celery("zenjobs", task_cls="Products.Jobber.task:ZenTask") +app.user_options["preload"].add( + Option( + "--config-file", default=None, help="Name of the configuration file" + ) ) # Allow considerably more time for the worker_process_init signal @@ -35,6 +37,7 @@ # celery 3.1.26 (remove once we update celery) from celery.concurrency import asynpool # noqa E402 + asynpool.PROC_ALIVE_TIMEOUT = 300 # celery 4.4.0+ From 2dcc689cd53447be2ae124e71c2f9e031bb71590 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Sun, 24 Mar 2024 14:43:07 -0500 Subject: [PATCH 107/147] fix: adjust zenjobs config for unit tests ZEN-34771 --- Products/Jobber/config.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 1b9005c8e5..0e6e510a68 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -52,16 +52,19 @@ } -_configuration = None +_configuration = {} def getConfig(filename=None): """Return a dict containing the configuration for zenjobs.""" global _configuration - if filename is None or _configuration is not None: + if _configuration: return _configuration + if filename is None: + filename = "zenjobs.conf" + if not os.path.exists(filename): filename = zenPath("etc", filename) @@ -137,10 +140,10 @@ class CeleryConfig(object): CELERYD_LOG_COLOR = attr.ib(default=False) @classmethod - def from_config(cls, cfg=None): + def from_config(cls, cfg={}): global _celery_config - if cfg is None or _celery_config is not None: + if _celery_config: return _celery_config args = { From 24e2829d8374592917229f52a9708665c56bebba Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 26 Mar 2024 14:25:49 -0500 Subject: [PATCH 108/147] fix: CELERY_TIMEZONE is optional ZEN-34771 --- Products/Jobber/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 0e6e510a68..06bbea29b0 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -102,12 +102,11 @@ def buildBrokerUrl(cfg): _celery_config = None -@attr.s(slots=True) +@attr.s(slots=True, kw_only=True) class CeleryConfig(object): """Celery configuration.""" BROKER_URL = attr.ib() - CELERY_TIMEZONE = attr.ib() CELERY_RESULT_BACKEND = attr.ib() CELERY_TASK_RESULT_EXPIRES = attr.ib() CELERYD_CONCURRENCY = attr.ib() @@ -116,6 +115,7 @@ class CeleryConfig(object): CELERYD_TASK_SOFT_TIME_LIMIT = attr.ib() CELERYBEAT_MAX_LOOP_INTERVAL = attr.ib() + CELERY_TIMEZONE = attr.ib(default=None) CELERY_ACCEPT_CONTENT = attr.ib(default=["without-unicode"]) CELERY_IMPORTS = attr.ib( default=[ From 2bc8855b4b5b381971921436312d667957e1e354 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 26 Mar 2024 23:54:17 -0500 Subject: [PATCH 109/147] fix: ensure Celery app has a useable default config. ZEN-34771 --- Products/Jobber/config.py | 42 +++++++++++++++----------------------- Products/Jobber/zenjobs.py | 22 +++++++++++++++----- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 06bbea29b0..99d80eefd9 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -62,22 +62,20 @@ def getConfig(filename=None): if _configuration: return _configuration - if filename is None: - filename = "zenjobs.conf" - - if not os.path.exists(filename): - filename = zenPath("etc", filename) - conf = _default_configs.copy() conf.update(getGlobalConfiguration()) - app_config_loader = ConfigLoader(filename, Config) - try: - conf.update(app_config_loader()) - except IOError as ex: - # Re-raise exception if the error is not "File not found" - if ex.errno != 2: - raise + if filename is not None: + if not os.path.exists(filename): + filename = zenPath("etc", filename) + + app_config_loader = ConfigLoader([filename], Config) + try: + conf.update(app_config_loader()) + except IOError as ex: + # Re-raise exception if the error is not "File not found" + if ex.errno != 2: + raise # Convert the configuration value types to useable types. for key, cast in _xform.items(): @@ -85,7 +83,10 @@ def getConfig(filename=None): continue conf[key] = cast(conf[key]) - _configuration = conf + # only save it if a filename was specified + if filename is not None: + _configuration = conf + return conf @@ -99,9 +100,6 @@ def buildBrokerUrl(cfg): return "amqp://{usr}:{pwd}@{host}:{port}/{vhost}".format(**locals()) -_celery_config = None - - @attr.s(slots=True, kw_only=True) class CeleryConfig(object): """Celery configuration.""" @@ -141,13 +139,8 @@ class CeleryConfig(object): @classmethod def from_config(cls, cfg={}): - global _celery_config - - if _celery_config: - return _celery_config - args = { - "BROKER_URL": buildBrokerUrl(_configuration), + "BROKER_URL": buildBrokerUrl(cfg), "CELERY_RESULT_BACKEND": cfg.get("redis-url"), "CELERY_TASK_RESULT_EXPIRES": cfg.get("zenjobs-job-expires"), "CELERYD_CONCURRENCY": cfg.get("concurrent-jobs"), @@ -162,5 +155,4 @@ def from_config(cls, cfg={}): if tz: args["CELERY_TIMEZONE"] = tz - _celery_config = cls(**args) - return _celery_config + return cls(**args) diff --git a/Products/Jobber/zenjobs.py b/Products/Jobber/zenjobs.py index a112f15529..7f07fd6dde 100644 --- a/Products/Jobber/zenjobs.py +++ b/Products/Jobber/zenjobs.py @@ -24,12 +24,24 @@ content_encoding="utf-8", ) -app = Celery("zenjobs", task_cls="Products.Jobber.task:ZenTask") -app.user_options["preload"].add( - Option( - "--config-file", default=None, help="Name of the configuration file" + +def _buildapp(): + from .config import CeleryConfig, getConfig + app = Celery( + "zenjobs", + task_cls="Products.Jobber.task:ZenTask", ) -) + default = CeleryConfig.from_config(getConfig()) + app.config_from_object(default) + app.user_options["preload"].add( + Option( + "--config-file", default=None, help="Name of the configuration file" + ) + ) + return app + + +app = _buildapp() # Allow considerably more time for the worker_process_init signal # to complete (rather than the default of 4 seconds). This is required From b4f093f023d2240d4539fae93fa33614f60e1b75 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 27 Mar 2024 12:38:50 -0500 Subject: [PATCH 110/147] fix: send build jobs for updated devices to pick up new configs Always send build jobs to pick up potentially missed configs to support devices, like vSphere, that defer their modeling until a later time. ZEN-34729 --- .../ZenCollector/configcache/invalidator.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 3a44f370c3..d4660465f8 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -174,7 +174,7 @@ def _process(self, invalidation, uid, buildlimit, minttl): if uid != stored_uid: self._changed_device_class(device, monitor, buildlimit) else: - self._updated_device(device, monitor, keys, minttl) + self._updated_device(device, monitor, keys, minttl, buildlimit) elif reason is InvalidationCause.Removed: self._removed_device(keys) else: @@ -230,7 +230,7 @@ def _changed_device_class(self, device, monitor, buildlimit): monitor, ) - def _updated_device(self, device, monitor, keys, minttl): + def _updated_device(self, device, monitor, keys, minttl, buildlimit): statuses = tuple( status for status in self.store.get_status(*keys) @@ -264,6 +264,27 @@ def _updated_device(self, device, monitor, keys, minttl): key.service, ) + # Send a job for for all config services that don't currently have + # an associated configuration. Some ZenPacks, i.e. vSphere, defer + # their modeling to a later time, so jobs for configuration services + # must be sent to pick up any new configs. + hasconfigs = tuple(key.service for key in keys) + noconfigkeys = tuple( + CacheKey(svcname, monitor, device.id) + for svcname in self.dispatcher.service_names + if svcname not in hasconfigs + ) + # Identify all no-config keys that already have a status. + skipkeys = tuple( + status.key for status in self.store.get_status(*noconfigkeys) + ) + now = time.time() + for key in (k for k in noconfigkeys if k not in skipkeys): + self.store.set_pending((key, now)) + self.dispatcher.dispatch( + key.service, key.monitor, key.device, buildlimit + ) + def _removed_device(self, keys): self.store.remove(*keys) for key in keys: From 1717aab9654542d4dfb4a0bd7eccf2fea6529e29 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 29 Mar 2024 16:41:27 -0500 Subject: [PATCH 111/147] fix: keep status while saving config when status is not Building. When config's status is changed to Expired or Pending while the config is building, save the built config without changing its status. ZEN-34754 --- .../ZenCollector/configcache/cache/storage.py | 37 ++-- .../ZenCollector/configcache/invalidator.py | 8 +- Products/ZenCollector/configcache/manager.py | 4 +- Products/ZenCollector/configcache/task.py | 171 ++++++++++++------ .../configcache/tests/test_dispatcher.py | 15 +- 5 files changed, 145 insertions(+), 90 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index b83d01f402..be5d1e51ea 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -140,6 +140,17 @@ def add(self, record): """ @type record: CacheRecord """ + self._add(record, self._delete_statuses) + + def put_config(self, record): + """ + Updates the config without changing its status. + + @type record: CacheRecord + """ + self._add(record) + + def _add(self, record, statushandler=lambda *args: None): svc, mon, dvc, uid, updated, config = _from_record(record) orphaned_keys = tuple( key @@ -159,18 +170,12 @@ def _add_impl(pipe): parts = (key.service, key.monitor, key.device) self.__config.delete(pipe, *parts) self.__age.delete(pipe, *parts) - self.__retired.delete(pipe, *parts) - self.__expired.delete(pipe, *parts) - self.__pending.delete(pipe, *parts) - self.__building.delete(pipe, *parts) + self._delete_statuses(pipe, *parts) if stored_uid != uid: self.__uids.set(pipe, dvc, uid) self.__config.set(pipe, svc, mon, dvc, config) self.__age.add(pipe, svc, mon, dvc, updated) - self.__retired.delete(pipe, svc, mon, dvc) - self.__expired.delete(pipe, svc, mon, dvc) - self.__pending.delete(pipe, svc, mon, dvc) - self.__building.delete(pipe, svc, mon, dvc) + statushandler(pipe, svc, mon, dvc) self.__client.transaction(_add_impl, *watch_keys) @@ -222,10 +227,7 @@ def remove(self, *keys): svc, mon, dvc = key.service, key.monitor, key.device self.__config.delete(pipe, svc, mon, dvc) self.__age.delete(pipe, svc, mon, dvc) - self.__retired.delete(pipe, svc, mon, dvc) - self.__expired.delete(pipe, svc, mon, dvc) - self.__pending.delete(pipe, svc, mon, dvc) - self.__building.delete(pipe, svc, mon, dvc) + self._delete_statuses(pipe, svc, mon, dvc) pipe.execute() devices = set(key.device for key in keys) @@ -259,12 +261,15 @@ def clear_status(self, *keys): with self.__client.pipeline() as pipe: for key in keys: svc, mon, dvc = key.service, key.monitor, key.device - self.__retired.delete(pipe, svc, mon, dvc) - self.__expired.delete(pipe, svc, mon, dvc) - self.__pending.delete(pipe, svc, mon, dvc) - self.__building.delete(pipe, svc, mon, dvc) + self._delete_statuses(pipe, svc, mon, dvc) pipe.execute() + def _delete_statuses(self, pipe, svc, mon, dvc): + self.__retired.delete(pipe, svc, mon, dvc) + self.__expired.delete(pipe, svc, mon, dvc) + self.__pending.delete(pipe, svc, mon, dvc) + self.__building.delete(pipe, svc, mon, dvc) + def set_retired(self, *pairs): """ Marks the indicated configuration(s) as retired. diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index d4660465f8..1fc776ec59 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -206,7 +206,7 @@ def _new_device(self, device, monitor, buildlimit): now = time.time() for key in keys: self.store.set_pending((key, now)) - self.dispatcher.dispatch_all(monitor, device.id, buildlimit) + self.dispatcher.dispatch_all(monitor, device.id, buildlimit, now) self.log.info( "submitted build jobs for new device device=%s collector=%s", device.id, @@ -222,7 +222,7 @@ def _changed_device_class(self, device, monitor, buildlimit): now = time.time() for key in keys: self.store.set_pending((key, now)) - self.dispatcher.dispatch_all(monitor, device.id, buildlimit) + self.dispatcher.dispatch_all(monitor, device.id, buildlimit, now) self.log.info( "submitted build jobs for device with new device class " "device=%s collector=%s", @@ -282,7 +282,7 @@ def _updated_device(self, device, monitor, keys, minttl, buildlimit): for key in (k for k in noconfigkeys if k not in skipkeys): self.store.set_pending((key, now)) self.dispatcher.dispatch( - key.service, key.monitor, key.device, buildlimit + key.service, key.monitor, key.device, buildlimit, now ) def _removed_device(self, keys): @@ -369,7 +369,7 @@ def _addNewOrChangedDevices(log, tool, timelimitmap, store, dispatcher): ) for key in keys: store.set_pending((key, now)) - dispatcher.dispatch_all(brain.collector, brain.id, timeout) + dispatcher.dispatch_all(brain.collector, brain.id, timeout, now) log.info( "submitted build jobs for device %s " "uid=%s collector=%s", ( diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index ec3154f0a9..ef10b96f23 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -197,12 +197,14 @@ def _rebuild_configs(self, statuses): count = 0 for status in statuses: timeout = buildlimitmap.get(status.uid) - self.store.set_pending((status.key, time())) + now = time() + self.store.set_pending((status.key, now)) self.dispatcher.dispatch( status.key.service, status.key.monitor, status.key.device, timeout, + now, ) if isinstance(status, ConfigStatus.Expired): self.log.info( diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 6acd9b2b41..1402207348 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -58,28 +58,39 @@ def build_device_config( # Check whether this is an old job, i.e. job pending timeout. # If it is an old job, skip it, manager already sent another one. status = next(store.get_status(key), None) - if status is not None and submitted is not None: - if isinstance(status, ConfigStatus.Pending): - pendinglimitmap = DevicePropertyMap.make_pending_timeout_map( - self.dmd.Devices + if _job_is_old(status, submitted, self.dmd.Devices, self.log): + return + + # If the status is Expired, another job is coming, so skip this job. + if isinstance(status, ConfigStatus.Expired): + self.log.warn( + "skipped this job because another job is coming " + "device=%s collector=%s service=%s submitted=%f", + key.device, + key.monitor, + key.service, + submitted, + ) + return + + # If the status is Pending, verify whether it's for this job, and if not, + # skip this job. + if isinstance(status, ConfigStatus.Pending): + s1 = int(submitted * 1000) + s2 = int(status.submitted * 1000) + if s1 != s2: + self.log.warn( + "skipped this job in favor of newer job " + "device=%s collector=%s service=%s submitted=%f", + key.device, + key.monitor, + key.service, + submitted, ) - now = time() - duration = pendinglimitmap.get(status.uid) - if submitted < (now - duration): - self.log.warn( - "dropped this job in favor of newer job " - "device=%s collector=%s service=%s submitted=%f %s=%s", - deviceid, - monitorname, - svcname, - submitted, - Constants.pending_timeout_id, - duration, - ) - return - - # Change the configuration's status from 'pending' to 'building' so - # that configcache-manager doesn't prematurely timeout the build. + return + + # Change the configuration's status to 'building' to indicate that + # a config is now building. store.set_building((key, time())) self.log.info( "building device configuration device=%s collector=%s service=%s", @@ -92,46 +103,87 @@ def build_device_config( result = service.remote_getDeviceConfigs((deviceid,)) config = result[0] if result else None if config is None: - self.log.info( - "no configuration built device=%s collector=%s service=%s", - deviceid, - monitorname, - svcname, - ) - oldkey = next( - store.search( - CacheQuery( - service=svcname, monitor=monitorname, device=deviceid - ) - ), - None, - ) - if oldkey is not None: - # No result means device was deleted or moved to another monitor. - store.remove(oldkey) - self.log.info( - "removed previously built configuration " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) - # Ensure any status on this key is removed - store.clear_status(key) + _delete_config(key, store, self.log) else: uid = self.dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() record = CacheRecord.make( svcname, monitorname, deviceid, uid, time(), config ) - store.add(record) - self.log.info( - "added/replaced config " - "updated=%s device=%s collector=%s service=%s", - datetime.fromtimestamp(record.updated).isoformat(), - deviceid, - monitorname, - svcname, + # Get the current status of the configuration. + status = next(store.get_status(key), None) + if isinstance(status, (ConfigStatus.Expired, ConfigStatus.Pending)): + # status is not ConfigStatus.Building, so another job will be + # submitted or has already been submitted. + store.put_config(record) + self.log.info( + "saved config without changing status " + "updated=%s device=%s collector=%s service=%s", + datetime.fromtimestamp(record.updated).isoformat(), + deviceid, + monitorname, + svcname, + ) + else: + verb = "replaced" if status is not None else "added" + store.add(record) + self.log.info( + "%s config updated=%s device=%s collector=%s service=%s", + verb, + datetime.fromtimestamp(record.updated).isoformat(), + deviceid, + monitorname, + svcname, + ) + + +def _delete_config(key, store, log): + log.info( + "no configuration built device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) + oldkey = next( + store.search( + CacheQuery( + service=key.service, monitor=key.monitor, device=key.device + ) + ), + None, + ) + if oldkey is not None: + store.remove(oldkey) + log.info( + "removed previously built configuration " + "device=%s collector=%s service=%s", + oldkey.device, + oldkey.monitor, + oldkey.service, + ) + # Ensure any status on this key is removed + store.clear_status(oldkey) + + +def _job_is_old(status, submitted, ctx, log): + if (submitted is None or status is None): + # job is not old (default state) + return False + pendinglimitmap = DevicePropertyMap.make_pending_timeout_map(ctx) + duration = pendinglimitmap.get(status.uid) + now = time() + if submitted < (now - duration): + log.warn( + "skipped this job because it's too old " + "device=%s collector=%s service=%s submitted=%f %s=%s", + status.key.device, + status.key.monitor, + status.key.service, + submitted, + Constants.pending_timeout_id, + duration, ) + return True + return False class BuildConfigTaskDispatcher(object): @@ -155,22 +207,21 @@ def __init__(self, configClasses): def service_names(self): return self._classnames.keys() - def dispatch_all(self, monitorid, deviceid, timeout): + def dispatch_all(self, monitorid, deviceid, timeout, submitted): """ Submit a task to build a device configuration from each configuration service. """ soft_limit, hard_limit = _get_limits(timeout) - now = time() for name in self._classnames.values(): build_device_config.apply_async( args=(monitorid, deviceid, name), - kwargs={"submitted": now}, + kwargs={"submitted": submitted}, soft_time_limit=soft_limit, time_limit=hard_limit, ) - def dispatch(self, servicename, monitorid, deviceid, timeout): + def dispatch(self, servicename, monitorid, deviceid, timeout, submitted): """ Submit a task to build device configurations for the specified device. @@ -184,7 +235,7 @@ def dispatch(self, servicename, monitorid, deviceid, timeout): soft_limit, hard_limit = _get_limits(timeout) build_device_config.apply_async( args=(monitorid, deviceid, name), - kwargs={"submitted": time()}, + kwargs={"submitted": submitted}, soft_time_limit=soft_limit, time_limit=hard_limit, ) diff --git a/Products/ZenCollector/configcache/tests/test_dispatcher.py b/Products/ZenCollector/configcache/tests/test_dispatcher.py index 10baf4eed9..fa8c078fd0 100644 --- a/Products/ZenCollector/configcache/tests/test_dispatcher.py +++ b/Products/ZenCollector/configcache/tests/test_dispatcher.py @@ -34,17 +34,15 @@ def setUp(t): t.bctd = BuildConfigTaskDispatcher((t.class_a, t.class_b)) - @patch("{src}.time".format(**PATH), autospec=True) @patch.object(build_device_config, "apply_async") - def test_dispatch_all(t, _apply_async, _time): + def test_dispatch_all(t, _apply_async): timeout = 100.0 soft = 100.0 hard = 110.0 submitted = 111.0 - _time.return_value = submitted monitor = "local" device = "linux" - t.bctd.dispatch_all(monitor, device, timeout) + t.bctd.dispatch_all(monitor, device, timeout, submitted) _apply_async.assert_has_calls( ( @@ -63,18 +61,16 @@ def test_dispatch_all(t, _apply_async, _time): ) ) - @patch("{src}.time".format(**PATH), autospec=True) @patch.object(build_device_config, "apply_async") - def test_dispatch(t, _apply_async, _time): + def test_dispatch(t, _apply_async): timeout = 100.0 soft = 100.0 hard = 110.0 submitted = 111.0 - _time.return_value = submitted monitor = "local" device = "linux" svcname = t.class_a.__module__ - t.bctd.dispatch(svcname, monitor, device, timeout) + t.bctd.dispatch(svcname, monitor, device, timeout, submitted) _apply_async.assert_called_once_with( args=(monitor, device, t.class_a_name), @@ -87,6 +83,7 @@ def test_dispatch_unknown_service(t): timeout = 100.0 monitor = "local" device = "linux" + submitted = 1111.0 with t.assertRaises(ValueError): - t.bctd.dispatch("unknown", monitor, device, timeout) + t.bctd.dispatch("unknown", monitor, device, timeout, submitted) From 2913d54293f93a6fad46da8d078aa261ead28a5c Mon Sep 17 00:00:00 2001 From: azam Date: Mon, 1 Apr 2024 17:24:57 +0300 Subject: [PATCH 112/147] ZEN-34777: ZenRelations Unit Test Error with DistributedCollector ZenPack --- Products/ZenRelations/tests/testEvents.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Products/ZenRelations/tests/testEvents.py b/Products/ZenRelations/tests/testEvents.py index c5d459be70..326996b6b5 100644 --- a/Products/ZenRelations/tests/testEvents.py +++ b/Products/ZenRelations/tests/testEvents.py @@ -24,6 +24,7 @@ BaseTestCase, init_model_catalog_for_tests, ) +from Products.ZenUtils.ZenDocTest import load_unittest_site class EventLogger(object): @@ -95,7 +96,7 @@ class EventLayer(ZopeLite): def setUp(cls): import Products # noqa F401 - zcml.load_site(force=True) + load_unittest_site(force=True) setHooks() # Register Model Catalog related stuff From 958e0268beabc60f090d063c1aec403d58d15980 Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:11:29 +0300 Subject: [PATCH 113/147] Update zenjobs to use Celery v4.4.7 (#4398) Fixes ZEN-33726. *Fixed unittests *Changed serialization registration for new version kombu compatibility *Added job registration *Task name attribute setting *Added config variables * Configcache changes for redis 3.5.3 compatibility --- Products/Jobber/config.py | 71 ++++++++++--------- Products/Jobber/jobs/facade.py | 6 +- Products/Jobber/jobs/misc.py | 5 ++ Products/Jobber/jobs/roles.py | 4 ++ Products/Jobber/jobs/subprocess.py | 4 ++ Products/Jobber/meta.py | 9 ++- Products/Jobber/monitor.py | 2 +- Products/Jobber/scheduler.py | 3 +- Products/Jobber/storage.py | 4 +- Products/Jobber/task/base.py | 4 +- Products/Jobber/task/event.py | 2 +- Products/Jobber/tests/test_manager.py | 4 +- Products/Jobber/tests/test_redisrecord.py | 4 ++ Products/Jobber/tests/test_zentask.py | 2 +- Products/Jobber/utils/log.py | 4 +- Products/Jobber/utils/utils.py | 32 +++++++++ Products/Jobber/zenjobs.py | 16 +---- .../ZenCollector/configcache/cache/storage.py | 5 +- .../configcache/cache/table/metadata.py | 2 +- Products/ZenModel/IpNetwork.py | 6 +- Products/ZenModel/ZDeviceLoader.py | 4 +- 21 files changed, 128 insertions(+), 65 deletions(-) create mode 100644 Products/Jobber/utils/utils.py diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 99d80eefd9..0edb2fa143 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -33,7 +33,9 @@ "concurrent-jobs": 1, "job-hard-time-limit": 21600, # 6 hours "job-soft-time-limit": 18000, # 5 hours + "zenjobs-worker-alive-timeout": 300.0, # 5 minutes "redis-url": DEFAULT_REDIS_URL, + "task-protocol": 1, } @@ -49,6 +51,7 @@ "zenjobs-job-expires": int, "zodb-max-retries": int, "zodb-retry-interval-limit": int, + "zenjobs-worker-alive-timeout": float, } @@ -104,55 +107,59 @@ def buildBrokerUrl(cfg): class CeleryConfig(object): """Celery configuration.""" - BROKER_URL = attr.ib() - CELERY_RESULT_BACKEND = attr.ib() - CELERY_TASK_RESULT_EXPIRES = attr.ib() - CELERYD_CONCURRENCY = attr.ib() - CELERYD_MAX_TASKS_PER_CHILD = attr.ib() - CELERYD_TASK_TIME_LIMIT = attr.ib() - CELERYD_TASK_SOFT_TIME_LIMIT = attr.ib() - CELERYBEAT_MAX_LOOP_INTERVAL = attr.ib() - - CELERY_TIMEZONE = attr.ib(default=None) - CELERY_ACCEPT_CONTENT = attr.ib(default=["without-unicode"]) - CELERY_IMPORTS = attr.ib( + broker_url = attr.ib() + result_backend = attr.ib() + result_expires = attr.ib() + worker_concurrency = attr.ib() + worker_max_tasks_per_child = attr.ib() + task_time_limit = attr.ib() + task_soft_time_limit = attr.ib() + beat_max_loop_interval = attr.ib() + + timezone = attr.ib(default=None) + accept_content = attr.ib(default=["without-unicode"]) + imports = attr.ib( default=[ "Products.Jobber.jobs", "Products.ZenCollector.configcache.task", ] ) - CELERY_ROUTES = attr.ib( + task_routes = attr.ib( default={"configcache.build_device_config": {"queue": "configcache"}} ) - CELERY_RESULT_SERIALIZER = attr.ib(default="without-unicode") - CELERYD_PREFETCH_MULTIPLIER = attr.ib(default=1) - CELERY_ACKS_LATE = attr.ib(default=True) - CELERY_IGNORE_RESULT = attr.ib(default=False) - CELERY_STORE_ERRORS_EVEN_IF_IGNORED = attr.ib(default=True) - CELERY_TASK_SERIALIZER = attr.ib(default="without-unicode") - CELERY_TRACK_STARTED = attr.ib(default=True) + result_serializer = attr.ib(default="without-unicode") + worker_prefetch_multiplier = attr.ib(default=1) + task_acks_late = attr.ib(default=True) + task_ignore_result = attr.ib(default=False) + task_store_errors_even_if_ignored = attr.ib(default=True) + task_serializer = attr.ib(default="without-unicode") + task_track_started = attr.ib(default=True) CELERYBEAT_REDIRECT_STDOUTS = attr.ib(default=True) CELERYBEAT_REDIRECT_STDOUTS_LEVEL = attr.ib(default="INFO") - CELERY_SEND_EVENTS = attr.ib(default=True) - CELERY_SEND_TASK_SENT_EVENT = attr.ib(default=True) - CELERYD_LOG_COLOR = attr.ib(default=False) + worker_send_task_events = attr.ib(default=True) + task_send_sent_event = attr.ib(default=True) + worker_log_color = attr.ib(default=False) + worker_proc_alive_timeout = attr.ib(default=300.0) + task_protocol = attr.ib(default=1) @classmethod def from_config(cls, cfg={}): args = { - "BROKER_URL": buildBrokerUrl(cfg), - "CELERY_RESULT_BACKEND": cfg.get("redis-url"), - "CELERY_TASK_RESULT_EXPIRES": cfg.get("zenjobs-job-expires"), - "CELERYD_CONCURRENCY": cfg.get("concurrent-jobs"), - "CELERYD_MAX_TASKS_PER_CHILD": cfg.get("max-jobs-per-worker"), - "CELERYD_TASK_TIME_LIMIT": cfg.get("job-hard-time-limit"), - "CELERYD_TASK_SOFT_TIME_LIMIT": cfg.get("job-soft-time-limit"), - "CELERYBEAT_MAX_LOOP_INTERVAL": cfg.get( + "broker_url": buildBrokerUrl(cfg), + "result_backend": cfg.get("redis-url"), + "result_expires": cfg.get("zenjobs-job-expires"), + "worker_concurrency": cfg.get("concurrent-jobs"), + "worker_max_tasks_per_child": cfg.get("max-jobs-per-worker"), + "task_time_limit": cfg.get("job-hard-time-limit"), + "task_soft_time_limit": cfg.get("job-soft-time-limit"), + "beat_max_loop_interval": cfg.get( "scheduler-max-loop-interval" ), + "worker_proc_alive_timeout": cfg.get("zenjobs-worker-alive-timeout"), + "task_protocol": cfg.get("task-protocol", 1), } tz = os.environ.get("TZ") if tz: - args["CELERY_TIMEZONE"] = tz + args["timezone"] = tz return cls(**args) diff --git a/Products/Jobber/jobs/facade.py b/Products/Jobber/jobs/facade.py index 4a7ed29877..d5288e5134 100644 --- a/Products/Jobber/jobs/facade.py +++ b/Products/Jobber/jobs/facade.py @@ -11,7 +11,7 @@ import inspect -from celery.utils import fun_takes_kwargs +from ..utils.utils import fun_takes_kwargs from zope.dottedname.resolve import resolve from ..exceptions import FacadeMethodJobFailed @@ -103,3 +103,7 @@ def _run(self, facadefqdn, method, *args, **kwargs): result, ) return result + + +from Products.Jobber.zenjobs import app +app.register_task(FacadeMethodJob) diff --git a/Products/Jobber/jobs/misc.py b/Products/Jobber/jobs/misc.py index 3905d56174..61fcb63105 100644 --- a/Products/Jobber/jobs/misc.py +++ b/Products/Jobber/jobs/misc.py @@ -73,6 +73,11 @@ def _run(self, seconds, *args, **kw): raise DelayedFailureError("slept for %s seconds" % seconds) +app.register_task(DeviceListJob) +app.register_task(PausingJob) +app.register_task(DelayedFailure) + + @app.task( bind=True, base=requires(DMD, Abortable), diff --git a/Products/Jobber/jobs/roles.py b/Products/Jobber/jobs/roles.py index 013d33909d..6a899285e6 100644 --- a/Products/Jobber/jobs/roles.py +++ b/Products/Jobber/jobs/roles.py @@ -37,3 +37,7 @@ def _run(self, organizerUid, *args, **kwargs): self.log.info("About to set local roles for uid: %s ", organizerUid) organizer = self.dmd.unrestrictedTraverse(organizerUid) organizer._setDeviceLocalRoles() + + +from Products.Jobber.zenjobs import app +app.register_task(DeviceSetLocalRolesJob) diff --git a/Products/Jobber/jobs/subprocess.py b/Products/Jobber/jobs/subprocess.py index 9bdbbe4495..0efad8ff65 100644 --- a/Products/Jobber/jobs/subprocess.py +++ b/Products/Jobber/jobs/subprocess.py @@ -130,6 +130,10 @@ def _handle_process(self, process): reader.join(timeout=1.0) +from Products.Jobber.zenjobs import app +app.register_task(SubprocessJob) + + @contextmanager def null_context(): """Do nothing context manager.""" diff --git a/Products/Jobber/meta.py b/Products/Jobber/meta.py index 8135795f66..d158a43ff0 100644 --- a/Products/Jobber/meta.py +++ b/Products/Jobber/meta.py @@ -41,8 +41,13 @@ def job(_context, **kw): raise ConfigurationError( ("Missing parameter:", "'task' or 'class'") ) - if task.name not in app.tasks: - app.tasks.register(task) + + if not task.name or task.name not in app.tasks: + try: + registered_task = app.register_task(task) + setattr(registered_task.__class__, 'name', registered_task.name) + except Exception as e: + raise Exception("Task registration failed: %s" % e) class ICelerySignal(Interface): diff --git a/Products/Jobber/monitor.py b/Products/Jobber/monitor.py index ae17b1b4cd..1d408da8c0 100644 --- a/Products/Jobber/monitor.py +++ b/Products/Jobber/monitor.py @@ -87,7 +87,7 @@ def _getTimeoutSummary(app, ex): return "Job killed after {}.".format( humanize_timedelta( timedelta( - seconds=app.conf.get("CELERYD_TASK_SOFT_TIME_LIMIT"), + seconds=app.conf.get("task_soft_time_limit"), ), ), ) diff --git a/Products/Jobber/scheduler.py b/Products/Jobber/scheduler.py index d60c86ff0d..090a2ee1f3 100644 --- a/Products/Jobber/scheduler.py +++ b/Products/Jobber/scheduler.py @@ -223,10 +223,11 @@ def _key(name): def _getClient(): """Create and return the ZenJobs JobStore client.""" return redis.StrictRedis.from_url( - CeleryConfig.from_config().CELERY_RESULT_BACKEND + CeleryConfig.from_config().result_backend ) + def handle_beat_init(*args, **kw): beat_logger = logging.getLogger("celery.beat") zenjobs_logger = logging.getLogger("zen.zenjobs") diff --git a/Products/Jobber/storage.py b/Products/Jobber/storage.py index 6d4adfa9d5..dde23dcf25 100644 --- a/Products/Jobber/storage.py +++ b/Products/Jobber/storage.py @@ -32,8 +32,8 @@ def makeJobStore(): """Create and return the ZenJobs JobStore client.""" cfg = CeleryConfig.from_config() - client = getRedisClient(url=cfg.CELERY_RESULT_BACKEND) - return JobStore(client, expires=cfg.CELERY_TASK_RESULT_EXPIRES) + client = getRedisClient(url=cfg.result_backend) + return JobStore(client, expires=cfg.result_expires) class _Converter(object): diff --git a/Products/Jobber/task/base.py b/Products/Jobber/task/base.py index 3353f2469f..87cb8aeffa 100644 --- a/Products/Jobber/task/base.py +++ b/Products/Jobber/task/base.py @@ -68,7 +68,7 @@ def description(self): kw = req.kwargs if req.kwargs else {} return type(self).description_from(*args, **kw) - def subtask(self, *args, **kw): + def signature(self, *args, **kw): """Return celery.signature object for this task. This overridden version adds the currently logged in user's ID @@ -88,7 +88,7 @@ def subtask(self, *args, **kw): headers["userid"] = userid if self.request.id is None: kw["task_id"] = str(uuid.uuid4()) - return super(ZenTask, self).subtask(*args, **kw) + return super(ZenTask, self).signature(*args, **kw) def on_failure(self, exc, task_id, args, kwargs, einfo): result = super(ZenTask, self).on_failure( diff --git a/Products/Jobber/task/event.py b/Products/Jobber/task/event.py index 30ca0d9115..4af174d300 100644 --- a/Products/Jobber/task/event.py +++ b/Products/Jobber/task/event.py @@ -70,7 +70,7 @@ def _send_event(task, exc, task_id, args, kwargs): def _getTimeoutSummary(task, ex): _, soft_limit = task.request.timelimit or (None, None) if soft_limit is None: - soft_limit = task.app.conf.get("CELERYD_TASK_SOFT_TIME_LIMIT") + soft_limit = task.app.conf.get("task_soft_time_limit") return "Job timed out after {}.".format( humanize_timedelta(timedelta(seconds=soft_limit)) ) diff --git a/Products/Jobber/tests/test_manager.py b/Products/Jobber/tests/test_manager.py index 1afcc58882..70af4d3467 100644 --- a/Products/Jobber/tests/test_manager.py +++ b/Products/Jobber/tests/test_manager.py @@ -138,7 +138,9 @@ def test_query_return_value(t): def test_getUnfinishedJobs_all_types(t): expected = [] - for idx, st in enumerate(states.ALL_STATES): + # in celery 4.4.7 REJECTED was added to UNREADY_STATES (only used in events) + # but it wasn't included to ALL_STATES + for idx, st in enumerate(states.ALL_STATES | states.UNREADY_STATES): rec = dict(t.full, status=st, jobid="abc-{}".format(idx)) t.store[rec["jobid"]] = rec if st in states.UNREADY_STATES: diff --git a/Products/Jobber/tests/test_redisrecord.py b/Products/Jobber/tests/test_redisrecord.py index 3793b3819b..679a24894c 100644 --- a/Products/Jobber/tests/test_redisrecord.py +++ b/Products/Jobber/tests/test_redisrecord.py @@ -169,6 +169,7 @@ class BuildRedisRecordFromJobTest(BaseBuildRedisRecord, TestCase): """Test the RedisRecord class with a Job.""" class TestJob(Job): + name = "TestJob" @classmethod def getJobType(cls): return "Test Job" @@ -177,6 +178,9 @@ def getJobType(cls): def getJobDescription(cls, *args, **kw): return "TestJob %s %s" % (args, kw) + from Products.Jobber.zenjobs import app + app.register_task(TestJob) + def setUp(t): t.task = t.TestJob() BaseBuildRedisRecord.setUp(t) diff --git a/Products/Jobber/tests/test_zentask.py b/Products/Jobber/tests/test_zentask.py index 3ffc10578a..770358ac5e 100644 --- a/Products/Jobber/tests/test_zentask.py +++ b/Products/Jobber/tests/test_zentask.py @@ -103,6 +103,6 @@ def test_subtask(t, _uuid): "headers": {"userid": None}, "task_id": task_id, } - task = t.simple_task.subtask() + task = t.simple_task.signature() t.assertIsInstance(task, Signature) t.assertDictEqual(expected, task.options) diff --git a/Products/Jobber/utils/log.py b/Products/Jobber/utils/log.py index f937a5d8f9..4a163bc55b 100644 --- a/Products/Jobber/utils/log.py +++ b/Products/Jobber/utils/log.py @@ -18,7 +18,7 @@ from functools import wraps -from celery.app import current_task +from celery._state import get_current_task from celery.utils.log import ( LoggingProxy as _LoggingProxy, logger_isa as _logger_isa, @@ -152,7 +152,7 @@ def __init__(self, base=None, task=None, datefmt=None): super(TaskFormatter, self).__init__(datefmt=datefmt) def format(self, record): # noqa: A003 - task = current_task() + task = get_current_task() if task and task.request: self._fmt = self._task record.__dict__.update( diff --git a/Products/Jobber/utils/utils.py b/Products/Jobber/utils/utils.py new file mode 100644 index 0000000000..7ed5e582ec --- /dev/null +++ b/Products/Jobber/utils/utils.py @@ -0,0 +1,32 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## +from inspect import getargspec + + +def fun_takes_kwargs(fun, kwlist=[]): + """With a function, and a list of keyword arguments, returns arguments + in the list which the function takes. + If the object has an `argspec` attribute that is used instead + of using the :meth:`inspect.getargspec` introspection. + :param fun: The function to inspect arguments of. + :param kwlist: The list of keyword arguments. + Examples + >>> def foo(self, x, y, logfile=None, loglevel=None): + ... return x * y + >>> fun_takes_kwargs(foo, ['logfile', 'loglevel', 'task_id']) + ['logfile', 'loglevel'] + >>> def foo(self, x, y, **kwargs): + >>> fun_takes_kwargs(foo, ['logfile', 'loglevel', 'task_id']) + ['logfile', 'loglevel', 'task_id'] + """ + + S = getattr(fun, 'argspec', getargspec(fun)) + if S.keywords is not None: + return kwlist + return [kw for kw in kwlist if kw in S.args] diff --git a/Products/Jobber/zenjobs.py b/Products/Jobber/zenjobs.py index 7f07fd6dde..6e088e01f3 100644 --- a/Products/Jobber/zenjobs.py +++ b/Products/Jobber/zenjobs.py @@ -11,12 +11,12 @@ from celery import Celery from celery.bin import Option -from kombu.serialization import register +from kombu import serialization from .serialization import without_unicode # Register custom serializer -register( +serialization.register( "without-unicode", without_unicode.dump, without_unicode.load, @@ -42,15 +42,3 @@ def _buildapp(): app = _buildapp() - -# Allow considerably more time for the worker_process_init signal -# to complete (rather than the default of 4 seconds). This is required -# because loading the zenoss environment / zenpacks can take a while. - -# celery 3.1.26 (remove once we update celery) -from celery.concurrency import asynpool # noqa E402 - -asynpool.PROC_ALIVE_TIMEOUT = 300 - -# celery 4.4.0+ -# Set Products.Jobber.config.Celery.worker_proc_alive_timeout = 300 diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index be5d1e51ea..23bd035148 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -60,6 +60,7 @@ from __future__ import absolute_import, print_function, division import ast +import json import logging from functools import partial @@ -510,7 +511,7 @@ def _range(client, metadata, svc, mon, minv=None, maxv=None): def _unjelly(data): - return unjelly(ast.literal_eval(data)) + return unjelly(json.loads(data)) def _to_score(ts): @@ -535,7 +536,7 @@ def _from_record(record): record.device, record.uid, _to_score(record.updated), - jelly(record.config), + json.dumps(jelly(record.config)), ) diff --git a/Products/ZenCollector/configcache/cache/table/metadata.py b/Products/ZenCollector/configcache/cache/table/metadata.py index b88411eac0..222186b366 100644 --- a/Products/ZenCollector/configcache/cache/table/metadata.py +++ b/Products/ZenCollector/configcache/cache/table/metadata.py @@ -96,7 +96,7 @@ def add(self, client, service, monitor, device, score): @type score: float """ key = self.make_key(service, monitor) - client.zadd(key, score, device) + client.zadd(key, {device: score}) def score(self, client, service, monitor, device): """ diff --git a/Products/ZenModel/IpNetwork.py b/Products/ZenModel/IpNetwork.py index 4f77e575cb..802d9153a7 100644 --- a/Products/ZenModel/IpNetwork.py +++ b/Products/ZenModel/IpNetwork.py @@ -933,7 +933,8 @@ class AutoDiscoveryJob(SubprocessJob): specifying IP ranges, not both. Also accepts a set of zProperties to be set on devices that are discovered. """ - def _run(self, nets=(), ranges=(), zProperties=(), collector='localhost'): + name = 'AutoDiscoveryJob' + def _run(self, nets=(), ranges=(), zProperties={}, collector='localhost'): # Store the nets and ranges self.nets = nets self.ranges = ranges @@ -967,6 +968,9 @@ def _run(self, nets=(), ranges=(), zProperties=(), collector='localhost'): SubprocessJob._run(self, cmd) +from Products.Jobber.zenjobs import app +app.register_task(AutoDiscoveryJob) + class IpNetworkPrinter(object): def __init__(self, out): diff --git a/Products/ZenModel/ZDeviceLoader.py b/Products/ZenModel/ZDeviceLoader.py index 176f0682cf..04092d756b 100644 --- a/Products/ZenModel/ZDeviceLoader.py +++ b/Products/ZenModel/ZDeviceLoader.py @@ -182,7 +182,7 @@ class CreateDeviceJob(Job): """ Create a new device object. """ - + name = 'CreateDeviceJob' # Declare DeviceExistsError as an expected exception so that a traceback # is not written to zenjobs' log. throws = Job.throws + (DeviceExistsError,) @@ -275,6 +275,8 @@ def setCustomProperty(self, dev, cProperty, value): return dev.setZenProperty(cProperty, value) +from Products.Jobber.zenjobs import app +app.register_task(CreateDeviceJob) # alias the DeviceCreationJob so zenpacks don't break DeviceCreationJob = CreateDeviceJob From 33a1002857c60a8c5b64028d07e0c7a7493e294d Mon Sep 17 00:00:00 2001 From: azam Date: Wed, 10 Apr 2024 12:29:45 +0300 Subject: [PATCH 114/147] ZEN-34783: zentrap gets no config with new configcache system --- Products/ZenEvents/TrapFilter.py | 10 +++++ Products/ZenEvents/zentrap.py | 45 ++++++++++++++++++++++ Products/ZenHub/services/SnmpTrapConfig.py | 8 ++++ 3 files changed, 63 insertions(+) diff --git a/Products/ZenEvents/TrapFilter.py b/Products/ZenEvents/TrapFilter.py index 2ca404cbac..6b4580ded7 100644 --- a/Products/ZenEvents/TrapFilter.py +++ b/Products/ZenEvents/TrapFilter.py @@ -131,6 +131,11 @@ def __init__(self): self._genericTraps = frozenset([0, 1, 2, 3, 4, 5]) + self._initialized = False + self.prevFiltersConf = None + self._resetFilters() + + def _resetFilters(self): # Map of SNMP V1 Generic Trap filters where key is the generic trap number and # value is a GenericTrapFilterDefinition self._v1Traps = dict() @@ -355,6 +360,11 @@ def initialize(self): self._readFilters() self._initialized = True + def updateFilter(self, trapFilters): + if trapFilters != None and trapFilters != self.prevFiltersConf: + self._readFilters(trapFilters) + self.prevFiltersConf = trapFilters + def transform(self, event): """ Performs any transforms of the specified event at the collector. diff --git a/Products/ZenEvents/zentrap.py b/Products/ZenEvents/zentrap.py index 02dc51bb81..875044e1d2 100644 --- a/Products/ZenEvents/zentrap.py +++ b/Products/ZenEvents/zentrap.py @@ -50,6 +50,7 @@ from Products.ZenUtils.captureReplay import CaptureReplay from Products.ZenUtils.observable import ObservableMixin from Products.ZenUtils.Utils import unused +from Products.ZenHub.PBDaemon import HubDown unused(DeviceProxy, User) @@ -126,10 +127,13 @@ def __init__(self): self.configCycleInterval = 20 * 60 self.task = None + self.dynamicConfTask = None def postStartupTasks(self): self.task = TrapTask('zentrap', configId='zentrap') yield self.task + self.dynamicConfTask = DynamicConfigLoader(taskName="zentrapDynamicConf", configId="zentrap") + yield self.dynamicConfTask def buildOptions(self, parser): """ @@ -265,6 +269,47 @@ def __call__(self, varbinds): return {name: ','.join(vals) for name, vals in result.iteritems()} +@implementer(IScheduledTask) +class DynamicConfigLoader(BaseTask): + """Handles retrieving additional dynamic configs for daemon from ZODB""" + + def __init__(self, taskName, configId, scheduleIntervalSeconds=5*60, taskConfig=None): + BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds, taskConfig) + self.log = log + # Needed for interface + self.name = taskName + self.configId = configId + self.state = TaskStates.STATE_IDLE + self.interval = scheduleIntervalSeconds + self._daemon = getUtility(ICollector) + self._preferences = self._daemon + + @defer.inlineCallbacks + def doTask(self): + """ + Contact zenhub and gather configuration data. + """ + log.debug("%s gathering dynamic config changes", self.name) + try: + remoteProxy = self._daemon.getRemoteConfigServiceProxy() + trapFilters = yield remoteProxy.callRemote('getTrapFilters') + self._daemon._trapFilter._resetFilters() + self._daemon._trapFilter.updateFilter(trapFilters) + log.debug("%s trap filters changes applied to %s", trapFilters, self.name) + oidMap = yield remoteProxy.callRemote('getOidMap') + self._daemon.oidMap = oidMap + log.debug("Oid map changes applied to %s", self.name) + except Exception as ex: + log.exception("task '%s' failed", self.name) + + if isinstance(ex, HubDown): + # Allow the loader to be reaped and re-added + self.state = TaskStates.STATE_COMPLETED + + def cleanup(self): + pass # Required by interface + + @implementer(IScheduledTask) class TrapTask(BaseTask, CaptureReplay): """ diff --git a/Products/ZenHub/services/SnmpTrapConfig.py b/Products/ZenHub/services/SnmpTrapConfig.py index ac4518ed90..c40432c467 100644 --- a/Products/ZenHub/services/SnmpTrapConfig.py +++ b/Products/ZenHub/services/SnmpTrapConfig.py @@ -125,6 +125,14 @@ def remote_createAllUsers(self): log.debug("SnmpTrapConfig.remote_createAllUsers %s users", len(users)) return users + def remote_getTrapFilters(self): + return self.zem.trapFilters + + def remote_getOidMap(self): + return dict( + (b.oid, b.id) for b in self.dmd.Mibs.mibSearch() if b.oid + ) + def _objectUpdated(self, object): user = self._create_user(object) if user: From 7c1ec611a7e345932e5c19d80d88408fb213c96b Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:38:12 +0300 Subject: [PATCH 115/147] Fixed zenjobs scheduler error while starting up (#4402) Fixes ZEN-34798. --- Products/Jobber/scheduler.py | 3 +-- Products/Jobber/storage.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Products/Jobber/scheduler.py b/Products/Jobber/scheduler.py index 090a2ee1f3..e265c8b33b 100644 --- a/Products/Jobber/scheduler.py +++ b/Products/Jobber/scheduler.py @@ -223,11 +223,10 @@ def _key(name): def _getClient(): """Create and return the ZenJobs JobStore client.""" return redis.StrictRedis.from_url( - CeleryConfig.from_config().result_backend + CeleryConfig.from_config(getConfig()).result_backend ) - def handle_beat_init(*args, **kw): beat_logger = logging.getLogger("celery.beat") zenjobs_logger = logging.getLogger("zen.zenjobs") diff --git a/Products/Jobber/storage.py b/Products/Jobber/storage.py index dde23dcf25..4b438d0655 100644 --- a/Products/Jobber/storage.py +++ b/Products/Jobber/storage.py @@ -19,7 +19,7 @@ from Products.ZenUtils.RedisUtils import getRedisClient -from .config import CeleryConfig +from .config import getConfig, CeleryConfig _keybase = "zenjobs:job:" _keypattern = _keybase + "*" @@ -31,7 +31,7 @@ def makeJobStore(): """Create and return the ZenJobs JobStore client.""" - cfg = CeleryConfig.from_config() + cfg = CeleryConfig.from_config(getConfig()) client = getRedisClient(url=cfg.result_backend) return JobStore(client, expires=cfg.result_expires) From 0bb5ebbcea6e32fff4c9b4ef54b8e266d32b1559 Mon Sep 17 00:00:00 2001 From: azam Date: Thu, 11 Apr 2024 16:53:32 +0300 Subject: [PATCH 116/147] - fixed issue with oidMap updates - added checksum for tracking changes --- Products/ZenEvents/TrapFilter.py | 4 +-- Products/ZenEvents/zentrap.py | 32 +++++++++++++++------- Products/ZenHub/services/SnmpTrapConfig.py | 15 ++++++---- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/Products/ZenEvents/TrapFilter.py b/Products/ZenEvents/TrapFilter.py index 6b4580ded7..bc8190908c 100644 --- a/Products/ZenEvents/TrapFilter.py +++ b/Products/ZenEvents/TrapFilter.py @@ -132,7 +132,6 @@ def __init__(self): self._genericTraps = frozenset([0, 1, 2, 3, 4, 5]) self._initialized = False - self.prevFiltersConf = None self._resetFilters() def _resetFilters(self): @@ -361,9 +360,8 @@ def initialize(self): self._initialized = True def updateFilter(self, trapFilters): - if trapFilters != None and trapFilters != self.prevFiltersConf: + if trapFilters != None: self._readFilters(trapFilters) - self.prevFiltersConf = trapFilters def transform(self, event): """ diff --git a/Products/ZenEvents/zentrap.py b/Products/ZenEvents/zentrap.py index 875044e1d2..0a686c7249 100644 --- a/Products/ZenEvents/zentrap.py +++ b/Products/ZenEvents/zentrap.py @@ -130,10 +130,11 @@ def __init__(self): self.dynamicConfTask = None def postStartupTasks(self): - self.task = TrapTask('zentrap', configId='zentrap') - yield self.task self.dynamicConfTask = DynamicConfigLoader(taskName="zentrapDynamicConf", configId="zentrap") + self.task = TrapTask('zentrap', configId='zentrap') + self.dynamicConfTask.attachAttributeObserver("oidMap", self.task.oidMapChangeListener) yield self.dynamicConfTask + yield self.task def buildOptions(self, parser): """ @@ -273,7 +274,7 @@ def __call__(self, varbinds): class DynamicConfigLoader(BaseTask): """Handles retrieving additional dynamic configs for daemon from ZODB""" - def __init__(self, taskName, configId, scheduleIntervalSeconds=5*60, taskConfig=None): + def __init__(self, taskName, configId, scheduleIntervalSeconds=1*60, taskConfig=None): BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds, taskConfig) self.log = log # Needed for interface @@ -282,6 +283,9 @@ def __init__(self, taskName, configId, scheduleIntervalSeconds=5*60, taskConfig= self.state = TaskStates.STATE_IDLE self.interval = scheduleIntervalSeconds self._daemon = getUtility(ICollector) + self.oidMap = {} + self.trapFilterCheckSum = None + self.oidMapCheckSum = None self._preferences = self._daemon @defer.inlineCallbacks @@ -292,13 +296,17 @@ def doTask(self): log.debug("%s gathering dynamic config changes", self.name) try: remoteProxy = self._daemon.getRemoteConfigServiceProxy() - trapFilters = yield remoteProxy.callRemote('getTrapFilters') - self._daemon._trapFilter._resetFilters() - self._daemon._trapFilter.updateFilter(trapFilters) - log.debug("%s trap filters changes applied to %s", trapFilters, self.name) - oidMap = yield remoteProxy.callRemote('getOidMap') - self._daemon.oidMap = oidMap - log.debug("Oid map changes applied to %s", self.name) + checkSum, trapFilters = yield remoteProxy.callRemote('getTrapFilters', self.trapFilterCheckSum) + if checkSum and trapFilters: + self.trapFilterCheckSum = checkSum + self._daemon._trapFilter._resetFilters() + self._daemon._trapFilter.updateFilter(trapFilters) + log.debug("%s new trap filters changes applied to %s", trapFilters, self.name) + checkSum, oidMap = yield remoteProxy.callRemote('getOidMap', self.oidMapCheckSum) + if checkSum and oidMap: + self.oidMapCheckSum = checkSum + self.oidMap = oidMap + log.debug("New oid map changes applied to %s", self.name) except Exception as ex: log.exception("task '%s' failed", self.name) @@ -612,6 +620,10 @@ def receiveTrap(self, pdu): stat = self._statService.getStatistic("events") stat.value = totalEvents + def oidMapChangeListener(self, observable, attrName, oldValue, newValue): + self.log.debug("Task %s changed %s. Updating it for task %s",observable.name, attrName, self.name) + self.oidMap = newValue + def getPacketIp(self, addr): """ For IPv4, convert a pointer to 4 bytes to a dotted-ip-address diff --git a/Products/ZenHub/services/SnmpTrapConfig.py b/Products/ZenHub/services/SnmpTrapConfig.py index c40432c467..e2c30aa30e 100644 --- a/Products/ZenHub/services/SnmpTrapConfig.py +++ b/Products/ZenHub/services/SnmpTrapConfig.py @@ -15,6 +15,8 @@ from __future__ import print_function import logging +import json +from hashlib import md5 from twisted.spread import pb @@ -125,13 +127,14 @@ def remote_createAllUsers(self): log.debug("SnmpTrapConfig.remote_createAllUsers %s users", len(users)) return users - def remote_getTrapFilters(self): - return self.zem.trapFilters + def remote_getTrapFilters(self, remoteCheckSum): + currentCheckSum = md5(self.zem.trapFilters).hexdigest() + return (None, None) if currentCheckSum == remoteCheckSum else (currentCheckSum, self.zem.trapFilters) - def remote_getOidMap(self): - return dict( - (b.oid, b.id) for b in self.dmd.Mibs.mibSearch() if b.oid - ) + def remote_getOidMap(self, remoteCheckSum): + oidMap = dict((b.oid, b.id) for b in self.dmd.Mibs.mibSearch() if b.oid) + currentCheckSum = md5(json.dumps(oidMap, sort_keys=True).encode('utf-8')).hexdigest() + return (None, None) if currentCheckSum == remoteCheckSum else (currentCheckSum, oidMap) def _objectUpdated(self, object): user = self._create_user(object) From 617fb2dda796ba0718c3b19d6a72a3473b7e91bb Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 15 Apr 2024 15:30:08 -0500 Subject: [PATCH 117/147] fix: clear cache statuses when no config is built. ZEN-34781 --- .../ZenCollector/configcache/cache/storage.py | 11 +++ .../ZenCollector/configcache/invalidator.py | 7 +- Products/ZenCollector/configcache/task.py | 52 ++++++------- .../configcache/tests/test_task.py | 74 +++++++++++++++++++ 4 files changed, 113 insertions(+), 31 deletions(-) create mode 100644 Products/ZenCollector/configcache/tests/test_task.py diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 23bd035148..3ce70e32c5 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -124,6 +124,17 @@ def __init__(self, client): }, )() + def __contains__(self, key): + """ + Returns True if a config for the given key exists. + + @type key: CacheKey + @rtype: Boolean + """ + return self.__config.exists( + self.__client, key.service, key.monitor, key.device + ) + def search(self, query=CacheQuery()): """ Returns the configuration keys matching the search criteria. diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index 1fc776ec59..f136b8562b 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -204,8 +204,7 @@ def _new_device(self, device, monitor, buildlimit): ) return now = time.time() - for key in keys: - self.store.set_pending((key, now)) + self.store.set_pending(*((key, now) for key in keys)) self.dispatcher.dispatch_all(monitor, device.id, buildlimit, now) self.log.info( "submitted build jobs for new device device=%s collector=%s", @@ -214,14 +213,12 @@ def _new_device(self, device, monitor, buildlimit): ) def _changed_device_class(self, device, monitor, buildlimit): - # Don't dispatch jobs if there're any statuses. keys = tuple( CacheKey(svcname, monitor, device.id) for svcname in self.dispatcher.service_names ) now = time.time() - for key in keys: - self.store.set_pending((key, now)) + self.store.set_pending(*((key, now) for key in keys)) self.dispatcher.dispatch_all(monitor, device.id, buildlimit, now) self.log.info( "submitted build jobs for device with new device class " diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index 1402207348..bdc327d49a 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -20,7 +20,7 @@ from Products.Jobber.task import requires, DMD, Abortable from Products.Jobber.zenjobs import app -from .cache import CacheKey, CacheQuery, CacheRecord, ConfigStatus +from .cache import CacheKey, CacheRecord, ConfigStatus from .constants import Constants from .propertymap import DevicePropertyMap @@ -50,6 +50,14 @@ def build_device_config( @param submitted: timestamp of when the job was submitted @type submitted: float """ + buildDeviceConfig( + self.dmd, self.log, monitorname, deviceid, configclassname, submitted + ) + + +def buildDeviceConfig( + dmd, log, monitorname, deviceid, configclassname, submitted +): svcconfigclass = resolve(configclassname) svcname = configclassname.rsplit(".", 1)[0] store = _getStore() @@ -58,12 +66,12 @@ def build_device_config( # Check whether this is an old job, i.e. job pending timeout. # If it is an old job, skip it, manager already sent another one. status = next(store.get_status(key), None) - if _job_is_old(status, submitted, self.dmd.Devices, self.log): + if _job_is_old(status, submitted, dmd.Devices, log): return # If the status is Expired, another job is coming, so skip this job. if isinstance(status, ConfigStatus.Expired): - self.log.warn( + log.warn( "skipped this job because another job is coming " "device=%s collector=%s service=%s submitted=%f", key.device, @@ -79,7 +87,7 @@ def build_device_config( s1 = int(submitted * 1000) s2 = int(status.submitted * 1000) if s1 != s2: - self.log.warn( + log.warn( "skipped this job in favor of newer job " "device=%s collector=%s service=%s submitted=%f", key.device, @@ -92,20 +100,20 @@ def build_device_config( # Change the configuration's status to 'building' to indicate that # a config is now building. store.set_building((key, time())) - self.log.info( + log.info( "building device configuration device=%s collector=%s service=%s", deviceid, monitorname, svcname, ) - service = svcconfigclass(self.dmd, monitorname) + service = svcconfigclass(dmd, monitorname) result = service.remote_getDeviceConfigs((deviceid,)) config = result[0] if result else None if config is None: - _delete_config(key, store, self.log) + _delete_config(key, store, log) else: - uid = self.dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() + uid = dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() record = CacheRecord.make( svcname, monitorname, deviceid, uid, time(), config ) @@ -115,7 +123,7 @@ def build_device_config( # status is not ConfigStatus.Building, so another job will be # submitted or has already been submitted. store.put_config(record) - self.log.info( + log.info( "saved config without changing status " "updated=%s device=%s collector=%s service=%s", datetime.fromtimestamp(record.updated).isoformat(), @@ -126,7 +134,7 @@ def build_device_config( else: verb = "replaced" if status is not None else "added" store.add(record) - self.log.info( + log.info( "%s config updated=%s device=%s collector=%s service=%s", verb, datetime.fromtimestamp(record.updated).isoformat(), @@ -143,29 +151,21 @@ def _delete_config(key, store, log): key.monitor, key.service, ) - oldkey = next( - store.search( - CacheQuery( - service=key.service, monitor=key.monitor, device=key.device - ) - ), - None, - ) - if oldkey is not None: - store.remove(oldkey) + if key in store: + store.remove(key) log.info( "removed previously built configuration " "device=%s collector=%s service=%s", - oldkey.device, - oldkey.monitor, - oldkey.service, + key.device, + key.monitor, + key.service, ) - # Ensure any status on this key is removed - store.clear_status(oldkey) + # Ensure all statuses for this key are deleted. + store.clear_status(key) def _job_is_old(status, submitted, ctx, log): - if (submitted is None or status is None): + if submitted is None or status is None: # job is not old (default state) return False pendinglimitmap = DevicePropertyMap.make_pending_timeout_map(ctx) diff --git a/Products/ZenCollector/configcache/tests/test_task.py b/Products/ZenCollector/configcache/tests/test_task.py new file mode 100644 index 0000000000..1e019edeae --- /dev/null +++ b/Products/ZenCollector/configcache/tests/test_task.py @@ -0,0 +1,74 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import mock + +from unittest import TestCase + +from Products.Jobber.tests.utils import RedisLayer + +from ..cache import CacheKey +from ..cache.storage import ConfigStore +from ..task import buildDeviceConfig + +PATH = { + "zenjobs": "Products.Jobber.zenjobs", + "task": "Products.ZenCollector.configcache.task", +} + + +class TestBuildDeviceConfig(TestCase): + + layer = RedisLayer + + def setUp(t): + t.device_name = "qadevice" + t.device_uid = "/zport/dmd/Devices/Server/Linux/devices/qadevice" + t.store = ConfigStore(t.layer.redis) + + def tearDown(t): + del t.store + + @mock.patch("{task}.DevicePropertyMap".format(**PATH), autospec=True) + @mock.patch("{task}.time".format(**PATH), autospec=True) + @mock.patch("{task}.createObject".format(**PATH), autospec=True) + @mock.patch("{task}.resolve".format(**PATH), autospec=True) + def test_no_config_built( + t, _resolve, _createObject, _time, _DevicePropertyMap + ): + monitor = "localhost" + clsname = "Products.ZenHub.services.PingService.PingService" + svcname = clsname.rsplit(".", 1)[0] + submitted = 123456.34 + svcclass = mock.Mock() + svc = mock.MagicMock() + dmd = mock.Mock() + log = mock.Mock() + dvc = mock.Mock() + key = CacheKey(svcname, monitor, t.device_name) + + _createObject.return_value = t.store + _resolve.return_value = svcclass + svcclass.return_value = svc + svc.remote_getDeviceConfigs.return_value = [] + dmd.Devices.findDeviceByIdExact.return_value = dvc + dvc.getPrimaryId.return_value = t.device_uid + _time.return_value = submitted + 10 + limitmap = mock.Mock() + _DevicePropertyMap.make_pending_timeout_map.return_value = limitmap + limitmap.get.return_value = 1000 + + t.store.set_pending((key, submitted)) + + buildDeviceConfig(dmd, log, monitor, t.device_name, clsname, submitted) + + status = next(t.store.get_status(key), None) + t.assertIsNone(status) From cda6636b9b02dcbe2375980c0deea8ee9892aab9 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 16 Apr 2024 16:07:36 -0500 Subject: [PATCH 118/147] Use ast.literal_eval instead of json.loads. json.loads makes a transformation that unjelly doesn't like. ZEN-34797 --- Products/ZenCollector/configcache/cache/storage.py | 2 +- Products/ZenCollector/configcache/tests/test_storage.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 3ce70e32c5..d26b4fb1de 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -522,7 +522,7 @@ def _range(client, metadata, svc, mon, minv=None, maxv=None): def _unjelly(data): - return unjelly(json.loads(data)) + return unjelly(ast.literal_eval(data)) def _to_score(ts): diff --git a/Products/ZenCollector/configcache/tests/test_storage.py b/Products/ZenCollector/configcache/tests/test_storage.py index 9acbab3e58..d9282c2fbd 100644 --- a/Products/ZenCollector/configcache/tests/test_storage.py +++ b/Products/ZenCollector/configcache/tests/test_storage.py @@ -1217,6 +1217,7 @@ def _make_config(_id, configId, guid): config.id = _id config._config_id = configId config._device_guid = guid + config.data = u"fancy" return config From f382c86a306c4b803451e14f416aaf1511b185ca Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 17 Apr 2024 09:32:20 -0500 Subject: [PATCH 119/147] fix: remove trailing commas from what should be statements ZEN-34781 --- Products/ZenCollector/configcache/cache/storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index d26b4fb1de..456b42f1a7 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -503,11 +503,11 @@ def get_newer(self, mintimestamp, service="*", monitor="*"): def _get_scores(self, key): service, monitor, device = attr.astuple(key) with self.__client.pipeline() as pipe: - self.__age.score(pipe, service, monitor, device), - self.__retired.score(pipe, service, monitor, device), - self.__expired.score(pipe, service, monitor, device), - self.__pending.score(pipe, service, monitor, device), - self.__building.score(pipe, service, monitor, device), + self.__age.score(pipe, service, monitor, device) + self.__retired.score(pipe, service, monitor, device) + self.__expired.score(pipe, service, monitor, device) + self.__pending.score(pipe, service, monitor, device) + self.__building.score(pipe, service, monitor, device) return pipe.execute() From 2187c0bcfebc66ee045ed15c4c3982850e5a7bac Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 22 Apr 2024 13:53:34 -0500 Subject: [PATCH 120/147] Add error handling to help identify issue. ZEN-34816 --- .../configcache/modelchange/filters.py | 11 ++++++++--- Products/ZenRelations/ToOneRelationship.py | 16 ++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Products/ZenCollector/configcache/modelchange/filters.py b/Products/ZenCollector/configcache/modelchange/filters.py index e752c4e345..99851fb0c4 100644 --- a/Products/ZenCollector/configcache/modelchange/filters.py +++ b/Products/ZenCollector/configcache/modelchange/filters.py @@ -181,9 +181,14 @@ def generateChecksum(self, organizer, md5_checksum): for tpl in organizer.rrdTemplates(): s.seek(0) s.truncate() - # TODO: exportXml is a bit of a hack. Sorted, etc. would be better. - tpl.exportXml(s) - md5_checksum.update(s.getvalue()) + try: + tpl.exportXml(s) + except Exception: + log.exception( + "unable to export XML of template template=%r", tpl + ) + else: + md5_checksum.update(s.getvalue()) # Include z/c properties from base class super(DeviceClassInvalidationFilter, self).generateChecksum( organizer, md5_checksum diff --git a/Products/ZenRelations/ToOneRelationship.py b/Products/ZenRelations/ToOneRelationship.py index 4e4a16588f..678b3895fd 100644 --- a/Products/ZenRelations/ToOneRelationship.py +++ b/Products/ZenRelations/ToOneRelationship.py @@ -200,10 +200,18 @@ def exportXml(self, ofile, ignorerels=[]): if not self.obj or self.remoteType() == ToManyCont: return - ofile.write( - "\n" - % (self.id, self.obj.getPrimaryId()) - ) + try: + ofile.write( + "\n" + % (self.id, self.obj.getPrimaryId()) + ) + except Exception: + log.exception( + "skipping %s object-type=%s object-id=%s", + self.id, + self.obj.__class__.__module__, + getattr(self.obj, "id", ""), + ) def checkRelation(self, repair=False): """Check to make sure that relationship bidirectionality is ok.""" From 54d942347684ee49b980d1aef58571784ca8256f Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 24 Apr 2024 11:09:02 -0500 Subject: [PATCH 121/147] Refactored invalidator to reduce redundant code. ZEN-34712 --- .../ZenCollector/configcache/dispatcher.py | 71 +++ .../ZenCollector/configcache/invalidator.py | 412 ++++++++++-------- Products/ZenCollector/configcache/manager.py | 2 +- Products/ZenCollector/configcache/task.py | 59 --- 4 files changed, 301 insertions(+), 243 deletions(-) create mode 100644 Products/ZenCollector/configcache/dispatcher.py diff --git a/Products/ZenCollector/configcache/dispatcher.py b/Products/ZenCollector/configcache/dispatcher.py new file mode 100644 index 0000000000..59940ad49b --- /dev/null +++ b/Products/ZenCollector/configcache/dispatcher.py @@ -0,0 +1,71 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024 all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import + +from .task import build_device_config + + +class BuildConfigTaskDispatcher(object): + """Encapsulates the act of dispatching the build_device_config task.""" + + def __init__(self, configClasses): + """ + Initialize a BuildConfigTaskDispatcher instance. + + The `configClasses` parameter should be the classes used to create + the device configurations. + + @type configClasses: Sequence[Class] + """ + self._classnames = { + cls.__module__: ".".join((cls.__module__, cls.__name__)) + for cls in configClasses + } + + @property + def service_names(self): + return self._classnames.keys() + + def dispatch_all(self, monitorid, deviceid, timeout, submitted): + """ + Submit a task to build a device configuration from each + configuration service. + """ + soft_limit, hard_limit = _get_limits(timeout) + for name in self._classnames.values(): + build_device_config.apply_async( + args=(monitorid, deviceid, name), + kwargs={"submitted": submitted}, + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) + + def dispatch(self, servicename, monitorid, deviceid, timeout, submitted): + """ + Submit a task to build device configurations for the specified device. + + @type servicename: str + @type monitorid: str + @type deviceId: str + """ + name = self._classnames.get(servicename) + if name is None: + raise ValueError("service name '%s' not found" % servicename) + soft_limit, hard_limit = _get_limits(timeout) + build_device_config.apply_async( + args=(monitorid, deviceid, name), + kwargs={"submitted": submitted}, + soft_time_limit=soft_limit, + time_limit=hard_limit, + ) + + +def _get_limits(timeout): + return timeout, (timeout + (timeout * 0.1)) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index f136b8562b..ebf0aa0013 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -9,11 +9,11 @@ from __future__ import print_function, absolute_import -import gc import logging import time -from Products.AdvancedQuery import And, Eq +from multiprocessing import Process + from zenoss.modelindex import constants from zope.component import createObject @@ -26,10 +26,14 @@ from .app.args import get_subparser from .cache import CacheKey, CacheQuery, ConfigStatus from .debug import Debug as DebugCommand +from .dispatcher import BuildConfigTaskDispatcher from .modelchange import InvalidationCause -from .propertymap import DevicePropertyMap -from .task import BuildConfigTaskDispatcher -from .utils import getConfigServices, RelStorageInvalidationPoller +from .utils import ( + get_build_timeout, + get_minimum_ttl, + getConfigServices, + RelStorageInvalidationPoller, +) _default_interval = 30.0 @@ -92,9 +96,14 @@ def __init__(self, config, context): client = getRedisClient(url=getRedisUrl()) self.store = createObject("configcache-store", client) + self._process = _InvalidationProcessor( + self.log, self.store, self.dispatcher + ) + self.interval = config["poll-interval"] def run(self): + # Handle changes that occurred when Invalidator wasn't running. self._synchronize() poller = RelStorageInvalidationPoller( @@ -104,56 +113,130 @@ def run(self): "polling for device changes every %s seconds", self.interval ) while not self.ctx.controller.shutdown: - self.ctx.session.sync() - gc.collect() - result = poller.poll() - if result: - self.log.debug("found %d relevant invalidations", len(result)) - self._process_all(result) - self.ctx.controller.wait(self.interval) + try: + self.ctx.session.sync() + invalidations = poller.poll() + if not invalidations: + continue + self._process_invalidations(invalidations) + finally: + # Call cacheGC to aggressively trim the ZODB cache + self.ctx.session.cacheGC() + self.ctx.controller.wait(self.interval) def _synchronize(self): - tool = IModelCatalogTool(self.ctx.dmd) - # TODO: if device changed monitors, the config should be same (?) - # so just rekey the config? - count = _removeDeleted(self.log, tool, self.store) - if count == 0: - self.log.info("no dangling configurations found") - timelimitmap = DevicePropertyMap.make_build_timeout_map( - self.ctx.dmd.Devices - ) - new_devices, changed_devices = _addNewOrChangedDevices( - self.log, tool, timelimitmap, self.store, self.dispatcher - ) - if len(new_devices) == 0: - self.log.info("no missing configurations found") - if len(changed_devices) == 0: - self.log.info("no devices with a different device class found") - - def _process_all(self, invalidations): - buildlimit_map = DevicePropertyMap.make_build_timeout_map( - self.ctx.dmd.Devices - ) - minttl_map = DevicePropertyMap.make_minimum_ttl_map( - self.ctx.dmd.Devices + sync_process = Process( + target=_synchronize_cache, + args=(self.log, self.ctx.dmd, self.dispatcher), ) - for invalidation in invalidations: - uid = invalidation.device.getPrimaryId() - buildlimit = buildlimit_map.get(uid) - minttl = minttl_map.get(uid) + sync_process.start() + sync_process.join() # blocks until subprocess has exited + + def _process_invalidations(self, invalidations): + self.log.debug("found %d relevant invalidations", len(invalidations)) + for inv in invalidations: try: - self._process(invalidation, uid, buildlimit, minttl) - except AttributeError: - self.log.info( - "invalidation device=%s reason=%s", - invalidation.device, - invalidation.reason, + self._process(inv.device, inv.oid, inv.reason) + except Exception: + self.log.exception( + "failed to process invalidation device=%s", + inv.device, ) - self.log.exception("failed while processing invalidation") - def _process(self, invalidation, uid, buildlimit, minttl): - device = invalidation.device - reason = invalidation.reason + +_solr_fields = ("id", "collector", "uid") + + +def _synchronize_cache(log, dmd, dispatcher): + store = createObject( + "configcache-store", getRedisClient(url=getRedisUrl()) + ) + tool = IModelCatalogTool(dmd) + catalog_results = tool.cursor_search( + types=("Products.ZenModel.Device.Device",), + limit=constants.DEFAULT_SEARCH_LIMIT, + fields=_solr_fields, + ).results + devices = { + (brain.id, brain.collector): brain.uid + for brain in catalog_results + if brain.collector is not None + } + _removeDeleted(log, store, devices) + _addNewOrChangedDevices(log, store, dispatcher, dmd, devices) + + +def _removeDeleted(log, store, devices): + """ + Remove deleted devices from the cache. + + @param devices: devices that currently exist + @type devices: Mapping[Sequence[str, str], str] + """ + devices_not_found = tuple( + key + for key in store.search() + if (key.device, key.monitor) not in devices + ) + if devices_not_found: + _RemoveConfigsHandler(log, store)(devices_not_found) + else: + log.info("no dangling configurations found") + + +def _addNewOrChangedDevices(log, store, dispatcher, dmd, devices): + # Add new devices to the config and metadata store. + # Also look for device that have changed their device class. + # Query the catalog for all devices + new_devices = 0 + changed_devices = 0 + handle = _NewDeviceHandler(log, store, dispatcher) + for (deviceId, monitorId), uid in devices.iteritems(): + try: + device = dmd.unrestrictedTraverse(uid) + except Exception as ex: + log.warning( + "failed to get device error-type=%s error=%s uid=%s", + type(ex), + ex, + uid, + ) + continue + timeout = get_build_timeout(device) + keys_with_configs = tuple( + store.search(CacheQuery(monitor=monitorId, device=deviceId)) + ) + uid = device.getPrimaryId() + if not keys_with_configs: + handle(deviceId, monitorId, timeout) + new_devices += 1 + else: + current_uid = store.get_uid(deviceId) + # A device with a changed device class will have a different uid. + if current_uid != uid: + handle(deviceId, monitorId, timeout, False) + changed_devices += 1 + if new_devices == 0: + log.info("no missing configurations found") + if changed_devices == 0: + log.info("no devices with a different device class found") + + +class _InvalidationProcessor(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self._remove = _RemoveConfigsHandler(log, store) + self._update = _DeviceUpdateHandler(log, store, dispatcher) + self._missing = _MissingConfigsHandler(log, store, dispatcher) + self._new = _NewDeviceHandler(log, store, dispatcher) + + def __call__(self, device, oid, reason): + uid = device.getPrimaryId() + self.log.info("handling device %s", uid) + buildlimit = get_build_timeout(device) + minttl = get_minimum_ttl(device) monitor = device.getPerformanceServerName() if monitor is None: self.log.warn( @@ -163,20 +246,23 @@ def _process(self, invalidation, uid, buildlimit, minttl): reason, ) return - keys = tuple( + keys_with_config = tuple( self.store.search(CacheQuery(monitor=monitor, device=device.id)) ) - if not keys: - self._new_device(device, monitor, buildlimit) + if not keys_with_config: + self._new(device.id, monitor, buildlimit) elif reason is InvalidationCause.Updated: # Check for device class change stored_uid = self.store.get_uid(device.id) if uid != stored_uid: - self._changed_device_class(device, monitor, buildlimit) + self._new(device.id, monitor, buildlimit, False) else: - self._updated_device(device, monitor, keys, minttl, buildlimit) + self._update(keys_with_config, minttl) + self._missing( + device.id, monitor, keys_with_config, buildlimit + ) elif reason is InvalidationCause.Removed: - self._removed_device(keys) + self._remove(keys_with_config) else: self.log.warn( "ignored unexpected reason " @@ -184,66 +270,85 @@ def _process(self, invalidation, uid, buildlimit, minttl): reason, device, monitor, - invalidation.oid, + oid, ) - def _new_device(self, device, monitor, buildlimit): - # Don't dispatch jobs if there're any statuses. - keys = tuple( - CacheKey(svcname, monitor, device.id) - for svcname in self.dispatcher.service_names - ) - for key in keys: - status = next(self.store.get_status(key), None) - if status is not None: - self.log.debug( - "build jobs already submitted for new device " - "device=%s collector=%s", - device.id, - monitor, - ) - return - now = time.time() - self.store.set_pending(*((key, now) for key in keys)) - self.dispatcher.dispatch_all(monitor, device.id, buildlimit, now) - self.log.info( - "submitted build jobs for new device device=%s collector=%s", - device.id, - monitor, - ) - def _changed_device_class(self, device, monitor, buildlimit): +class _NewDeviceHandler(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self.dispatcher = dispatcher + + def __call__(self, deviceId, monitor, buildlimit, newDevice=True): keys = tuple( - CacheKey(svcname, monitor, device.id) + CacheKey(svcname, monitor, deviceId) for svcname in self.dispatcher.service_names ) + keys_with_pending_status = set( + status.key + for status in self.store.get_status(*keys) + if isinstance(status, ConfigStatus.Pending) + ) + for key in keys_with_pending_status: + self.log.debug( + "build job already submitted for this config " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) + keys_without_pending_status = set(keys) - keys_with_pending_status now = time.time() - self.store.set_pending(*((key, now) for key in keys)) - self.dispatcher.dispatch_all(monitor, device.id, buildlimit, now) - self.log.info( - "submitted build jobs for device with new device class " - "device=%s collector=%s", - device.id, - monitor, + self.store.set_pending( + *((key, now) for key in keys_without_pending_status) ) + for key in keys_without_pending_status: + self.dispatcher.dispatch( + key.service, key.monitor, key.device, buildlimit, now + ) + self.log.info( + "submitted build job for %s " + "device=%s collector=%s service=%s", + "new device" if newDevice else "device with new device class", + key.device, + key.monitor, + key.service, + ) + + +class _DeviceUpdateHandler(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self.dispatcher = dispatcher - def _updated_device(self, device, monitor, keys, minttl, buildlimit): - statuses = tuple( + def __call__(self, keys, minttl): + current_statuses = tuple( status for status in self.store.get_status(*keys) if isinstance(status, ConfigStatus.Current) ) + now = time.time() - limit = now - minttl + retirement = now - minttl + retired = set( - status.key for status in statuses if status.updated >= limit + status.key + for status in current_statuses + if status.updated >= retirement ) expired = set( - status.key for status in statuses if status.key not in retired + status.key + for status in current_statuses + if status.key not in retired ) - now = time.time() + self.store.set_retired(*((key, now) for key in retired)) self.store.set_expired(*((key, now) for key in expired)) + for key in retired: self.log.info( "retired configuration of changed device " @@ -261,13 +366,26 @@ def _updated_device(self, device, monitor, keys, minttl, buildlimit): key.service, ) + +class _MissingConfigsHandler(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self.dispatcher = dispatcher + + def __call__(self, deviceId, monitor, keys, buildlimit): + """ + @param keys: These keys are associated with a config + @type keys: Sequence[CacheKey] + """ # Send a job for for all config services that don't currently have # an associated configuration. Some ZenPacks, i.e. vSphere, defer # their modeling to a later time, so jobs for configuration services # must be sent to pick up any new configs. hasconfigs = tuple(key.service for key in keys) noconfigkeys = tuple( - CacheKey(svcname, monitor, device.id) + CacheKey(svcname, monitor, deviceId) for svcname in self.dispatcher.service_names if svcname not in hasconfigs ) @@ -281,12 +399,8 @@ def _updated_device(self, device, monitor, keys, minttl, buildlimit): self.dispatcher.dispatch( key.service, key.monitor, key.device, buildlimit, now ) - - def _removed_device(self, keys): - self.store.remove(*keys) - for key in keys: - self.log.info( - "removed configuration of deleted device " + self.log.debug( + "submitted build job for possibly missing config " "device=%s collector=%s service=%s", key.device, key.monitor, @@ -294,87 +408,19 @@ def _removed_device(self, keys): ) -_solr_fields = ("id", "collector", "uid") - - -def _deviceExistsInCatalog(tool, monitorId, deviceId): - query = And(Eq("id", deviceId), Eq("collector", monitorId)) - brain = next( - iter(tool.search_model_catalog(query, fields=_solr_fields)), None - ) - return brain is not None - - -def _removeDeleted(log, tool, store): - # Remove deleted devices from the config and metadata store. - devices_not_found = tuple( - key - for key in store.search() - if not _deviceExistsInCatalog(tool, key.monitor, key.device) - ) - store.remove(*devices_not_found) - for key in devices_not_found: - log.info( - "removed configuration for deleted device " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) - return len(devices_not_found) +class _RemoveConfigsHandler(object): + def __init__(self, log, store): + self.log = log + self.store = store -def _addNewOrChangedDevices(log, tool, timelimitmap, store, dispatcher): - # Add new devices to the config and metadata store. - # Also look for device that have changed their device class. - # Query the catalog for all devices - catalog_results = tool.cursor_search( - types=("Products.ZenModel.Device.Device",), - limit=constants.DEFAULT_SEARCH_LIMIT, - fields=_solr_fields, - ).results - new_devices = [] - changed_devices = [] - jobs_args = [] - for brain in catalog_results: - if brain.collector is None: - log.warn( - "ignoring device having undefined collector device=%s uid=%s", - brain.id, - brain.uid, - ) - continue - keys = tuple( - store.search(CacheQuery(monitor=brain.collector, device=brain.id)) - ) - if not keys: - timeout = timelimitmap.get(brain.uid) - jobs_args.append((brain, timeout)) - new_devices.append(brain.id) - else: - current_uid = store.get_uid(brain.id) - if current_uid != brain.uid: - timeout = timelimitmap.get(brain.uid) - jobs_args.append((brain, timeout)) - changed_devices.append(brain.id) - - now = time.time() - for brain, timeout in jobs_args: - keys = tuple( - CacheKey(svcname, brain.collector, brain.id) - for svcname in dispatcher.service_names - ) + def __call__(self, keys): + self.store.remove(*keys) for key in keys: - store.set_pending((key, now)) - dispatcher.dispatch_all(brain.collector, brain.id, timeout, now) - log.info( - "submitted build jobs for device %s " "uid=%s collector=%s", - ( - "without any configurations" - if brain.id in new_devices - else "with a new device class" - ), - brain.uid, - brain.collector, - ) - return (new_devices, changed_devices) + self.log.info( + "removed configuration of deleted device " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index ef10b96f23..3bc7c4baff 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -24,8 +24,8 @@ from .cache import ConfigStatus from .constants import Constants from .debug import Debug as DebugCommand +from .dispatcher import BuildConfigTaskDispatcher from .propertymap import DevicePropertyMap -from .task import BuildConfigTaskDispatcher from .utils import getConfigServices _default_interval = 30.0 # seconds diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index bdc327d49a..a478d8f276 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -186,65 +186,6 @@ def _job_is_old(status, submitted, ctx, log): return False -class BuildConfigTaskDispatcher(object): - """Encapsulates the act of dispatching the build_device_config task.""" - - def __init__(self, configClasses): - """ - Initialize a BuildConfigTaskDispatcher instance. - - The `configClasses` parameter should be the classes used to create - the device configurations. - - @type configClasses: Sequence[Class] - """ - self._classnames = { - cls.__module__: ".".join((cls.__module__, cls.__name__)) - for cls in configClasses - } - - @property - def service_names(self): - return self._classnames.keys() - - def dispatch_all(self, monitorid, deviceid, timeout, submitted): - """ - Submit a task to build a device configuration from each - configuration service. - """ - soft_limit, hard_limit = _get_limits(timeout) - for name in self._classnames.values(): - build_device_config.apply_async( - args=(monitorid, deviceid, name), - kwargs={"submitted": submitted}, - soft_time_limit=soft_limit, - time_limit=hard_limit, - ) - - def dispatch(self, servicename, monitorid, deviceid, timeout, submitted): - """ - Submit a task to build device configurations for the specified device. - - @type servicename: str - @type monitorid: str - @type deviceId: str - """ - name = self._classnames.get(servicename) - if name is None: - raise ValueError("service name '%s' not found" % servicename) - soft_limit, hard_limit = _get_limits(timeout) - build_device_config.apply_async( - args=(monitorid, deviceid, name), - kwargs={"submitted": submitted}, - soft_time_limit=soft_limit, - time_limit=hard_limit, - ) - - -def _get_limits(timeout): - return timeout, (timeout + (timeout * 0.1)) - - def _getStore(): client = getRedisClient(url=getRedisUrl()) return createObject("configcache-store", client) From 649e7862988c945cffcdeef38f21db1a14ebf862 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 23 Apr 2024 16:50:25 -0500 Subject: [PATCH 122/147] Small changes to hopefully reduce memory usage. ZEN-34712 --- Products/ZenCollector/configcache/manager.py | 5 ++- .../configcache/tests/test_dispatcher.py | 4 +- .../configcache/utils/__init__.py | 10 +++++ .../ZenCollector/configcache/utils/zprops.py | 43 +++++++++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 Products/ZenCollector/configcache/utils/zprops.py diff --git a/Products/ZenCollector/configcache/manager.py b/Products/ZenCollector/configcache/manager.py index 3bc7c4baff..9a4a71a107 100644 --- a/Products/ZenCollector/configcache/manager.py +++ b/Products/ZenCollector/configcache/manager.py @@ -9,7 +9,6 @@ from __future__ import print_function -import gc import logging from datetime import datetime @@ -88,7 +87,6 @@ def run(self): while not self.ctx.controller.shutdown: try: self.ctx.session.sync() - gc.collect() timedout = tuple(self._get_build_timeouts()) if not timedout: self.log.debug("no configuration builds have timed out") @@ -106,6 +104,9 @@ def run(self): self._rebuild_configs(statuses) except Exception as ex: self.log.exception("unexpected error %s", ex) + finally: + # Call cacheGC to aggressively trim the ZODB cache + self.ctx.session.cacheGC() self.ctx.controller.wait(self.interval) def _get_build_timeouts(self): diff --git a/Products/ZenCollector/configcache/tests/test_dispatcher.py b/Products/ZenCollector/configcache/tests/test_dispatcher.py index fa8c078fd0..7fe1145f29 100644 --- a/Products/ZenCollector/configcache/tests/test_dispatcher.py +++ b/Products/ZenCollector/configcache/tests/test_dispatcher.py @@ -13,10 +13,10 @@ from mock import call, patch -from ..task import BuildConfigTaskDispatcher, build_device_config +from ..dispatcher import BuildConfigTaskDispatcher, build_device_config -PATH = {"src": "Products.ZenCollector.configcache.task"} +PATH = {"src": "Products.ZenCollector.configcache.dispatcher"} class BuildConfigTaskDispatcherTest(TestCase): diff --git a/Products/ZenCollector/configcache/utils/__init__.py b/Products/ZenCollector/configcache/utils/__init__.py index b59560283b..4c7e5957ba 100644 --- a/Products/ZenCollector/configcache/utils/__init__.py +++ b/Products/ZenCollector/configcache/utils/__init__.py @@ -11,9 +11,19 @@ from .pollers import RelStorageInvalidationPoller from .services import getConfigServices +from .zprops import ( + get_ttl, + get_minimum_ttl, + get_pending_timeout, + get_build_timeout, +) __all__ = ( "getConfigServices", + "get_build_timeout", + "get_minimum_ttl", + "get_pending_timeout", + "get_ttl", "RelStorageInvalidationPoller", ) diff --git a/Products/ZenCollector/configcache/utils/zprops.py b/Products/ZenCollector/configcache/utils/zprops.py new file mode 100644 index 0000000000..936e4ea785 --- /dev/null +++ b/Products/ZenCollector/configcache/utils/zprops.py @@ -0,0 +1,43 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from ..constants import Constants + + +def get_ttl(device): + return _getZProperty( + device, Constants.time_to_live_id, Constants.time_to_live_value + ) + + +def get_minimum_ttl(device): + return _getZProperty( + device, + Constants.minimum_time_to_live_id, + Constants.minimum_time_to_live_value, + ) + + +def get_pending_timeout(device): + return _getZProperty( + device, Constants.pending_timeout_id, Constants.pending_timeout_value + ) + + +def get_build_timeout(device): + return _getZProperty( + device, Constants.build_timeout_id, Constants.build_timeout_value + ) + + +def _getZProperty(obj, propname, default): + value = obj.getZ(propname) + if value is None: + return default + return value From f419d4f1d428caa05b598a62718f91ec3e35adfd Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 25 Apr 2024 15:44:03 -0500 Subject: [PATCH 123/147] fix: replace DevicePropertyMap with a direct lookup. The DevicePropertyMap is replaced with a direct zproperty lookup, which requires fewer resources to run. ZEN-34835 --- Products/ZenCollector/configcache/task.py | 16 ++++++++-------- .../ZenCollector/configcache/tests/test_task.py | 9 ++------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index a478d8f276..a2f7de96cd 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -22,7 +22,7 @@ from .cache import CacheKey, CacheRecord, ConfigStatus from .constants import Constants -from .propertymap import DevicePropertyMap +from .utils import get_pending_timeout @app.task( @@ -66,7 +66,8 @@ def buildDeviceConfig( # Check whether this is an old job, i.e. job pending timeout. # If it is an old job, skip it, manager already sent another one. status = next(store.get_status(key), None) - if _job_is_old(status, submitted, dmd.Devices, log): + device = dmd.Devices.findDeviceByIdExact(deviceid) + if _job_is_old(status, submitted, device, log): return # If the status is Expired, another job is coming, so skip this job. @@ -113,7 +114,7 @@ def buildDeviceConfig( if config is None: _delete_config(key, store, log) else: - uid = dmd.Devices.findDeviceByIdExact(deviceid).getPrimaryId() + uid = device.getPrimaryId() record = CacheRecord.make( svcname, monitorname, deviceid, uid, time(), config ) @@ -164,14 +165,13 @@ def _delete_config(key, store, log): store.clear_status(key) -def _job_is_old(status, submitted, ctx, log): +def _job_is_old(status, submitted, device, log): if submitted is None or status is None: # job is not old (default state) return False - pendinglimitmap = DevicePropertyMap.make_pending_timeout_map(ctx) - duration = pendinglimitmap.get(status.uid) + limit = get_pending_timeout(device) now = time() - if submitted < (now - duration): + if submitted < (now - limit): log.warn( "skipped this job because it's too old " "device=%s collector=%s service=%s submitted=%f %s=%s", @@ -180,7 +180,7 @@ def _job_is_old(status, submitted, ctx, log): status.key.service, submitted, Constants.pending_timeout_id, - duration, + limit, ) return True return False diff --git a/Products/ZenCollector/configcache/tests/test_task.py b/Products/ZenCollector/configcache/tests/test_task.py index 1e019edeae..61a043ec40 100644 --- a/Products/ZenCollector/configcache/tests/test_task.py +++ b/Products/ZenCollector/configcache/tests/test_task.py @@ -37,13 +37,10 @@ def setUp(t): def tearDown(t): del t.store - @mock.patch("{task}.DevicePropertyMap".format(**PATH), autospec=True) @mock.patch("{task}.time".format(**PATH), autospec=True) @mock.patch("{task}.createObject".format(**PATH), autospec=True) @mock.patch("{task}.resolve".format(**PATH), autospec=True) - def test_no_config_built( - t, _resolve, _createObject, _time, _DevicePropertyMap - ): + def test_no_config_built(t, _resolve, _createObject, _time): monitor = "localhost" clsname = "Products.ZenHub.services.PingService.PingService" svcname = clsname.rsplit(".", 1)[0] @@ -62,9 +59,7 @@ def test_no_config_built( dmd.Devices.findDeviceByIdExact.return_value = dvc dvc.getPrimaryId.return_value = t.device_uid _time.return_value = submitted + 10 - limitmap = mock.Mock() - _DevicePropertyMap.make_pending_timeout_map.return_value = limitmap - limitmap.get.return_value = 1000 + dvc.getZ.return_value = 1000 t.store.set_pending((key, submitted)) From 191bf55ae77074c20706d7c917bd4fe399d5860a Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 26 Apr 2024 07:48:07 -0500 Subject: [PATCH 124/147] fix: restore setting thresEventData to None by default. Setting a function argument's default value to a container is bad practice for Python programming. ZEN-34655 --- Products/ZenCollector/daemon.py | 6 +++--- Products/ZenCollector/interfaces.py | 8 ++++---- Products/ZenUtils/metricwriter.py | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 1933738230..0f61023a81 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -450,7 +450,7 @@ def writeMetric( timestamp="N", min="U", max="U", - threshEventData={}, + threshEventData=None, deviceId=None, contextUUID=None, deviceUUID=None, @@ -526,7 +526,7 @@ def writeMetricWithMetadata( timestamp="N", min="U", max="U", - threshEventData={}, + threshEventData=None, metadata=None, ): metadata = metadata or {} @@ -567,7 +567,7 @@ def writeRRD( cycleTime=None, min="U", max="U", - threshEventData={}, + threshEventData=None, timestamp="N", allowStaleDatapoint=True, ): diff --git a/Products/ZenCollector/interfaces.py b/Products/ZenCollector/interfaces.py index d85e61c89b..3adceb2132 100644 --- a/Products/ZenCollector/interfaces.py +++ b/Products/ZenCollector/interfaces.py @@ -423,7 +423,7 @@ def writeMetric( timestamp="N", min="U", max="U", - threshEventData={}, + threshEventData=None, deviceId=None, contextUUID=None, deviceUUID=None, @@ -452,7 +452,7 @@ def writeMetric( @param max: used in the derive the max value for the metric. @type max: float @param threshEventData: extra data put into threshold events. - @type threshEventData: dict + @type threshEventData: dict | None @param deviceId: the id of the device for this metric. @type deviceId: str @param contextUUID: The device/component UUID value @@ -468,7 +468,7 @@ def writeMetricWithMetadata( timestamp="N", min="U", max="U", - threshEventData={}, + threshEventData=None, metadata=None, ): """ @@ -540,7 +540,7 @@ def writeRRD( @type max: number @param threshEventData: on threshold violation, update the event with this data. - @type threshEventData: dictionary + @type threshEventData: dictionary | None @param allowStaleDatapoint: attempt to write datapoint even if a newer datapoint has already been written. @type allowStaleDatapoint: boolean diff --git a/Products/ZenUtils/metricwriter.py b/Products/ZenUtils/metricwriter.py index 5c20bd4ce6..a34d5ba8f5 100644 --- a/Products/ZenUtils/metricwriter.py +++ b/Products/ZenUtils/metricwriter.py @@ -249,7 +249,7 @@ def notify( metric, timestamp, value, - thresh_event_data={}, + thresh_event_data=None, ): """ Check the specified value against thresholds and send any generated @@ -264,6 +264,7 @@ def notify( @return: """ if self._thresholds and value is not None: + thresh_event_data = thresh_event_data or {} if "eventKey" in thresh_event_data: eventKeyPrefix = [thresh_event_data["eventKey"]] else: From eca7622803cfb1f78f280b4365507134f356a969 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Fri, 26 Apr 2024 15:01:38 -0500 Subject: [PATCH 125/147] fix: Push Changes UI command forces configs to be built. ZEN-34837 --- Products/ZenCollector/configcache/api.py | 77 ++++++++ Products/ZenCollector/configcache/handlers.py | 166 ++++++++++++++++ .../ZenCollector/configcache/invalidator.py | 177 ++---------------- Products/Zuul/facades/devicefacade.py | 6 +- 4 files changed, 262 insertions(+), 164 deletions(-) create mode 100644 Products/ZenCollector/configcache/api.py create mode 100644 Products/ZenCollector/configcache/handlers.py diff --git a/Products/ZenCollector/configcache/api.py b/Products/ZenCollector/configcache/api.py new file mode 100644 index 0000000000..00cf68c1a1 --- /dev/null +++ b/Products/ZenCollector/configcache/api.py @@ -0,0 +1,77 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import + +import logging + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from .cache import CacheQuery +from .dispatcher import BuildConfigTaskDispatcher +from .handlers import ( + NewDeviceHandler, + DeviceUpdateHandler, + MissingConfigsHandler, +) +from .utils import get_build_timeout, get_minimum_ttl, getConfigServices + +log = logging.getLogger("zen.configcache") + + +class ConfigCache(object): + """ + Implements an API for manipulating the Configuration Cache to the rest + of the system. + """ + + @classmethod + def new(cls): + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + configClasses = getConfigServices() + dispatcher = BuildConfigTaskDispatcher(configClasses) + return cls(store, dispatcher) + + def __init__(self, store, dispatcher): + self.__new = NewDeviceHandler(log, store, dispatcher) + self.__update = DeviceUpdateHandler(log, store, dispatcher) + self.__missing = MissingConfigsHandler(log, store, dispatcher) + self.__store = store + + def update(self, device): + """ + Expires or retires existing configs for the device and sends build + jobs to speculatively create new configurations for the device. + May also delete configurations if a job produces no config for + configuration that existed previously. + """ + monitor = device.getPerformanceServerName() + if monitor is None: + raise RuntimeError( + "Device '%s' is not a member of a collector" % (device.id,) + ) + buildlimit = get_build_timeout(device) + # Check for device class change + stored_uid = self.__store.get_uid(device.id) + if device.getPrimaryPath() != stored_uid: + self.__new(device.id, monitor, buildlimit, False) + else: + # Note: the store's `search` method only returns keys for configs + # that exist. + keys_with_config = tuple( + self.__store.search( + CacheQuery(monitor=monitor, device=device.id) + ) + ) + minttl = get_minimum_ttl(device) + self.__update(keys_with_config, minttl) + self.__missing(device.id, monitor, keys_with_config, buildlimit) diff --git a/Products/ZenCollector/configcache/handlers.py b/Products/ZenCollector/configcache/handlers.py new file mode 100644 index 0000000000..d234b6119e --- /dev/null +++ b/Products/ZenCollector/configcache/handlers.py @@ -0,0 +1,166 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2023, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import + +import time + +from .cache import CacheKey, ConfigStatus + + +class NewDeviceHandler(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self.dispatcher = dispatcher + + def __call__(self, deviceId, monitor, buildlimit, newDevice=True): + keys = tuple( + CacheKey(svcname, monitor, deviceId) + for svcname in self.dispatcher.service_names + ) + keys_with_pending_status = set( + status.key + for status in self.store.get_status(*keys) + if isinstance(status, ConfigStatus.Pending) + ) + for key in keys_with_pending_status: + self.log.debug( + "build job already submitted for this config " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) + keys_without_pending_status = set(keys) - keys_with_pending_status + now = time.time() + self.store.set_pending( + *((key, now) for key in keys_without_pending_status) + ) + for key in keys_without_pending_status: + self.dispatcher.dispatch( + key.service, key.monitor, key.device, buildlimit, now + ) + self.log.info( + "submitted build job for %s " + "device=%s collector=%s service=%s", + "new device" if newDevice else "device with new device class", + key.device, + key.monitor, + key.service, + ) + + +class DeviceUpdateHandler(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self.dispatcher = dispatcher + + def __call__(self, keys, minttl): + current_statuses = tuple( + status + for status in self.store.get_status(*keys) + if isinstance(status, ConfigStatus.Current) + ) + + now = time.time() + retirement = now - minttl + + retired = set( + status.key + for status in current_statuses + if status.updated >= retirement + ) + expired = set( + status.key + for status in current_statuses + if status.key not in retired + ) + + self.store.set_retired(*((key, now) for key in retired)) + self.store.set_expired(*((key, now) for key in expired)) + + for key in retired: + self.log.info( + "retired configuration of changed device " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) + for key in expired: + self.log.info( + "expired configuration of changed device " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) + + +class MissingConfigsHandler(object): + + def __init__(self, log, store, dispatcher): + self.log = log + self.store = store + self.dispatcher = dispatcher + + def __call__(self, deviceId, monitor, keys, buildlimit): + """ + @param keys: These keys are associated with a config + @type keys: Sequence[CacheKey] + """ + # Send a job for for all config services that don't currently have + # an associated configuration. Some ZenPacks, i.e. vSphere, defer + # their modeling to a later time, so jobs for configuration services + # must be sent to pick up any new configs. + hasconfigs = tuple(key.service for key in keys) + noconfigkeys = tuple( + CacheKey(svcname, monitor, deviceId) + for svcname in self.dispatcher.service_names + if svcname not in hasconfigs + ) + # Identify all no-config keys that already have a status. + skipkeys = tuple( + status.key for status in self.store.get_status(*noconfigkeys) + ) + now = time.time() + for key in (k for k in noconfigkeys if k not in skipkeys): + self.store.set_pending((key, now)) + self.dispatcher.dispatch( + key.service, key.monitor, key.device, buildlimit, now + ) + self.log.debug( + "submitted build job for possibly missing config " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) + + +class RemoveConfigsHandler(object): + + def __init__(self, log, store): + self.log = log + self.store = store + + def __call__(self, keys): + self.store.remove(*keys) + for key in keys: + self.log.info( + "removed configuration of deleted device " + "device=%s collector=%s service=%s", + key.device, + key.monitor, + key.service, + ) diff --git a/Products/ZenCollector/configcache/invalidator.py b/Products/ZenCollector/configcache/invalidator.py index ebf0aa0013..b4f0776ba0 100644 --- a/Products/ZenCollector/configcache/invalidator.py +++ b/Products/ZenCollector/configcache/invalidator.py @@ -10,7 +10,6 @@ from __future__ import print_function, absolute_import import logging -import time from multiprocessing import Process @@ -24,9 +23,15 @@ from .app import Application from .app.args import get_subparser -from .cache import CacheKey, CacheQuery, ConfigStatus +from .cache import CacheQuery from .debug import Debug as DebugCommand from .dispatcher import BuildConfigTaskDispatcher +from .handlers import ( + NewDeviceHandler, + DeviceUpdateHandler, + MissingConfigsHandler, + RemoveConfigsHandler, +) from .modelchange import InvalidationCause from .utils import ( get_build_timeout, @@ -179,7 +184,7 @@ def _removeDeleted(log, store, devices): if (key.device, key.monitor) not in devices ) if devices_not_found: - _RemoveConfigsHandler(log, store)(devices_not_found) + RemoveConfigsHandler(log, store)(devices_not_found) else: log.info("no dangling configurations found") @@ -190,7 +195,7 @@ def _addNewOrChangedDevices(log, store, dispatcher, dmd, devices): # Query the catalog for all devices new_devices = 0 changed_devices = 0 - handle = _NewDeviceHandler(log, store, dispatcher) + handle = NewDeviceHandler(log, store, dispatcher) for (deviceId, monitorId), uid in devices.iteritems(): try: device = dmd.unrestrictedTraverse(uid) @@ -227,10 +232,10 @@ class _InvalidationProcessor(object): def __init__(self, log, store, dispatcher): self.log = log self.store = store - self._remove = _RemoveConfigsHandler(log, store) - self._update = _DeviceUpdateHandler(log, store, dispatcher) - self._missing = _MissingConfigsHandler(log, store, dispatcher) - self._new = _NewDeviceHandler(log, store, dispatcher) + self._remove = RemoveConfigsHandler(log, store) + self._update = DeviceUpdateHandler(log, store, dispatcher) + self._missing = MissingConfigsHandler(log, store, dispatcher) + self._new = NewDeviceHandler(log, store, dispatcher) def __call__(self, device, oid, reason): uid = device.getPrimaryId() @@ -258,9 +263,7 @@ def __call__(self, device, oid, reason): self._new(device.id, monitor, buildlimit, False) else: self._update(keys_with_config, minttl) - self._missing( - device.id, monitor, keys_with_config, buildlimit - ) + self._missing(device.id, monitor, keys_with_config, buildlimit) elif reason is InvalidationCause.Removed: self._remove(keys_with_config) else: @@ -272,155 +275,3 @@ def __call__(self, device, oid, reason): monitor, oid, ) - - -class _NewDeviceHandler(object): - - def __init__(self, log, store, dispatcher): - self.log = log - self.store = store - self.dispatcher = dispatcher - - def __call__(self, deviceId, monitor, buildlimit, newDevice=True): - keys = tuple( - CacheKey(svcname, monitor, deviceId) - for svcname in self.dispatcher.service_names - ) - keys_with_pending_status = set( - status.key - for status in self.store.get_status(*keys) - if isinstance(status, ConfigStatus.Pending) - ) - for key in keys_with_pending_status: - self.log.debug( - "build job already submitted for this config " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) - keys_without_pending_status = set(keys) - keys_with_pending_status - now = time.time() - self.store.set_pending( - *((key, now) for key in keys_without_pending_status) - ) - for key in keys_without_pending_status: - self.dispatcher.dispatch( - key.service, key.monitor, key.device, buildlimit, now - ) - self.log.info( - "submitted build job for %s " - "device=%s collector=%s service=%s", - "new device" if newDevice else "device with new device class", - key.device, - key.monitor, - key.service, - ) - - -class _DeviceUpdateHandler(object): - - def __init__(self, log, store, dispatcher): - self.log = log - self.store = store - self.dispatcher = dispatcher - - def __call__(self, keys, minttl): - current_statuses = tuple( - status - for status in self.store.get_status(*keys) - if isinstance(status, ConfigStatus.Current) - ) - - now = time.time() - retirement = now - minttl - - retired = set( - status.key - for status in current_statuses - if status.updated >= retirement - ) - expired = set( - status.key - for status in current_statuses - if status.key not in retired - ) - - self.store.set_retired(*((key, now) for key in retired)) - self.store.set_expired(*((key, now) for key in expired)) - - for key in retired: - self.log.info( - "retired configuration of changed device " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) - for key in expired: - self.log.info( - "expired configuration of changed device " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) - - -class _MissingConfigsHandler(object): - - def __init__(self, log, store, dispatcher): - self.log = log - self.store = store - self.dispatcher = dispatcher - - def __call__(self, deviceId, monitor, keys, buildlimit): - """ - @param keys: These keys are associated with a config - @type keys: Sequence[CacheKey] - """ - # Send a job for for all config services that don't currently have - # an associated configuration. Some ZenPacks, i.e. vSphere, defer - # their modeling to a later time, so jobs for configuration services - # must be sent to pick up any new configs. - hasconfigs = tuple(key.service for key in keys) - noconfigkeys = tuple( - CacheKey(svcname, monitor, deviceId) - for svcname in self.dispatcher.service_names - if svcname not in hasconfigs - ) - # Identify all no-config keys that already have a status. - skipkeys = tuple( - status.key for status in self.store.get_status(*noconfigkeys) - ) - now = time.time() - for key in (k for k in noconfigkeys if k not in skipkeys): - self.store.set_pending((key, now)) - self.dispatcher.dispatch( - key.service, key.monitor, key.device, buildlimit, now - ) - self.log.debug( - "submitted build job for possibly missing config " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) - - -class _RemoveConfigsHandler(object): - - def __init__(self, log, store): - self.log = log - self.store = store - - def __call__(self, keys): - self.store.remove(*keys) - for key in keys: - self.log.info( - "removed configuration of deleted device " - "device=%s collector=%s service=%s", - key.device, - key.monitor, - key.service, - ) diff --git a/Products/Zuul/facades/devicefacade.py b/Products/Zuul/facades/devicefacade.py index 166efd6471..7dceab75e6 100644 --- a/Products/Zuul/facades/devicefacade.py +++ b/Products/Zuul/facades/devicefacade.py @@ -30,6 +30,7 @@ from Products.Jobber.jobs import FacadeMethodJob from Products.Zuul.tree import SearchResults from Products.DataCollector.Plugins import CoreImporter, PackImporter, loadPlugins +from Products.ZenCollector.configcache.api import ConfigCache from Products.ZenModel.DeviceOrganizer import DeviceOrganizer from Products.ZenModel.ComponentGroup import ComponentGroup from Products.ZenModel.DeviceGroup import DeviceGroup @@ -529,8 +530,11 @@ def setMonitor(self, uids, monitor=False): def pushChanges(self, uids): devs = imap(self._getObject, uids) + if not devs: + return + configcache = ConfigCache.new() for dev in devs: - dev.pushConfig() + configcache.update(dev) def modelDevices(self, uids): devs = imap(self._getObject, uids) From 1fdfaad5c4e24bcc6720b196c2c2e9b9b7d1fc09 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 29 Apr 2024 11:50:11 -0500 Subject: [PATCH 126/147] fix: configure Celery on initialization Set the Celery config early during startup because Celery doesn't appear to support being configured multiple times. Only the initial configuration is used by Celery. ZEN-34828 --- Products/Jobber/bin.py | 21 ++++++++++++++++ Products/Jobber/config.py | 49 ++++++++++++++++++------------------ Products/Jobber/signals.zcml | 5 ---- Products/Jobber/worker.py | 12 +-------- Products/Jobber/zenjobs.py | 10 +------- 5 files changed, 48 insertions(+), 49 deletions(-) diff --git a/Products/Jobber/bin.py b/Products/Jobber/bin.py index 5c0a6cc20a..5131b57edf 100644 --- a/Products/Jobber/bin.py +++ b/Products/Jobber/bin.py @@ -16,9 +16,30 @@ def main(): from celery.bin.celery import main from Products.ZenUtils.Utils import load_config + # Dynamic configuration shenanigans because Celery can't be re-configured + # after its initial configuration has been set. + _configure_celery() + load_config("signals.zcml", Products.Jobber) # All calls to celery need the 'app instance' for zenjobs. sys.argv[1:] = ["-A", "Products.Jobber.zenjobs"] + sys.argv[1:] sys.exit(main()) + + +def _configure_celery(): + import argparse + import sys + from Products.Jobber import config + + parser = argparse.ArgumentParser() + parser.add_argument("--config-file") + + args, remainder = parser.parse_known_args() + if not args.config_file: + return + + cfg = config.getConfig(args.config_file) + config.ZenCeleryConfig = config.CeleryConfig.from_config(cfg) + sys.argv[1:] = remainder diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 0edb2fa143..7935c57256 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -35,7 +35,6 @@ "job-soft-time-limit": 18000, # 5 hours "zenjobs-worker-alive-timeout": 300.0, # 5 minutes "redis-url": DEFAULT_REDIS_URL, - "task-protocol": 1, } @@ -62,23 +61,24 @@ def getConfig(filename=None): """Return a dict containing the configuration for zenjobs.""" global _configuration - if _configuration: - return _configuration - - conf = _default_configs.copy() - conf.update(getGlobalConfiguration()) - + configfile_contents = {} if filename is not None: if not os.path.exists(filename): filename = zenPath("etc", filename) - - app_config_loader = ConfigLoader([filename], Config) - try: - conf.update(app_config_loader()) - except IOError as ex: - # Re-raise exception if the error is not "File not found" - if ex.errno != 2: - raise + try: + configfile_contents = ConfigLoader([filename], Config)() + except IOError as ex: + # Re-raise exception if the error is not "File not found" + if ex.errno != 2: + raise + + conf = _configuration.setdefault(filename, {}) + if conf: + return conf + + conf.update(_default_configs) + conf.update(getGlobalConfiguration()) + conf.update(configfile_contents) # Convert the configuration value types to useable types. for key, cast in _xform.items(): @@ -86,10 +86,6 @@ def getConfig(filename=None): continue conf[key] = cast(conf[key]) - # only save it if a filename was specified - if filename is not None: - _configuration = conf - return conf @@ -115,6 +111,7 @@ class CeleryConfig(object): task_time_limit = attr.ib() task_soft_time_limit = attr.ib() beat_max_loop_interval = attr.ib() + worker_proc_alive_timeout = attr.ib() timezone = attr.ib(default=None) accept_content = attr.ib(default=["without-unicode"]) @@ -127,6 +124,7 @@ class CeleryConfig(object): task_routes = attr.ib( default={"configcache.build_device_config": {"queue": "configcache"}} ) + result_extended = attr.ib(default=True) result_serializer = attr.ib(default="without-unicode") worker_prefetch_multiplier = attr.ib(default=1) task_acks_late = attr.ib(default=True) @@ -134,13 +132,13 @@ class CeleryConfig(object): task_store_errors_even_if_ignored = attr.ib(default=True) task_serializer = attr.ib(default="without-unicode") task_track_started = attr.ib(default=True) - CELERYBEAT_REDIRECT_STDOUTS = attr.ib(default=True) - CELERYBEAT_REDIRECT_STDOUTS_LEVEL = attr.ib(default="INFO") worker_send_task_events = attr.ib(default=True) task_send_sent_event = attr.ib(default=True) worker_log_color = attr.ib(default=False) - worker_proc_alive_timeout = attr.ib(default=300.0) - task_protocol = attr.ib(default=1) + + # Are these still used? + CELERYBEAT_REDIRECT_STDOUTS = attr.ib(default=True) + CELERYBEAT_REDIRECT_STDOUTS_LEVEL = attr.ib(default="INFO") @classmethod def from_config(cls, cfg={}): @@ -156,10 +154,13 @@ def from_config(cls, cfg={}): "scheduler-max-loop-interval" ), "worker_proc_alive_timeout": cfg.get("zenjobs-worker-alive-timeout"), - "task_protocol": cfg.get("task-protocol", 1), } tz = os.environ.get("TZ") if tz: args["timezone"] = tz return cls(**args) + + +# Initialized with default values (for when --config-file is not specified) +ZenCeleryConfig = CeleryConfig.from_config(getConfig()) diff --git a/Products/Jobber/signals.zcml b/Products/Jobber/signals.zcml index 90380c8484..6166c9a65d 100644 --- a/Products/Jobber/signals.zcml +++ b/Products/Jobber/signals.zcml @@ -3,11 +3,6 @@ - - Date: Tue, 30 Apr 2024 14:34:32 -0500 Subject: [PATCH 127/147] fix: use ZenCeleryConfig object where possible. ZEN-34828 --- Products/Jobber/scheduler.py | 6 ++---- Products/Jobber/storage.py | 7 +++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Products/Jobber/scheduler.py b/Products/Jobber/scheduler.py index e265c8b33b..a6e8aabaf7 100644 --- a/Products/Jobber/scheduler.py +++ b/Products/Jobber/scheduler.py @@ -21,7 +21,7 @@ from celery.beat import Scheduler from celery.schedules import crontab -from .config import getConfig, CeleryConfig +from .config import getConfig, ZenCeleryConfig class ZenJobsScheduler(Scheduler): @@ -222,9 +222,7 @@ def _key(name): def _getClient(): """Create and return the ZenJobs JobStore client.""" - return redis.StrictRedis.from_url( - CeleryConfig.from_config(getConfig()).result_backend - ) + return redis.StrictRedis.from_url(ZenCeleryConfig.result_backend) def handle_beat_init(*args, **kw): diff --git a/Products/Jobber/storage.py b/Products/Jobber/storage.py index 4b438d0655..e620cf5a37 100644 --- a/Products/Jobber/storage.py +++ b/Products/Jobber/storage.py @@ -19,7 +19,7 @@ from Products.ZenUtils.RedisUtils import getRedisClient -from .config import getConfig, CeleryConfig +from .config import ZenCeleryConfig _keybase = "zenjobs:job:" _keypattern = _keybase + "*" @@ -31,9 +31,8 @@ def makeJobStore(): """Create and return the ZenJobs JobStore client.""" - cfg = CeleryConfig.from_config(getConfig()) - client = getRedisClient(url=cfg.result_backend) - return JobStore(client, expires=cfg.result_expires) + client = getRedisClient(url=ZenCeleryConfig.result_backend) + return JobStore(client, expires=ZenCeleryConfig.result_expires) class _Converter(object): From 5c269b985a9b7c098abbcd67752924d4fd2e7297 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 1 May 2024 09:16:02 -0500 Subject: [PATCH 128/147] Handle unicode inputs into the build_device_config task. ZEN-34840 --- Products/Jobber/serialization.py | 6 ++++-- Products/Jobber/task/dmd.py | 9 ++++++--- Products/ZenCollector/configcache/cache/model.py | 12 ++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Products/Jobber/serialization.py b/Products/Jobber/serialization.py index 3e3b400271..844911827b 100644 --- a/Products/Jobber/serialization.py +++ b/Products/Jobber/serialization.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, print_function +import six + from json import ( loads as json_loads, dumps as json_dumps, @@ -25,7 +27,7 @@ def _process_list(seq): while stack: lst = stack.pop() for idx, item in enumerate(lst): - if isinstance(item, unicode): + if isinstance(item, six.text_type): lst[idx] = str(item) if isinstance(item, list): stack.append(item) @@ -37,7 +39,7 @@ def _decode_hook(*args, **kw): for n, i in enumerate(args): for pair in i: k, v = pair - if isinstance(v, unicode): + if isinstance(v, six.text_type): v = str(v) elif isinstance(v, list): v = _process_list(v) diff --git a/Products/Jobber/task/dmd.py b/Products/Jobber/task/dmd.py index 7a26a2a43f..7273ada1e0 100644 --- a/Products/Jobber/task/dmd.py +++ b/Products/Jobber/task/dmd.py @@ -48,9 +48,12 @@ class DMD(object): def __call__(self, *args, **kwargs): """Override to attach a zodb root object to the task.""" - # NOTE: work-around for Celery >= 4.0 - # userid = getattr(self.request, "userid", None) - userid = self.request.headers.get("userid") + # Celery < 4.0 had a 'headers' attribute + headers = getattr(self.request, "headers", None) + if headers is not None: + userid = headers.get("userid") + else: + userid = getattr(self.request, "userid", None) with zodb(self.app.db, userid, self.log) as dmd: self.__dmd = dmd try: diff --git a/Products/ZenCollector/configcache/cache/model.py b/Products/ZenCollector/configcache/cache/model.py index 4cafb636d3..8f81714fdf 100644 --- a/Products/ZenCollector/configcache/cache/model.py +++ b/Products/ZenCollector/configcache/cache/model.py @@ -18,16 +18,16 @@ @attr.s(frozen=True, slots=True) class CacheQuery(object): - service = attr.ib(validator=instance_of(str), default="*") - monitor = attr.ib(validator=instance_of(str), default="*") - device = attr.ib(validator=instance_of(str), default="*") + service = attr.ib(converter=str, validator=instance_of(str), default="*") + monitor = attr.ib(converter=str, validator=instance_of(str), default="*") + device = attr.ib(converter=str, validator=instance_of(str), default="*") @attr.s(frozen=True, slots=True) class CacheKey(object): - service = attr.ib(validator=instance_of(str)) - monitor = attr.ib(validator=instance_of(str)) - device = attr.ib(validator=instance_of(str)) + service = attr.ib(converter=str, validator=instance_of(str)) + monitor = attr.ib(converter=str, validator=instance_of(str)) + device = attr.ib(converter=str, validator=instance_of(str)) @attr.s(slots=True) From 65799987fdc3bb6d32019f42f75b4a5a9c62b3db Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 24 Apr 2024 13:40:13 -0500 Subject: [PATCH 129/147] Display statuses for configurations that don't have config data. ZEN-34827 --- .../ZenCollector/configcache/cache/storage.py | 74 ++++++++++++++++++- Products/ZenCollector/configcache/cli/list.py | 26 ++++--- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 456b42f1a7..29e1bb3302 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -62,6 +62,8 @@ import ast import json import logging +import operator +import re from functools import partial from itertools import chain, islice @@ -382,7 +384,7 @@ def _get_watch_keys(self, keys): def get_status(self, *keys): """ - Returns an interable of ConfigStatus objects. + Returns an iterable of ConfigStatus objects. @rtype: Iterable[ConfigStatus] """ @@ -393,6 +395,76 @@ def get_status(self, *keys): if status is not None: yield status + def get_statuses(self, query=CacheQuery()): + """ + Return all status objects matching the query. + + @type query: CacheQuery + @rtype: Iterable[ConfigStatus] + """ + statuses = [] + keys = set() + uids = {} + tables = ( + (self.__expired, ConfigStatus.Expired), + (self.__retired, ConfigStatus.Retired), + (self.__pending, ConfigStatus.Pending), + (self.__building, ConfigStatus.Building), + ) + + def accept_all(_): + return True + + def filter_regex(regex, value): + m = regex.match(value) + return m is not None + + if query.device == "*": + predicate = accept_all + elif "*" in query.device: + expr = query.device.replace("*", ".*") + regex = re.compile(expr) + predicate = partial(filter_regex, regex) + else: + predicate = partial(operator.eq, query.device) + + for table, cls in tables: + for key, ts in self._get_metadata( + table, query.service, query.monitor + ): + if predicate(key.device): + keys.add(key) + uid = self._get_uid(uids, key.device) + statuses.append(cls(key, uid, ts)) + for key, ts in self._get_metadata( + self.__age, query.service, query.monitor + ): + # Skip age (aka 'current') data for keys that already have + # some other status. + if key in keys: + continue + if predicate(key.device): + uid = self._get_uid(uids, key.device) + statuses.append(ConfigStatus.Current(key, uid, ts)) + return statuses + + def _get_uid(self, uids, device): + uid = uids.get(device) + if uid is None: + uid = self.__uids.get(self.__client, device) + if uid: + uids[device] = uid + return uid + + def _get_metadata(self, table, service, monitor): + pairs = table.get_pairs( + self.__client, service=service, monitor=monitor + ) + return ( + (CacheKey(svcId, monId, devId), _to_ts(score)) + for svcId, monId, devId, score in table.scan(self.__client, pairs) + ) + def _get_status_from_scores(self, scores, key, uid): age, retired, expired, pending, building = scores if building is not None: diff --git a/Products/ZenCollector/configcache/cli/list.py b/Products/ZenCollector/configcache/cli/list.py index 7d18019370..d58d6608e4 100644 --- a/Products/ZenCollector/configcache/cli/list.py +++ b/Products/ZenCollector/configcache/cli/list.py @@ -88,30 +88,32 @@ def run(self): monitor=self._monitor, device=self._devices[0], ) + elif len(self._devices) == 1: + query = CacheQuery( + service=self._service, + monitor=self._monitor, + device=self._devices[0], + ) else: query = CacheQuery(service=self._service, monitor=self._monitor) - results = store.get_status(*store.search(query)) + data = store.get_statuses(query) if self._states: - results = ( - status - for status in results - if isinstance(status, self._states) + data = ( + status for status in data if isinstance(status, self._states) ) - rows = [] - maxd, maxs, maxm = 0, 0, 0 - if len(self._devices) > 0: + if len(self._devices) > 1: data = ( status - for status in results + for status in data if status.key.device in self._devices ) - else: - data = results + rows = [] + maxd, maxs, maxm = 1, 1, 1 for status in sorted( data, key=lambda x: (x.key.device, x.key.service) ): if self._showuid: - devid = status.uid + devid = status.uid or status.key.device else: devid = status.key.device status_text = _format_status(status) From 0157941d4a6a2e2b5dd0f79df17952ae3122f105 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 1 May 2024 14:29:09 -0500 Subject: [PATCH 130/147] Add error handling around getParentNode. Sometimes, an object doesn't have the getParentNode method, so record this unexpected situation into the log. ZEN-34834 --- Products/ZenCollector/configcache/modelchange/oids.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Products/ZenCollector/configcache/modelchange/oids.py b/Products/ZenCollector/configcache/modelchange/oids.py index 4da659b80b..d57f5fffd5 100644 --- a/Products/ZenCollector/configcache/modelchange/oids.py +++ b/Products/ZenCollector/configcache/modelchange/oids.py @@ -165,7 +165,11 @@ def transformOid(self, oid): ) obj = self._entity while not isinstance(obj, DeviceClass): - obj = obj.getParentNode() + try: + obj = obj.getParentNode() + except Exception: + log.exception("unable to find device class entity=%r", obj) + return None return _getDevicesFromDeviceClass(obj) From 74114b7b11f66e700d3655f8bff7f8d26a7de6e6 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 2 May 2024 14:50:26 -0500 Subject: [PATCH 131/147] Inputs into save_jobrecord have changed. Enabling the Celery's message protocol v2 has changed the structure of the data passed to the save_jobrecord function, so this commit updates the function to handle the changed structure of the inputs. ZEN-34843 --- Products/Jobber/model.py | 28 +++++++++++++++-------- Products/Jobber/tests/test_redisrecord.py | 18 ++++----------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/Products/Jobber/model.py b/Products/Jobber/model.py index d6227fe727..b04caebda7 100644 --- a/Products/Jobber/model.py +++ b/Products/Jobber/model.py @@ -280,10 +280,9 @@ def from_signal(cls, body, headers, properties): """Return a RedisRecord object built from the arguments passed to a before_task_publish signal handler. """ - jobid = body.get("id") - taskname = body.get("task") - args = body.get("args", ()) - kwargs = body.get("kwargs", {}) + jobid = headers.get("id") + taskname = headers.get("task") + args, kwargs, _ = body return cls._build(jobid, taskname, args, kwargs, headers, properties) @classmethod @@ -313,19 +312,26 @@ def save_jobrecord(log, body=None, headers=None, properties=None, **ignored): :param dict headers: Headers to accompany message sent to Celery worker :param dict properties: Additional task and custom key/value pairs """ + if headers is None: + # If headers is None, bad signal so ignore. + log.info("no headers, bad signal?") + return + if not body: # If body is empty (or None), no job to save. log.info("no body, so no job") return - if headers is None: - # If headers is None, bad signal so ignore. - log.info("no headers, bad signal?") + if not isinstance(body, tuple): + # body is not in protocol V2 format + log.warning("task data not in protocol V2 format") return - task = get_app().tasks.get(body.get("task")) + taskname = headers.get("task") + task = get_app().tasks.get(taskname) + if task is None: - log.warn("Ignoring unknown task: %s", body.get("task")) + log.warn("Ignoring unknown task: %s", taskname) return # If the result of tasks is ignored, don't create a job record. @@ -350,8 +356,10 @@ def save_jobrecord(log, body=None, headers=None, properties=None, **ignored): if not saved: return + _, _, canvas = body + # Iterate over the callbacks. - callbacks = body.get("callbacks") or [] + callbacks = canvas.get("callbacks") or [] links = [] for cb in callbacks: links.extend(cb.flatten_links()) diff --git a/Products/Jobber/tests/test_redisrecord.py b/Products/Jobber/tests/test_redisrecord.py index 679a24894c..c753c090b8 100644 --- a/Products/Jobber/tests/test_redisrecord.py +++ b/Products/Jobber/tests/test_redisrecord.py @@ -140,25 +140,15 @@ def test_from_signature_with_custom_description(t): def test_from_signal(t): userid = "blink" t.expected["userid"] = userid - body = { - "id": t.jobid, - "task": t.task.name, - "args": t.args, - "kwargs": t.kw, - } - headers = {"userid": userid} + body = (t.args, t.kw, {}) + headers = {"userid": userid, "task": t.task.name, "id": t.jobid} properties = {} actual = RedisRecord.from_signal(body, headers, properties) t.assertDictEqual(t.expected, actual) def test_from_signal_with_details(t): - body = { - "id": t.jobid, - "task": t.task.name, - "args": t.args, - "kwargs": t.kw, - } - headers = {} + body = (t.args, t.kw, {}) + headers = {"id": t.jobid, "task": t.task.name} properties = {"a": 1, "b": 2} t.expected["details"] = properties actual = RedisRecord.from_signal(body, headers, properties) From 60279302786e3ac23ee34c5e6926d7411ef996f9 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Thu, 2 May 2024 16:59:42 -0500 Subject: [PATCH 132/147] Handle local and device class templates differently. Also handle local and device class thresholds differently. ZEN-34836 --- .../configcache/modelchange/oids.py | 89 +++++++++++++------ 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/Products/ZenCollector/configcache/modelchange/oids.py b/Products/ZenCollector/configcache/modelchange/oids.py index d57f5fffd5..4eb425748d 100644 --- a/Products/ZenCollector/configcache/modelchange/oids.py +++ b/Products/ZenCollector/configcache/modelchange/oids.py @@ -16,7 +16,7 @@ from zope.interface import implementer from Products.ZenHub.interfaces import IInvalidationOid -from Products.ZenModel.DeviceClass import DeviceClass +from Products.ZenModel.Device import Device from Products.ZenRelations.RelationshipBase import IRelationship from Products.Zuul.catalog.interfaces import IModelCatalogTool @@ -95,14 +95,20 @@ def transformOid(self, oid): if not template: return () dc = _getDeviceClass(template) - if not dc: - return () + if dc: + log.debug( + "[DataPointToDevice] return OIDs of devices associated " + "with DataPoint entity=%s", + self._entity, + ) + return _getDevicesFromDeviceClass(dc) + log.debug( - "[DataPointToDevice] return OIDs of devices associated " - "with DataPoint entity=%s ", + "[DataPointToDevice] return OID of device associated " + "with DataPoint of local RRDTemplate entity=%s", self._entity, ) - return _getDevicesFromDeviceClass(dc) + return _getDeviceFromLocalTemplate(template) @implementer(IInvalidationOid) @@ -114,14 +120,20 @@ def transformOid(self, oid): if not template: return () dc = _getDeviceClass(template) - if not dc: - return () + if dc: + log.debug( + "[DataSourceToDevice] return OIDs of devices associated " + "with DataSource entity=%s", + self._entity, + ) + return _getDevicesFromDeviceClass(dc) + log.debug( - "[DataSourceToDevice] return OIDs of devices associated " - "with DataSource entity=%s", + "[DataSourceToDevice] return OID of device associated " + "with DataSource of local RRDTemplate entity=%s", self._entity, ) - return _getDevicesFromDeviceClass(dc) + return _getDeviceFromLocalTemplate(template) @implementer(IInvalidationOid) @@ -130,14 +142,20 @@ class TemplateToDevice(BaseTransform): def transformOid(self, oid): dc = _getDeviceClass(self._entity) - if not dc: - return () + if dc: + log.debug( + "[TemplateToDevice] return OIDs of devices associated " + "with RRDTemplate entity=%s", + self._entity, + ) + return _getDevicesFromDeviceClass(dc) + log.debug( - "[TemplateToDevice] return OIDs of devices associated " - "with RRDTemplate entity=%s ", + "[TemplateToDevice] return OID of device associated " + "with local RRDTemplate entity=%s", self._entity, ) - return _getDevicesFromDeviceClass(dc) + return _getDeviceFromLocalTemplate(self._entity) @implementer(IInvalidationOid) @@ -147,7 +165,7 @@ class DeviceClassToDevice(BaseTransform): def transformOid(self, oid): log.debug( "[DeviceClassToDevice] return OIDs of devices associated " - "with DeviceClass entity=%s ", + "with DeviceClass entity=%s", self._entity, ) return _getDevicesFromDeviceClass(self._entity) @@ -158,19 +176,24 @@ class ThresholdToDevice(BaseTransform): """Return the device OIDs in the DeviceClass hierarchy.""" def transformOid(self, oid): + template = _getTemplate(self._entity) + if not template: + return () + dc = _getDeviceClass(template) + if dc: + log.debug( + "[ThresholdToDevice] return OIDs of devices associated " + "with threshold entity=%s", + self._entity, + ) + return _getDevicesFromDeviceClass(dc) + log.debug( - "[ThresholdToDevice] return OIDs of devices associated " - "with Threshold entity=%s ", + "[ThresholdToDevice] return OID of device associated " + "with Threshold of local RRDTemplate entity=%s", self._entity, ) - obj = self._entity - while not isinstance(obj, DeviceClass): - try: - obj = obj.getParentNode() - except Exception: - log.exception("unable to find device class entity=%r", obj) - return None - return _getDevicesFromDeviceClass(obj) + return _getDeviceFromLocalTemplate(template) def _getDataSource(dp): @@ -200,6 +223,18 @@ def _getDeviceClass(template): return dc.primaryAq() +def _getDeviceFromLocalTemplate(template): + obj = template + while not isinstance(obj, Device): + try: + obj = obj.getParentNode() + except Exception: + if log.isEnabledFor(logging.DEBUG): + log.warn("unable to find device template=%r", template) + return None + return obj._p_oid + + def _getDevicesFromDeviceClass(dc): tool = IModelCatalogTool(dc.dmd.Devices) query, _ = tool._build_query( From 644c1956a29c69e50148efad7f1bcbe3141aa25a Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Mon, 6 May 2024 08:18:10 -0500 Subject: [PATCH 133/147] Add `dmd_read_only` flag to `DMD` mixin class. Add the `dmd_read_only` attribute to tasks using the `DMD` mixin class. When set to True, the transaction is aborted when the task completes. By default, `dmd_read_only` is False. Removed Device.getRRDTemplates doctest because it was dependent on zodb being in a specific state to succeed. ZEN-34849 --- Products/Jobber/task/base.py | 5 +- Products/Jobber/task/dmd.py | 16 ++++- Products/Jobber/tests/test_dmd.py | 77 +++++++++++++++++++++++ Products/ZenCollector/configcache/task.py | 1 + Products/ZenModel/Device.py | 5 -- 5 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 Products/Jobber/tests/test_dmd.py diff --git a/Products/Jobber/task/base.py b/Products/Jobber/task/base.py index 87cb8aeffa..d1d4db134d 100644 --- a/Products/Jobber/task/base.py +++ b/Products/Jobber/task/base.py @@ -15,7 +15,7 @@ from AccessControl.SecurityManagement import getSecurityManager from celery import Task -from celery.exceptions import Ignore, SoftTimeLimitExceeded +from celery.exceptions import SoftTimeLimitExceeded from ..config import getConfig from ..utils.log import get_task_logger, get_logger @@ -118,9 +118,6 @@ def __call__(self, *args, **kwargs): del self.__run def __exec(self, *args, **kwargs): - if self.request.id is None: - self.log.error("Bad task: No ID found request=%s", self.request) - raise Ignore() self.log.info("Job started") mlog.debug("Job started request=%s", self.request) start = time.time() diff --git a/Products/Jobber/task/dmd.py b/Products/Jobber/task/dmd.py index 7273ada1e0..28d21c585a 100644 --- a/Products/Jobber/task/dmd.py +++ b/Products/Jobber/task/dmd.py @@ -45,6 +45,12 @@ class DMD(object): """ abstract = True + dmd_read_only = False + + def __new__(cls, *args, **kwargs): + task = super(DMD, cls).__new__(cls, *args, **kwargs) + task.__dmd = None + return task def __call__(self, *args, **kwargs): """Override to attach a zodb root object to the task.""" @@ -68,8 +74,12 @@ def __call__(self, *args, **kwargs): def __retry_on_conflict(self, *args, **kw): try: result = self.__run(*args, **kw) - transaction.commit() - self.log.debug("Transaction committed") + if not self.dmd_read_only: + transaction.commit() + self.log.debug("Transaction committed") + else: + transaction.abort() + self.log.debug("Transaction aborted reason=read-only-task") return result except (ReadConflictError, ConflictError) as ex: transaction.abort() @@ -98,7 +108,7 @@ def zodb(db, userid, log): :param db: ZODB database connection. :param str userid: The ID of the user to authenticate with. """ - session = db.open() + session = db.open() # type: ZODB.Connection try: mlog.debug("Started ZODB session") root = session.root() diff --git a/Products/Jobber/tests/test_dmd.py b/Products/Jobber/tests/test_dmd.py new file mode 100644 index 0000000000..72785e35a1 --- /dev/null +++ b/Products/Jobber/tests/test_dmd.py @@ -0,0 +1,77 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import absolute_import, print_function + +import logging + +from unittest import TestCase + +from mock import MagicMock, patch + +from ..task import requires, DMD +from ..zenjobs import app + + +class DMDTest(TestCase): + """Test the DMD mixin class.""" + + def setUp(self): + log = logging.getLogger() + log.setLevel(logging.FATAL + 1) + + def tearDown(self): + log = logging.getLogger() + log.setLevel(logging.NOTSET) + + @app.task(bind=True, base=requires(DMD)) + def dmd_task_rw(self): + pass + + @app.task(bind=True, base=requires(DMD), dmd_read_only=True) + def dmd_task_ro(self): + pass + + def test_rw_defaults(t): + t.assertIsInstance(t.dmd_task_rw, DMD) + t.assertFalse(t.dmd_task_rw.dmd_read_only) + t.assertIsNone(t.dmd_task_rw.dmd) + + def test_ro_defaults(t): + t.assertIsInstance(t.dmd_task_ro, DMD) + t.assertTrue(t.dmd_task_ro.dmd_read_only) + t.assertIsNone(t.dmd_task_ro.dmd) + + @patch("Products.Jobber.task.dmd.transaction") + def test_rw(t, transaction_): + db = MyMagicMock() + app.db = db + try: + t.dmd_task_rw() + transaction_.abort.assert_not_called() + transaction_.commit.assert_called_with() + finally: + del app.db + + @patch("Products.Jobber.task.dmd.transaction") + def test_ro(t, transaction_): + db = MyMagicMock() + app.db = db + try: + t.dmd_task_ro() + transaction_.commit.assert_not_called() + transaction_.abort.assert_called_with() + finally: + del app.db + + +class MyMagicMock(MagicMock): + + def __of__(self, *args, **kw): + return self diff --git a/Products/ZenCollector/configcache/task.py b/Products/ZenCollector/configcache/task.py index a2f7de96cd..61f28f7325 100644 --- a/Products/ZenCollector/configcache/task.py +++ b/Products/ZenCollector/configcache/task.py @@ -32,6 +32,7 @@ summary="Create Device Configuration Task", description_template="Create the configuration for device {2}.", ignore_result=True, + dmd_read_only=True, ) def build_device_config( self, monitorname, deviceid, configclassname, submitted=None diff --git a/Products/ZenModel/Device.py b/Products/ZenModel/Device.py index c2371b8eb3..ed1230e55a 100644 --- a/Products/ZenModel/Device.py +++ b/Products/ZenModel/Device.py @@ -459,11 +459,6 @@ def getRRDTemplates(self): Returns all the templates bound to this Device @rtype: list - - >>> from Products.ZenModel.Device import manage_addDevice - >>> manage_addDevice(devices, 'test') - >>> devices.test.getRRDTemplates() - [] """ if not hasattr(self, "zDeviceTemplates"): return ManagedEntity.getRRDTemplates(self) From b32c6d775564cc54226c5b75e43b6474d6eea1ad Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 7 May 2024 14:42:08 -0500 Subject: [PATCH 134/147] Compare user input device ID to the device ID. A device's config may have a `configId` value different from the `id` value, so compare a user specified device ID with the config's `id` value. ZEN-34850 --- Products/ZenCollector/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenCollector/daemon.py b/Products/ZenCollector/daemon.py index 0f61023a81..9813b09154 100644 --- a/Products/ZenCollector/daemon.py +++ b/Products/ZenCollector/daemon.py @@ -656,7 +656,7 @@ def _singleDeviceConfigCallback(self, new, updated, removed): ( cfg for cfg in itertools.chain(new, updated) - if self.options.device == cfg.configId + if self.options.device == cfg.id ), None, ) From ca84e46cff27fae46c3b63e72ca2bb7998981d67 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Tue, 7 May 2024 15:35:17 -0500 Subject: [PATCH 135/147] ensure old-style jobs are registered with zenjobs. ZEN-33726 --- Products/Jobber/config.py | 2 ++ Products/ZenModel/IpNetwork.py | 2 +- Products/ZenModel/ZDeviceLoader.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Products/Jobber/config.py b/Products/Jobber/config.py index 7935c57256..d4da489d6b 100644 --- a/Products/Jobber/config.py +++ b/Products/Jobber/config.py @@ -119,6 +119,8 @@ class CeleryConfig(object): default=[ "Products.Jobber.jobs", "Products.ZenCollector.configcache.task", + "Products.ZenModel.IpNetwork", # ensure task is registered + "Products.ZenModel.ZDeviceLoader", # ensure task is registered ] ) task_routes = attr.ib( diff --git a/Products/ZenModel/IpNetwork.py b/Products/ZenModel/IpNetwork.py index 802d9153a7..6be2d2ff45 100644 --- a/Products/ZenModel/IpNetwork.py +++ b/Products/ZenModel/IpNetwork.py @@ -33,6 +33,7 @@ from Products.ZenModel.ZenossSecurity import * from Products.ZenModel.interfaces import IObjectEventsSubscriber +from Products.Jobber.zenjobs import app from Products.ZenUtils.IpUtil import * from Products.ZenRelations.RelSchema import * from IpAddress import IpAddress @@ -968,7 +969,6 @@ def _run(self, nets=(), ranges=(), zProperties={}, collector='localhost'): SubprocessJob._run(self, cmd) -from Products.Jobber.zenjobs import app app.register_task(AutoDiscoveryJob) class IpNetworkPrinter(object): diff --git a/Products/ZenModel/ZDeviceLoader.py b/Products/ZenModel/ZDeviceLoader.py index 04092d756b..b5839c339b 100644 --- a/Products/ZenModel/ZDeviceLoader.py +++ b/Products/ZenModel/ZDeviceLoader.py @@ -29,6 +29,7 @@ from DateTime import DateTime from OFS.SimpleItem import SimpleItem +from Products.Jobber.zenjobs import app from Products.ZenUtils.Utils import isXmlRpc, setupLoggingHeader from Products.ZenUtils.Utils import clearWebLoggingStream from Products.ZenUtils.IpUtil import getHostByName, ipwrap @@ -275,7 +276,6 @@ def setCustomProperty(self, dev, cProperty, value): return dev.setZenProperty(cProperty, value) -from Products.Jobber.zenjobs import app app.register_task(CreateDeviceJob) # alias the DeviceCreationJob so zenpacks don't break DeviceCreationJob = CreateDeviceJob From c1cb1fc89cb528cf2c88ba543998ec78a00be74a Mon Sep 17 00:00:00 2001 From: azam Date: Wed, 15 May 2024 12:18:30 +0300 Subject: [PATCH 136/147] ZEN-34587: ZEN-34454 potentially causes silent zenping failure --- Products/ZenHub/services/PingPerformanceConfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Products/ZenHub/services/PingPerformanceConfig.py b/Products/ZenHub/services/PingPerformanceConfig.py index 1740580013..0d18fde5f3 100644 --- a/Products/ZenHub/services/PingPerformanceConfig.py +++ b/Products/ZenHub/services/PingPerformanceConfig.py @@ -42,6 +42,7 @@ def __init__( ds=None, perfServer="localhost", metadata=None, + context=None, ): self.ip = ipunwrap(ip) self.ipVersion = ipVersion @@ -70,6 +71,7 @@ def __init__( dp.rrdmin, dp.rrdmax, metadata, + dp.getTags(context), ) self.points.append(ipdData) @@ -144,6 +146,7 @@ def _getComponentConfig(self, iface, perfServer, monitoredIps): ds=dsList[0], perfServer=perfServer, metadata=metadata, + context=iface ) monitoredIps.append(ipProxy) @@ -173,6 +176,7 @@ def _addManageIp(self, device, perfServer, proxy): ds=dsList[0], perfServer=perfServer, metadata=metadata, + context=device ) proxy.monitoredIps.append(ipProxy) addedIp = True From 9d09d5d7753ed8938d62b00e29f223fc39e519fd Mon Sep 17 00:00:00 2001 From: azam Date: Wed, 15 May 2024 13:01:43 +0300 Subject: [PATCH 137/147] ZEN-34861: Time is shown 1 hour back if use America/Mexico_City time zone in Maitenance Window --- Products/ZenModel/MaintenanceWindow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenModel/MaintenanceWindow.py b/Products/ZenModel/MaintenanceWindow.py index dba95f7588..afe09c35b5 100644 --- a/Products/ZenModel/MaintenanceWindow.py +++ b/Products/ZenModel/MaintenanceWindow.py @@ -176,7 +176,7 @@ def niceStartDate(self): def niceStartDateTime(self): "Return start time as a string with nice sort qualities" - return "%s %s" % (Time.convertTimestampToTimeZone(self.start, self.timezone), self.timezone) + return "%s %s" % (datetime.fromtimestamp(self.start, self.tzInstance).strftime("%Y/%m/%d %H:%M:%S"), self.timezone) def niceStartProductionState(self): "Return a string version of the startProductionState" From 511c35baea0d07b38b4f6ab99d12ec5588388daa Mon Sep 17 00:00:00 2001 From: azam Date: Wed, 15 May 2024 13:17:50 +0300 Subject: [PATCH 138/147] ZEN-34862: Resequencing mappings inconsistent do to UI sorting and bad methodology --- .../resources/js/zenoss/eventclasses/ClassPanels.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Products/ZenUI3/browser/resources/js/zenoss/eventclasses/ClassPanels.js b/Products/ZenUI3/browser/resources/js/zenoss/eventclasses/ClassPanels.js index 94100d96ac..90110979dc 100644 --- a/Products/ZenUI3/browser/resources/js/zenoss/eventclasses/ClassPanels.js +++ b/Products/ZenUI3/browser/resources/js/zenoss/eventclasses/ClassPanels.js @@ -29,7 +29,9 @@ Ext.onReady(function(){ config = config || {}; Ext.applyIf(config, { model: 'Zenoss.sequencegrid.Model', - initialSortColumn: "id", + remoteSort: true, + initialSortColumn: 'sequence', + lockedOrder: 'asc', directFn: Zenoss.remote.EventClassesRouter.getSequence, root: 'data' }); @@ -98,17 +100,20 @@ Ext.onReady(function(){ } return value; }, - flex: 1 + flex: 1, + sortable: false },{ header: _t("Event Class"), id: 'class_seq_id', dataIndex: 'eventClass', - width: 200 + width: 200, + sortable: false },{ header: _t("EventClass Key"), id: 'key_seq_id', dataIndex: 'eventClassKey', - flex: 1 + flex: 1, + sortable: false },{ header: _t("Evaluation"), id: 'eval_seq_id', From 524fc8d0ace2e06ba60c04061ae18e48fef03b02 Mon Sep 17 00:00:00 2001 From: azam Date: Wed, 15 May 2024 15:59:18 +0300 Subject: [PATCH 139/147] ZEN-34863: DeviceRouter remodel method returning Unauthorized exception when using the /zport/dmd/device_router endpoint instead of device object path in url --- Products/Zuul/routers/device.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Products/Zuul/routers/device.py b/Products/Zuul/routers/device.py index 4e74ad88b4..ea01c3c96f 100644 --- a/Products/Zuul/routers/device.py +++ b/Products/Zuul/routers/device.py @@ -1651,13 +1651,17 @@ def addDevice(self, deviceName, deviceClass, title=None, audit('UI.Device.Add', deviceUid, data_=auditData) return DirectResponse.succeed(new_jobs=Zuul.marshal(jobrecords, keys=('uuid', 'description'))) - @require('Manage Device') + def remodel_device_permissions(self, deviceUid, collectPlugins='', background=True): + ctx = self.context if deviceUid is None else self._getFacade()._getObject(deviceUid) + return Zuul.checkPermission(ZEN_MANAGE_DEVICE, ctx) + + @require(remodel_device_permissions) def remodel(self, deviceUid, collectPlugins='', background=True): """ Submit a job to have a device remodeled. @type deviceUid: string - @param deviceUid: Device uid to have local template + @param deviceUid: Device uid to remodel @type collectPlugins: string @param collectPlugins: (optional) Modeler plugins to use. Takes a regular expression (default: '') From 27e0dd22c99f173d0bc178364952ed8e0da26321 Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Thu, 16 May 2024 12:39:00 +0300 Subject: [PATCH 140/147] ZEN-34864:ZEN-32760 Fix Event Archives pop-up view permissions (#4443) Fixes ZEN-34864. --- Products/ZenUI3/browser/eventconsole/configure.zcml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenUI3/browser/eventconsole/configure.zcml b/Products/ZenUI3/browser/eventconsole/configure.zcml index 9dae600b2b..368d42d5ac 100644 --- a/Products/ZenUI3/browser/eventconsole/configure.zcml +++ b/Products/ZenUI3/browser/eventconsole/configure.zcml @@ -64,7 +64,7 @@ name="viewHistoryDetail" for="Products.ZenModel.EventView.IEventView" template="historydetail.pt" - permission="zenoss.View" + permission="zenoss.Common" /> Date: Thu, 16 May 2024 14:33:09 +0300 Subject: [PATCH 141/147] ZEN-34866:ZEN-33637 Fix the confirmation dialog for device deletion (#4444) Fixes ZEN-34866. --- Products/ZenUI3/browser/resources/js/zenoss/itinfrastructure.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenUI3/browser/resources/js/zenoss/itinfrastructure.js b/Products/ZenUI3/browser/resources/js/zenoss/itinfrastructure.js index 7d59182e2f..b1edc4fb3b 100644 --- a/Products/ZenUI3/browser/resources/js/zenoss/itinfrastructure.js +++ b/Products/ZenUI3/browser/resources/js/zenoss/itinfrastructure.js @@ -378,7 +378,7 @@ Ext.onReady(function () { permission: 'Delete Device', handler: function () { var selnode = getSelectionModel().getSelectedNode(), - isclass = Zenoss.types.type(selnode.data.uid) === 'DeviceClass', + isclass = selnode.data.uid.includes('Device'), grpText = selnode.data.text.text; var win = new Zenoss.FormDialog({ title: _t('Remove Devices'), From cdfc4452261484a25ced8f7127cbdaea7f370b05 Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Wed, 22 May 2024 10:47:23 +0300 Subject: [PATCH 142/147] ZEN-34882:ZEN-33814 ComponentGraphs fails to load high component count (#4447) Fixes ZEN-34882. --- .../js/zenoss/form/ComponentGraphPanel.js | 61 ++++++++++++++++--- Products/Zuul/facades/devicefacade.py | 7 ++- Products/Zuul/infos/metricserver.py | 5 +- Products/Zuul/routers/device.py | 10 ++- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/Products/ZenUI3/browser/resources/js/zenoss/form/ComponentGraphPanel.js b/Products/ZenUI3/browser/resources/js/zenoss/form/ComponentGraphPanel.js index 1c71c2e43e..18ce3d4306 100644 --- a/Products/ZenUI3/browser/resources/js/zenoss/form/ComponentGraphPanel.js +++ b/Products/ZenUI3/browser/resources/js/zenoss/form/ComponentGraphPanel.js @@ -73,6 +73,13 @@ '1y-ago': 31536000000 }; + /* + * If a given request is over GRAPHPAGESIZE then + * the results will be paginated. + * Lower the number of graphs that are displayed for IE + * since it dramatically speeds up the rendering speed. + **/ + GRAPHPAGESIZE = Ext.isIE ? 25 : 50; /** * An example of using a custom renderer to show stacked graphs * for processes if you are viewing the memory or cpu and @@ -290,17 +297,29 @@ scope: me, select: me.onSelectGraph } + },{ + xtype: 'textfield', + name: 'graphsOnSame', + ref: '../graphsOnSame', + fieldLabel: 'Amount', + allowBlank: false, + width: 90, + value: 50, + labelSeparator: "", + labelWidth: 60, },{ xtype: 'checkbox', baseCls: 'zencheckbox_allonsame', - boxLabel: _t('All on same graph'), + boxLabel: _t('on same graph'), boxLabelAlign: 'before', labelAlign: 'right', margin: '0 10 0 20', ref: '../allOnSame', listeners: { - change: me.updateGraphs, - scope: me + change: function () { + me.updateGraphs(0) + }, + scope: me } },{ xtype: 'button', @@ -431,6 +450,8 @@ onSelectComponentType: function (combo, selected) { this.compType = selected[0].get('value'); var store, i, graphIds = this.componentGraphs[this.compType], data = []; + // set lastShown 0 to reset pagination limits + this.lastShown = 0; for (i = 0; i < graphIds.length; i++) { data.push([ graphIds[i] @@ -457,19 +478,31 @@ // go to the server and return a list of graph configs // from which we can create EuropaGraphs from var graphId = selected[0].get('name'); + this.lastShown = 0; this.graphId = graphId; this.updateGraphs(); }, - updateGraphs: function () { + updateGraphs: function (lastShown) { var meta_type = this.compType, uid = this.uid, - graphId = this.graphId, allOnSame = this.allOnSame.checked; + graphId = this.graphId, allOnSame = this.allOnSame.checked, + graphsOnSame = parseInt(this.graphsOnSame.getValue()), + me = this, + start = lastShown === undefined ? this.lastShown : lastShown, + end = start + GRAPHPAGESIZE; + if (isNaN(graphsOnSame)){ + // set to default value + graphsOnSame = 50 + } + if (graphId !== undefined) { Zenoss.remote.DeviceRouter.getComponentGraphs({ uid: uid, meta_type: meta_type, graphId: graphId, - allOnSame: allOnSame + graphsOnSame: graphsOnSame, + allOnSame: allOnSame, + limit: {'start': start, 'end': end}, }, function (response) { if (response.success) { var graphs = [], fn; @@ -503,6 +536,9 @@ }); } + // set up for the next page + this.lastShown = end; + var gp = { 'drange': this.rangeToMilliseconds(this.drange), 'end': this.end.valueOf(), @@ -517,7 +553,18 @@ graphs[i].aggregationText = this.aggregationMenu.getText(); grCols[c].items.push(graphs[i]); } - + // if we have more to show, add a button + if ((response.data_length - end) > 0) { + grCols[c].items.push({ + xtype: 'button', + text: _t('Show more results...'), + handler: function(t) { + t.hide(); + // will show the next page by looking at this.lastShown + me.updateGraphs() + } + }) + } this.add(grCols); } }, this); diff --git a/Products/Zuul/facades/devicefacade.py b/Products/Zuul/facades/devicefacade.py index 7dceab75e6..2dc6de90d1 100644 --- a/Products/Zuul/facades/devicefacade.py +++ b/Products/Zuul/facades/devicefacade.py @@ -1024,7 +1024,7 @@ def getGraphDefinitionsForComponent(self, uid): graphDefs[component.meta_type] = current_def return graphDefs - def getComponentGraphs(self, uid, meta_type, graphId, allOnSame=False): + def getComponentGraphs(self, uid, meta_type, graphId, limit, graphsOnSame, allOnSame=False): obj = self._getObject(uid) # get the components we are rendering graphs for @@ -1049,7 +1049,7 @@ def getComponentGraphs(self, uid, meta_type, graphId, allOnSame=False): return [] if allOnSame: - return [MultiContextMetricServiceGraphDefinition(graphDefault, components)] + return [MultiContextMetricServiceGraphDefinition(graphDefault, components, graphsOnSame)] graphs = [] for comp in components: @@ -1057,7 +1057,8 @@ def getComponentGraphs(self, uid, meta_type, graphId, allOnSame=False): if graph: info = getMultiAdapter((graph, comp), IMetricServiceGraphDefinition) graphs.append(info) - return graphs + + return {"data": graphs[limit['start']:limit['end']], "data_length": len(graphs)} def getDevTypes(self, uid): """ diff --git a/Products/Zuul/infos/metricserver.py b/Products/Zuul/infos/metricserver.py index 060974c4c5..c03b40b897 100644 --- a/Products/Zuul/infos/metricserver.py +++ b/Products/Zuul/infos/metricserver.py @@ -32,10 +32,11 @@ class MetricServiceGraph(HasUuidInfoMixin): - def __init__(self, graph, context): + def __init__(self, graph, context, graphsOnSame=None): self._object = graph self._context = context self._showContextTitle = False + self._graphsOnSame = graphsOnSame class MetricServiceGraphDefinition(MetricServiceGraph): @@ -401,6 +402,8 @@ def _getGraphPoints(self, klass): self._updateRPNForMultiContext(infos, knownDatapointNames) + if self._graphsOnSame: + return infos[:self._graphsOnSame] return infos def _updateRPNForMultiContext(self, infos, knownDatapointNames): diff --git a/Products/Zuul/routers/device.py b/Products/Zuul/routers/device.py index ea01c3c96f..90429b98d9 100644 --- a/Products/Zuul/routers/device.py +++ b/Products/Zuul/routers/device.py @@ -1998,14 +1998,18 @@ def getGraphDefintionsForComponents(self, uid): data = facade.getGraphDefinitionsForComponent(uid) return DirectResponse.succeed(data=Zuul.marshal(data)) - def getComponentGraphs(self, uid, meta_type, graphId, allOnSame=False): + def getComponentGraphs(self, uid, meta_type, graphId, limit, graphsOnSame, allOnSame=False): """ Returns the graph denoted by graphId for every component in device (uid) with the meta_type meta_type """ + data_length = 0 facade = self._getFacade() - data = facade.getComponentGraphs(uid, meta_type, graphId, allOnSame=allOnSame) - return DirectResponse.succeed(data=Zuul.marshal(data)) + data = facade.getComponentGraphs(uid, meta_type, graphId, limit, graphsOnSame, allOnSame=allOnSame) + if type(data) is dict: + data_length = data['data_length'] + data = data['data'] + return DirectResponse.succeed(data=Zuul.marshal(data), data_length=data_length) def getDevTypes(self, uid, filter=None): """ From 40b6fb668183cc61610bd98ac3030a1b2c011c79 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 22 May 2024 14:59:15 -0500 Subject: [PATCH 143/147] Use redis' 'count' argument to reduce load. Setting the 'count' argument on calls to the redis' `scan_iter` method controls how many queries are made to Redis. The way Redis is designed, fewer calls with more data returned per call usually results in better performance for all users of Redis. ZEN-34539 --- Products/Jobber/storage.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/Products/Jobber/storage.py b/Products/Jobber/storage.py index e620cf5a37..aeeb03df43 100644 --- a/Products/Jobber/storage.py +++ b/Products/Jobber/storage.py @@ -148,6 +148,7 @@ def __init__(self, client, expires=None): """ self.__client = client self.__expires = expires + self.__scan_count = 1000 def search(self, **fields): """Return the job IDs for jobs matching the search criteria. @@ -192,7 +193,9 @@ def get_fields(key): return ( self.__client.hget(key, "jobid") - for key in self.__client.scan_iter(match=_keypattern) + for key in self.__client.scan_iter( + match=_keypattern, count=self.__scan_count + ) if matchers == dict(zip(field_names, get_fields(key))) ) @@ -261,7 +264,9 @@ def keys(self): """ return ( self.__client.hget(key, "jobid") - for key in self.__client.scan_iter(match=_keypattern) + for key in self.__client.scan_iter( + match=_keypattern, count=self.__scan_count + ) ) def values(self): @@ -269,7 +274,7 @@ def values(self): :rtype: Iterator[Dict[str, Union[str, float]]] """ - items = _iteritems(self.__client) + items = _iteritems(self.__client, self.__scan_count) return ( {k: Fields[k].loads(v) for k, v in fields.iteritems()} for _, fields in items @@ -280,7 +285,7 @@ def items(self): :rtype: Iterator[Tuple[str, Dict[str, Union[str, float]]]] """ - items = _iteritems(self.__client) + items = _iteritems(self.__client, self.__scan_count) return ( ( fields["jobid"], @@ -392,7 +397,12 @@ def __contains__(self, jobid): return self.__client.exists(_key(jobid)) def __len__(self): - return sum(1 for _ in self.__client.scan_iter(match=_keypattern)) + return sum( + 1 + for _ in self.__client.scan_iter( + match=_keypattern, count=self.__scan_count + ) + ) def __iter__(self): """Return an iterator producing all the job IDs in the datastore. @@ -416,12 +426,12 @@ def _key(jobid): return _keytemplate.format(jobid) -def _iteritems(client): +def _iteritems(client, count): """Return an iterable of (redis key, job data) pairs. Only (key, data) pairs where data is not None are returned. """ - keys = client.scan_iter(match=_keypattern) + keys = client.scan_iter(match=_keypattern, count=count) raw = ((key, client.hgetall(key)) for key in keys) return ((key, data) for key, data in raw if data) From 292020407ab80c8d1453eac34cb6c5c3457fd9f8 Mon Sep 17 00:00:00 2001 From: Jason Peacock Date: Wed, 22 May 2024 14:57:15 -0500 Subject: [PATCH 144/147] Add 'stats' subcommand to configcache command. The 'stats' subcommand will return a report on different statistics regarding the current state of the configuration cache. The set of statistics include count, average age, median age, minimum age, and maximum age. These statistics are grouped by service class, collector (monitor), status, and device. ZEN-34892 --- .../ZenCollector/configcache/cache/storage.py | 71 ++++-- .../ZenCollector/configcache/cli/__init__.py | 3 +- .../ZenCollector/configcache/cli/_groups.py | 231 ++++++++++++++++++ .../ZenCollector/configcache/cli/_json.py | 83 +++++++ .../ZenCollector/configcache/cli/_stats.py | 181 ++++++++++++++ .../ZenCollector/configcache/cli/_tables.py | 124 ++++++++++ Products/ZenCollector/configcache/cli/list.py | 94 +++---- .../ZenCollector/configcache/cli/stats.py | 158 ++++++++++++ .../ZenCollector/configcache/cli/stats.zcml | 13 + .../ZenCollector/configcache/configcache.py | 7 +- 10 files changed, 896 insertions(+), 69 deletions(-) create mode 100644 Products/ZenCollector/configcache/cli/_groups.py create mode 100644 Products/ZenCollector/configcache/cli/_json.py create mode 100644 Products/ZenCollector/configcache/cli/_stats.py create mode 100644 Products/ZenCollector/configcache/cli/_tables.py create mode 100644 Products/ZenCollector/configcache/cli/stats.py create mode 100644 Products/ZenCollector/configcache/cli/stats.zcml diff --git a/Products/ZenCollector/configcache/cache/storage.py b/Products/ZenCollector/configcache/cache/storage.py index 29e1bb3302..10d4363964 100644 --- a/Products/ZenCollector/configcache/cache/storage.py +++ b/Products/ZenCollector/configcache/cache/storage.py @@ -9,13 +9,13 @@ # Key structure # ============= -# modelchange:device:uid: -# modelchange:device:config::: -# modelchange:device:age:: [(, ), ...] -# modelchange:device:retired:: [(, ), ...] -# modelchange:device:expired:: [(, ), ...] -# modelchange:device:pending:: [(, ), ...] -# modelchange:device:building:: [(, ), ...] +# configcache:device:uid: +# configcache:device:config::: +# configcache:device:age:: [(, ), ...] +# configcache:device:retired:: [(, ), ...] +# configcache:device:expired:: [(, ), ...] +# configcache:device:pending:: [(, ), ...] +# configcache:device:building:: [(, ), ...] # # While "device" seems redundant, other values in this position could be # "threshold" and "property". @@ -62,7 +62,6 @@ import ast import json import logging -import operator import re from functools import partial @@ -137,6 +136,17 @@ def __contains__(self, key): self.__client, key.service, key.monitor, key.device ) + def __iter__(self): + """ + Returns an iterable over the known keys. + + @rtype: Iterator[CacheKey] + """ + return iter( + CacheKey(service, monitor, device) + for service, monitor, device in self.__config.scan(self.__client) + ) + def search(self, query=CacheQuery()): """ Returns the configuration keys matching the search criteria. @@ -206,6 +216,7 @@ def get_updated(self, key): Return the timestamp of when the config was built. @type key: CacheKey + @rtype: float """ return _to_ts( self.__age.score( @@ -213,6 +224,23 @@ def get_updated(self, key): ) ) + def query_updated(self, query=CacheQuery()): + """ + Return the last update timestamp of every configuration selected + by the query. + + @type query: CacheQuery + @rtype: Iterable[Tuple[CacheKey, float]] + """ + predicate = self._get_device_predicate(query.device) + return ( + (key, ts) + for key, ts in self._get_metadata( + self.__age, query.service, query.monitor + ) + if predicate(key.device) + ) + def get(self, key, default=None): """ @type key: CacheKey @@ -412,21 +440,7 @@ def get_statuses(self, query=CacheQuery()): (self.__building, ConfigStatus.Building), ) - def accept_all(_): - return True - - def filter_regex(regex, value): - m = regex.match(value) - return m is not None - - if query.device == "*": - predicate = accept_all - elif "*" in query.device: - expr = query.device.replace("*", ".*") - regex = re.compile(expr) - predicate = partial(filter_regex, regex) - else: - predicate = partial(operator.eq, query.device) + predicate = self._get_device_predicate(query.device) for table, cls in tables: for key, ts in self._get_metadata( @@ -448,6 +462,17 @@ def filter_regex(regex, value): statuses.append(ConfigStatus.Current(key, uid, ts)) return statuses + def _get_device_predicate(self, spec): + + if spec == "*": + return lambda _: True + elif "*" in spec: + expr = spec.replace("*", ".*") + regex = re.compile(expr) + return lambda value: regex.match(value) is not None + else: + return lambda value: value == spec + def _get_uid(self, uids, device): uid = uids.get(device) if uid is None: diff --git a/Products/ZenCollector/configcache/cli/__init__.py b/Products/ZenCollector/configcache/cli/__init__.py index fb026aaa52..f7409ce684 100644 --- a/Products/ZenCollector/configcache/cli/__init__.py +++ b/Products/ZenCollector/configcache/cli/__init__.py @@ -14,6 +14,7 @@ from .list import List_ from .remove import Remove from .show import Show +from .stats import Stats -__all__ = ("Expire", "List_", "Remove", "Show") +__all__ = ("Expire", "List_", "Remove", "Show", "Stats") diff --git a/Products/ZenCollector/configcache/cli/_groups.py b/Products/ZenCollector/configcache/cli/_groups.py new file mode 100644 index 0000000000..7796a45db3 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/_groups.py @@ -0,0 +1,231 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import, division + +from collections import defaultdict +from itertools import chain + +import attr + +from ._stats import UniqueCountStat + + +class DeviceGroup(object): + + name = "devices" + order = 1 + + def __init__(self, stats): + # Only one row, so use summary + self._summary = tuple(s() for s in stats) + try: + # DeviceGroup doesn't want CountStat + posn = stats.index(UniqueCountStat) + except ValueError: + # Not found, so don't worry about it + self._counter = None + self._otherstats = self._summary + else: + # Found, replace it with UniqueCountStat + self._counter = self._summary[posn] + self._otherstats = self._summary[0:posn] + self._summary[posn+1:] + self._stats = stats + self._samples = 0 + + def handle_key(self, key): + if self._counter is None: + return + self._counter.mark(key.device) + self._samples += 1 + + def handle_timestamp(self, key, ts): + for stat in self._otherstats: + stat.mark(ts) + self._samples += 1 + + def handle_status(self, status): + pass + + def headings(self): + return [s.name for s in self._stats] + + def hints(self): + return [s.type_ for s in self._stats] + + def rows(self): + return [] + + def summary(self): + if self._samples == 0: + return [] + return list(s.value() for s in self._summary) + + +class ServiceGroup(object): + + name = "services" + order = 2 + + def __init__(self, stats): + self._stats = stats + self._byrow = defaultdict(self._makerowvalue) + self._summary = tuple(s() for s in stats) + self._samples = 0 + + def _makerowvalue(self): + return tuple(stat() for stat in self._stats) + + def handle_key(self, key): + pass + + def handle_timestamp(self, key, ts): + for stat in self._byrow[key.service]: + stat.mark(ts) + for stat in self._summary: + stat.mark(ts) + self._samples += 1 + + def handle_status(self, status): + pass + + def headings(self): + headings = ["configuration service class"] + headings.extend(s.name for s in self._stats) + return headings + + def hints(self): + hints = ["str"] + hints.extend(s.type_ for s in self._stats) + return hints + + def rows(self): + if self._samples == 0: + return [] + return ( + self._makerow(svcname, stats) + for svcname, stats in self._byrow.iteritems() + ) + + def _makerow(self, svcname, stats): + return tuple(chain((svcname,), (s.value() for s in stats))) + + def summary(self): + if self._samples == 0: + return [] + return list(s.value() for s in self._summary) + + +class MonitorGroup(object): + + name = "monitors" + order = 3 + + def __init__(self, stats): + self._stats = stats + self._byrow = defaultdict(self._makerowvalue) + self._summary = tuple(s() for s in stats) + self._samples = 0 + + def _makerowvalue(self): + return tuple(stat() for stat in self._stats) + + def handle_key(self, key): + pass + + def handle_timestamp(self, key, ts): + for stat in self._byrow[key.monitor]: + stat.mark(ts) + for stat in self._summary: + stat.mark(ts) + self._samples += 1 + + def handle_status(self, status): + pass + + def headings(self): + headings = ["collector"] + headings.extend(s.name for s in self._stats) + return headings + + def hints(self): + hints = ["str"] + hints.extend(s.type_ for s in self._stats) + return hints + + def rows(self): + if self._samples == 0: + return [] + return ( + self._makerow(name, stats) + for name, stats in self._byrow.iteritems() + ) + + def _makerow(self, name, stats): + return tuple(chain((name,), (s.value() for s in stats))) + + def summary(self): + if self._samples == 0: + return [] + return list(s.value() for s in self._summary) + + +class StatusGroup(object): + + name = "statuses" + order = 4 + + def __init__(self, stats): + self._stats = stats + self._byrow = defaultdict(self._makerowvalue) + self._summary = tuple(s() for s in stats) + self._samples = 0 + + def _makerowvalue(self): + return tuple(stat() for stat in self._stats) + + def handle_key(self, key): + pass + + def handle_timestamp(self, key, ts): + pass + + def handle_status(self, status): + data = attr.astuple(status) + for stat in self._byrow[type(status).__name__]: + stat.mark(data[-1]) + for stat in self._summary: + stat.mark(data[-1]) + self._samples += 1 + + def headings(self): + headings = ["status"] + headings.extend(s.name for s in self._stats) + return headings + + def hints(self): + hints = ["str"] + hints.extend(s.type_ for s in self._stats) + return hints + + def rows(self): + if self._samples == 0: + return [] + return ( + self._makerow(name, stats) + for name, stats in self._byrow.iteritems() + ) + + def _makerow(self, name, stats): + return tuple(chain((name,), (s.value() for s in stats))) + + def summary(self): + if self._samples == 0: + return [] + return list(s.value() for s in self._summary) diff --git a/Products/ZenCollector/configcache/cli/_json.py b/Products/ZenCollector/configcache/cli/_json.py new file mode 100644 index 0000000000..01c9f0404e --- /dev/null +++ b/Products/ZenCollector/configcache/cli/_json.py @@ -0,0 +1,83 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import, division + +import json + + +class JSONOutput(object): + """ + { + "devices": [ + "summary" : { + "number_of_devices": 4, + ... + } + ], + "services": { + "data": [ + {: , ... }, ... + ], + "summary": { + : , # except first column + ... + } + }, + "monitors": { + "data": [ + {: , ... }, ... + ], + "summary": { + : , # except first column + ... + } + }, + "statuses": { + "data": [ + {: , ... }, ... + ], + "summary": { + : , # except first column + ... + } + } + } + """ + + def write(self, *groups): + result = {} + for group in groups: + rows = list(group.rows()) + summary = group.summary() + headings = [ + hdr.replace(" ", "_").lower() for hdr in group.headings() + ] + + if len(rows) == 0 and len(summary) == 0: + continue + + if len(headings) == 1 and len(rows) == 1: + result[group.name] = [ + {headings[0].replace(" ", "_").lower(): rows[0][0]} + ] + continue + + rows = [ + {hdr: value for hdr, value in zip(headings, row)} + for row in rows + ] + if len(rows) == 0: + summary = {hdr: value for hdr, value in zip(headings, summary)} + else: + summary = { + hdr: value for hdr, value in zip(headings[1:], summary) + } + result[group.name] = {"data": rows, "summary": summary} + print(json.dumps(result)) diff --git a/Products/ZenCollector/configcache/cli/_stats.py b/Products/ZenCollector/configcache/cli/_stats.py new file mode 100644 index 0000000000..e2f0f2aa52 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/_stats.py @@ -0,0 +1,181 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import, division + +import sys +import time + +_current_time = None + + +def _current_time_unset(): + global _current_time + _current_time = time.time() + try: + return _current_time + finally: + global _get_current_time + _get_current_time = _current_time_set + + +def _current_time_set(): + global _current_time + return _current_time + + +_get_current_time = _current_time_unset + + +class CountStat(object): + + name = "count" + type_ = "int" + + def __init__(self): + self._count = 0 + + def mark(self, *args): + self._count += 1 + + def value(self): + return self._count + + +class UniqueCountStat(CountStat): + + name = "count of devices" + + def __init__(self): + self._values = set() + + def mark(self, value): + self._values.add(value) + + def value(self): + return len(self._values) + + +class AverageStat(object): + + name = "average" + type_ = "timedelta" + + def __init__(self): + self._total = 0 + self._count = 0 + + def mark(self, value): + self._count += 1 + self._total += value + + def value(self): + if self._count == 0: + return 0 + return self._total / self._count + + +class AverageAgeStat(AverageStat): + + name = "average age" + + def value(self): + avg = super(AverageAgeStat, self).value() + if avg == 0: + return 0 + return _get_current_time() - avg + + +class MedianStat(object): + + name = "median" + type_ = "timedelta" + + def __init__(self): + self._min = sys.maxsize + self._max = 0 + + def mark(self, value): + value = int(value) + self._min = min(self._min, value) + self._max = max(self._max, value) + + def value(self): + if self._min == sys.maxsize: + return 0 + return (self._min + self._max) / 2 + + +class MedianAgeStat(MedianStat): + + name = "median age" + + def value(self): + median = super(MedianAgeStat, self).value() + if median == 0: + return 0 + return _get_current_time() - median + + +class MinStat(object): + + name = "min" + type_ = "float" + + def __init__(self): + self._min = sys.maxsize + + def mark(self, value): + self._min = min(self._min, int(value)) + + def value(self): + if self._min == sys.maxsize: + return 0 + return self._min + + +class MaxAgeStat(MinStat): + + name = "max age" + type_ = "timedelta" + + def value(self): + maxv = super(MaxAgeStat, self).value() + if maxv == 0: + return 0 + return _get_current_time() - maxv + + +class MaxStat(object): + + name = "max" + type_ = "float" + + def __init__(self): + self._max = 0 + + def mark(self, value): + self._max = max(self._max, int(value)) + + def value(self): + if self._max == 0: + return 0 + return self._max + + +class MinAgeStat(MaxStat): + + name = "min age" + type_ = "timedelta" + + def value(self): + minv = super(MinAgeStat, self).value() + if minv == 0: + return 0 + return _get_current_time() - minv diff --git a/Products/ZenCollector/configcache/cli/_tables.py b/Products/ZenCollector/configcache/cli/_tables.py new file mode 100644 index 0000000000..83675fd7a5 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/_tables.py @@ -0,0 +1,124 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import, division + +import datetime + +from itertools import chain + + +class TablesOutput(object): + + def write(self, *groups): + for group in groups: + self._display( + list(group.rows()), + group.summary(), + group.headings(), + group.hints(), + ) + + def _display(self, rows, summary, headings, hints): + if not rows and not summary: + return + + # Transform row values for presentation + if rows: + rows = [ + tuple(_xform(value, hint) for value, hint in zip(row, hints)) + for row in sorted(rows, key=lambda x: x[0]) + ] + + # Transform total values for presentation + if summary: + if rows: + summary = tuple( + _xform(v, h) for v, h in zip([""] + summary, hints) + ) + else: + summary = tuple( + _xform(v, h) for v, h in zip(summary, hints) + ) + + # Transform column headers for presentation + if summary and not rows: + headings = [hdr.capitalize() for hdr in headings] + else: + headings = [hdr.upper() for hdr in headings] + + # Initialize maxwidth values for each column + maxwidths = [0 for _ in headings] + + if summary and not rows: + hdrmaxw = max(len(hdr) for hdr in headings) + maxwidths = [hdrmaxw] * len(headings) + else: + for row in rows: + for idx, (mw, col) in enumerate(zip(maxwidths, row)): + maxwidths[idx] = max(mw, len(str(col))) + for idx, (mw, hd) in enumerate(zip(maxwidths, headings)): + maxwidths[idx] = max(mw, len(hd)) + for idx, (mw, tv) in enumerate(zip(maxwidths[1:], summary)): + maxwidths[idx + 1] = max(mw, len(str(tv))) + + offset = len(maxwidths) + tmplt = " ".join( + "{{{0}:{{{1}}}}}".format(idx, idx + offset) + for idx in range(0, offset) + ) + fmtspecs = [ + _get_fmt_spec(mw, hint) for mw, hint in zip(maxwidths, hints) + ] + print() + if summary and not rows: + for hdr, value in zip(headings, summary): + print("{0:{2}}: {1}".format(hdr, value, maxwidths[0])) + else: + if headings: + print(tmplt.format(*chain(headings, fmtspecs))) + sep = ["-" * c for c in maxwidths] + print(tmplt.format(*chain(sep, maxwidths))) + + for row in rows: + print(tmplt.format(*chain(row, fmtspecs))) + + if summary: + print(tmplt.format(*chain(sep, maxwidths))) + print(tmplt.format(*chain(summary, fmtspecs))) + + +def _xform(value, hint): + if hint == "timedelta": + td = datetime.timedelta(seconds=value) + hours = td.seconds // 3600 + minutes = (td.seconds - (hours * 3600)) // 60 + seconds = td.seconds - (hours * 3600) - (minutes * 60) + return "{0} {1:02}:{2:02}:{3:02}".format( + ( + "" + if td.days == 0 + else "{} day{}".format(td.days, "" if td.days == 1 else "s") + ), + hours, + minutes, + seconds, + ).strip() + else: + return value + + +def _get_fmt_spec(mw, hint): + if hint == "int": + return ">{}".format(mw) + elif hint == "timedelta": + return ">{}".format(mw) + elif hint == "float": + return ">{}.2f".format(mw) + return mw diff --git a/Products/ZenCollector/configcache/cli/list.py b/Products/ZenCollector/configcache/cli/list.py index d58d6608e4..14d2f488e1 100644 --- a/Products/ZenCollector/configcache/cli/list.py +++ b/Products/ZenCollector/configcache/cli/list.py @@ -11,8 +11,12 @@ import argparse import sys +import time -from datetime import datetime +from datetime import datetime, timedelta +from itertools import chain + +import attr from zope.component import createObject @@ -103,12 +107,11 @@ def run(self): ) if len(self._devices) > 1: data = ( - status - for status in data - if status.key.device in self._devices + status for status in data if status.key.device in self._devices ) rows = [] - maxd, maxs, maxm = 1, 1, 1 + maxd, maxs, maxt, maxa, maxm = 1, 1, 1, 1, 1 + now = time.time() for status in sorted( data, key=lambda x: (x.key.device, x.key.service) ): @@ -117,36 +120,39 @@ def run(self): else: devid = status.key.device status_text = _format_status(status) + ts = attr.astuple(status)[-1] + ts_text = _format_date(ts) + age_text = _format_timedelta(now - ts) maxd = max(maxd, len(devid)) maxs = max(maxs, len(status_text)) + maxt = max(maxt, len(ts_text)) + maxa = max(maxa, len(age_text)) maxm = max(maxm, len(status.key.monitor)) rows.append( - (devid, status_text, status.key.monitor, status.key.service) - ) - if rows: - print( - "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( - "DEVICE", - "STATUS", - "COLLECTOR", - "SERVICE", - maxd=maxd, - maxs=maxs, - maxm=maxm, + ( + devid, + status_text, + ts_text, + age_text, + status.key.monitor, + status.key.service, ) ) + hdr_tmplt = "{0:{6}} {1:{7}} {2:^{8}} {3:^{9}} {4:{10}} {5}" + row_tmplt = "{0:{6}} {1:{7}} {2:{8}} {3:>{9}} {4:{10}} {5}" + headings = ( + "DEVICE", + "STATUS", + "LAST CHANGE", + "AGE", + "COLLECTOR", + "SERVICE", + ) + widths = (maxd, maxs, maxt, maxa, maxm) + if rows: + print(hdr_tmplt.format(*chain(headings, widths))) for row in rows: - print( - "{0:{maxd}} {1:{maxs}} {2:{maxm}} {3}".format( - row[0], - row[1], - row[2], - row[3], - maxd=maxd, - maxs=maxs, - maxm=maxm, - ) - ) + print(row_tmplt.format(*chain(row, widths))) _name_state_lookup = { @@ -158,21 +164,25 @@ def run(self): } +def _format_timedelta(value): + td = timedelta(seconds=value) + hours = td.seconds // 3600 + minutes = (td.seconds - (hours * 3600)) // 60 + seconds = td.seconds - (hours * 3600) - (minutes * 60) + return "{0} {1:02}:{2:02}:{3:02}".format( + ( + "" + if td.days == 0 + else "{} day{}".format(td.days, "" if td.days == 1 else "s") + ), + hours, + minutes, + seconds, + ).strip() + + def _format_status(status): - if isinstance(status, ConfigStatus.Current): - return "current since {}".format(_format_date(status.updated)) - elif isinstance(status, ConfigStatus.Retired): - return "retired since {}".format(_format_date(status.retired)) - elif isinstance(status, ConfigStatus.Expired): - return "expired since {}".format(_format_date(status.expired)) - elif isinstance(status, ConfigStatus.Pending): - return "waiting to build since {}".format( - _format_date(status.submitted) - ) - elif isinstance(status, ConfigStatus.Building): - return "build started {}".format(_format_date(status.started)) - else: - return "????" + return type(status).__name__.lower() def _format_date(ts): diff --git a/Products/ZenCollector/configcache/cli/stats.py b/Products/ZenCollector/configcache/cli/stats.py new file mode 100644 index 0000000000..7d6fc7d91e --- /dev/null +++ b/Products/ZenCollector/configcache/cli/stats.py @@ -0,0 +1,158 @@ +############################################################################## +# +# Copyright (C) Zenoss, Inc. 2024, all rights reserved. +# +# This content is made available according to terms specified in +# License.zenoss under the directory where your Zenoss product is installed. +# +############################################################################## + +from __future__ import print_function, absolute_import, division + +import argparse +import sys + +from zope.component import createObject + +from Products.ZenUtils.RedisUtils import getRedisClient, getRedisUrl + +from ..app import initialize_environment +from ..app.args import get_subparser +from ..cache import CacheQuery + +from .args import get_common_parser, MultiChoice +from ._tables import TablesOutput +from ._json import JSONOutput +from ._stats import ( + AverageAgeStat, + CountStat, + MaxAgeStat, + MedianAgeStat, + MinAgeStat, + UniqueCountStat, +) +from ._groups import DeviceGroup, ServiceGroup, MonitorGroup, StatusGroup + + +class Stats(object): + description = "Show statistics about the configuration cache" + + configs = (("stats.zcml", __name__),) + + _groups = ("collector", "device", "service", "status") + _statistics = ("count", "avg_age", "median_age", "min_age", "max_age") + + @staticmethod + def add_arguments(parser, subparsers): + subp = get_subparser( + subparsers, "stats", Stats.description, parent=get_common_parser() + ) + subp.add_argument( + "-S", + dest="statistic", + action=MultiChoice, + choices=Stats._statistics, + default=argparse.SUPPRESS, + help="Specify the statistics to return. One or more statistics " + "may be specified (comma separated). By default, all " + "statistics are returned.", + ) + subp.add_argument( + "-G", + dest="group", + action=MultiChoice, + choices=Stats._groups, + default=argparse.SUPPRESS, + help="Specify the statistics groupings to return. One or more " + "groupings may be specified (comma separated). By default, all " + "groupings are returned.", + ) + subp.add_argument( + "-f", + dest="format", + choices=("tables", "json"), + default="tables", + help="Output statistics in the specified format", + ) + subp.set_defaults(factory=Stats) + + def __init__(self, args): + stats = [] + for statId in getattr(args, "statistic", Stats._statistics): + if statId == "count": + stats.append(CountStat) + elif statId == "avg_age": + stats.append(AverageAgeStat) + elif statId == "median_age": + stats.append(MedianAgeStat) + elif statId == "min_age": + stats.append(MinAgeStat) + elif statId == "max_age": + stats.append(MaxAgeStat) + self._groups = [] + for groupId in getattr(args, "group", Stats._groups): + if groupId == "collector": + self._groups.append(MonitorGroup(stats)) + elif groupId == "device": + try: + # DeviceGroup doesn't want CountStat + posn = stats.index(CountStat) + except ValueError: + # Not found, so don't worry about it + dg_stats = stats + pass + else: + # Found, replace it with UniqueCountStat + dg_stats = list(stats) + dg_stats[posn] = UniqueCountStat + self._groups.append(DeviceGroup(dg_stats)) + if groupId == "service": + self._groups.append(ServiceGroup(stats)) + elif groupId == "status": + self._groups.append(StatusGroup(stats)) + if args.format == "tables": + self._format = TablesOutput() + elif args.format == "json": + self._format = JSONOutput() + self._monitor = "*{}*".format(args.collector).replace("***", "*") + self._service = "*{}*".format(args.service).replace("***", "*") + self._devices = getattr(args, "device", []) + + def run(self): + haswildcard = any("*" in d for d in self._devices) + if haswildcard and len(self._devices) > 1: + print( + "Only one DEVICE argument supported when a wildcard is used.", + file=sys.stderr, + ) + return + initialize_environment(configs=self.configs, useZope=False) + client = getRedisClient(url=getRedisUrl()) + store = createObject("configcache-store", client) + + if len(self._devices) == 1: + query = CacheQuery(self._service, self._monitor, self._devices[0]) + else: + query = CacheQuery(self._service, self._monitor) + include = _get_device_predicate(self._devices) + for key, ts in store.query_updated(query): + if not include(key.device): + continue + for group in self._groups: + group.handle_key(key) + group.handle_timestamp(key, ts) + for status in store.get_statuses(query): + if not include(status.key.device): + continue + for group in self._groups: + group.handle_status(status) + + self._format.write( + *(group for group in sorted(self._groups, key=lambda x: x.order)) + ) + + +def _get_device_predicate(devices): + if len(devices) < 2: + return lambda _: True + return lambda x: next((True for d in devices if x == d), False) diff --git a/Products/ZenCollector/configcache/cli/stats.zcml b/Products/ZenCollector/configcache/cli/stats.zcml new file mode 100644 index 0000000000..8ec2993701 --- /dev/null +++ b/Products/ZenCollector/configcache/cli/stats.zcml @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/Products/ZenCollector/configcache/configcache.py b/Products/ZenCollector/configcache/configcache.py index c8e7c7bce3..8ce4764ea8 100644 --- a/Products/ZenCollector/configcache/configcache.py +++ b/Products/ZenCollector/configcache/configcache.py @@ -10,7 +10,7 @@ from __future__ import absolute_import, print_function from .app.args import get_arg_parser -from .cli import Expire, List_, Remove, Show +from .cli import Expire, List_, Remove, Show, Stats from .invalidator import Invalidator from .manager import Manager from .version import Version @@ -24,10 +24,11 @@ def main(argv=None): Version.add_arguments(parser, subparsers) Manager.add_arguments(parser, subparsers) Invalidator.add_arguments(parser, subparsers) - List_.add_arguments(parser, subparsers) - Show.add_arguments(parser, subparsers) Expire.add_arguments(parser, subparsers) + List_.add_arguments(parser, subparsers) Remove.add_arguments(parser, subparsers) + Show.add_arguments(parser, subparsers) + Stats.add_arguments(parser, subparsers) args = parser.parse_args() args.factory(args).run() From b34c94476a0be448e07b11fd24a6548c63dfe85d Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Mon, 27 May 2024 11:27:21 +0300 Subject: [PATCH 145/147] ZEN-34883:ZEN-34422 Fix maintenance window edit permissions (#4453) Fixes ZEN-34883. --- .../resources/js/zenoss/devicemanagement/Administration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Products/ZenUI3/browser/resources/js/zenoss/devicemanagement/Administration.js b/Products/ZenUI3/browser/resources/js/zenoss/devicemanagement/Administration.js index a82169b7f6..34cc8151df 100644 --- a/Products/ZenUI3/browser/resources/js/zenoss/devicemanagement/Administration.js +++ b/Products/ZenUI3/browser/resources/js/zenoss/devicemanagement/Administration.js @@ -535,7 +535,7 @@ Ext.ns('Zenoss', 'Zenoss.devicemanagement'); }; - if (Zenoss.Security.hasPermission('Manage Device')) { + if (Zenoss.Security.hasPermission('Maintenance Windows Edit')) { dialog = new Zenoss.SmartFormDialog(config); dialog.show(); }else{ return false; } From 83789deb131844b30c3b68fda90b2cddc9690f44 Mon Sep 17 00:00:00 2001 From: vsaliieva <91525276+vsaliieva@users.noreply.github.com> Date: Mon, 27 May 2024 19:17:15 +0300 Subject: [PATCH 146/147] ZEN-34885:ZEN-34809 Use latest versions of moment.js (2.30.1 and 0.5.44) (#4454) Fixes ZEN-34885. --- .../js/timezone/moment-timezone-with-data.min.js | 8 +------- .../ZenUI3/browser/resources/js/timezone/moment.min.js | 9 ++------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/Products/ZenUI3/browser/resources/js/timezone/moment-timezone-with-data.min.js b/Products/ZenUI3/browser/resources/js/timezone/moment-timezone-with-data.min.js index b2d8851ce8..540d0550a2 100644 --- a/Products/ZenUI3/browser/resources/js/timezone/moment-timezone-with-data.min.js +++ b/Products/ZenUI3/browser/resources/js/timezone/moment-timezone-with-data.min.js @@ -1,7 +1 @@ -//! moment-timezone.js -//! version : 0.4.0 -//! author : Tim Wood -//! license : MIT -//! github.com/moment/moment-timezone -!function(a,b){"use strict";"function"==typeof define&&define.amd?define(["moment"],b):"object"==typeof exports?module.exports=b(require("moment")):b(a.moment)}(this,function(a){"use strict";function b(a){return a>96?a-87:a>64?a-29:a-48}function c(a){var c,d=0,e=a.split("."),f=e[0],g=e[1]||"",h=1,i=0,j=1;for(45===a.charCodeAt(0)&&(d=1,j=-1),d;dc;c++)a[c]=Math.round((a[c-1]||0)+6e4*a[c]);a[b-1]=1/0}function f(a,b){var c,d=[];for(c=0;cz||2===z&&6>A)&&q("Moment Timezone requires Moment.js >= 2.6.0. You are using Moment.js "+a.version+". See momentjs.com"),h.prototype={_set:function(a){this.name=a.name,this.abbrs=a.abbrs,this.untils=a.untils,this.offsets=a.offsets},_index:function(a){var b,c=+a,d=this.untils;for(b=0;be;e++)if(b=g[e],c=g[e+1],d=g[e?e-1:e],c>b&&r.moveAmbiguousForward?b=c:b>d&&r.moveInvalidForward&&(b=d),fz||2===z&&9>A)&&q("Moment Timezone setDefault() requires Moment.js >= 2.9.0. You are using Moment.js "+a.version+"."),a.defaultZone=b?k(b):null,a};var C=a.momentProperties;return"[object Array]"===Object.prototype.toString.call(C)?(C.push("_z"),C.push("_a")):C&&(C._z=null),n({version:"2015d",zones:["Africa/Abidjan|LMT GMT|g.8 0|01|-2ldXH.Q","Africa/Accra|LMT GMT GHST|.Q 0 -k|012121212121212121212121212121212121212121212121|-26BbX.8 6tzX.8 MnE 1BAk MnE 1BAk MnE 1BAk MnE 1C0k MnE 1BAk MnE 1BAk MnE 1BAk MnE 1C0k MnE 1BAk MnE 1BAk MnE 1BAk MnE 1C0k MnE 1BAk MnE 1BAk MnE 1BAk MnE 1C0k MnE 1BAk MnE 1BAk MnE 1BAk MnE 1C0k MnE 1BAk MnE 1BAk MnE","Africa/Addis_Ababa|LMT EAT BEAT BEAUT|-2r.g -30 -2u -2J|01231|-1F3Cr.g 3Dzr.g okMu MFXJ","Africa/Algiers|PMT WET WEST CET CEST|-9.l 0 -10 -10 -20|0121212121212121343431312123431213|-2nco9.l cNb9.l HA0 19A0 1iM0 11c0 1oo0 Wo0 1rc0 QM0 1EM0 UM0 DA0 Imo0 rd0 De0 9Xz0 1fb0 1ap0 16K0 2yo0 mEp0 hwL0 jxA0 11A0 dDd0 17b0 11B0 1cN0 2Dy0 1cN0 1fB0 1cL0","Africa/Bangui|LMT WAT|-d.A -10|01|-22y0d.A","Africa/Bissau|LMT WAT GMT|12.k 10 0|012|-2ldWV.E 2xonV.E","Africa/Blantyre|LMT CAT|-2a.k -20|01|-2GJea.k","Africa/Cairo|EET EEST|-20 -30|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-1bIO0 vb0 1ip0 11z0 1iN0 1nz0 12p0 1pz0 10N0 1pz0 16p0 1jz0 s3d0 Vz0 1oN0 11b0 1oO0 10N0 1pz0 10N0 1pb0 10N0 1pb0 10N0 1pb0 10N0 1pz0 10N0 1pb0 10N0 1pb0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1WL0 rd0 1Rz0 wp0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1qL0 Xd0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1ny0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 WL0 1qN0 Rb0 1wp0 On0 1zd0 Lz0 1EN0 Fb0 c10 8n0 8Nd0 gL0 e10 mn0","Africa/Casablanca|LMT WET WEST CET|u.k 0 -10 -10|012121212121212121312121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2gMnt.E 130Lt.E rb0 Dd0 dVb0 b6p0 TX0 EoB0 LL0 gnd0 rz0 43d0 AL0 1Nd0 XX0 1Cp0 pz0 dEp0 4mn0 SyN0 AL0 1Nd0 wn0 1FB0 Db0 1zd0 Lz0 1Nf0 wM0 co0 go0 1o00 s00 dA0 vc0 11A0 A00 e00 y00 11A0 uo0 e00 DA0 11A0 rA0 e00 Jc0 WM0 m00 gM0 M00 WM0 jc0 e00 RA0 11A0 dA0 e00 Uo0 11A0 800 gM0 Xc0 11A0 5c0 e00 17A0 WM0 2o0 e00 1ao0 19A0 1g00 16M0 1iM0 1400 1lA0 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qo0 1200 1kM0 14M0 1i00","Africa/Ceuta|WET WEST CET CEST|0 -10 -10 -20|010101010101010101010232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-25KN0 11z0 drd0 18o0 3I00 17c0 1fA0 1a00 1io0 1a00 1y7p0 LL0 gnd0 rz0 43d0 AL0 1Nd0 XX0 1Cp0 pz0 dEp0 4VB0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Africa/El_Aaiun|LMT WAT WET WEST|Q.M 10 0 -10|0123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1rDz7.c 1GVA7.c 6L0 AL0 1Nd0 XX0 1Cp0 pz0 1cBB0 AL0 1Nd0 wn0 1FB0 Db0 1zd0 Lz0 1Nf0 wM0 co0 go0 1o00 s00 dA0 vc0 11A0 A00 e00 y00 11A0 uo0 e00 DA0 11A0 rA0 e00 Jc0 WM0 m00 gM0 M00 WM0 jc0 e00 RA0 11A0 dA0 e00 Uo0 11A0 800 gM0 Xc0 11A0 5c0 e00 17A0 WM0 2o0 e00 1ao0 19A0 1g00 16M0 1iM0 1400 1lA0 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qo0 1200 1kM0 14M0 1i00","Africa/Johannesburg|SAST SAST SAST|-1u -20 -30|012121|-2GJdu 1Ajdu 1cL0 1cN0 1cL0","Africa/Juba|LMT CAT CAST EAT|-2a.8 -20 -30 -30|01212121212121212121212121212121213|-1yW2a.8 1zK0a.8 16L0 1iN0 17b0 1jd0 17b0 1ip0 17z0 1i10 17X0 1hB0 18n0 1hd0 19b0 1gp0 19z0 1iN0 17b0 1ip0 17z0 1i10 18n0 1hd0 18L0 1gN0 19b0 1gp0 19z0 1iN0 17z0 1i10 17X0 yGd0","Africa/Monrovia|MMT LRT GMT|H.8 I.u 0|012|-23Lzg.Q 29s01.m","Africa/Ndjamena|LMT WAT WAST|-10.c -10 -20|0121|-2le10.c 2J3c0.c Wn0","Africa/Tripoli|LMT CET CEST EET|-Q.I -10 -20 -20|012121213121212121212121213123123|-21JcQ.I 1hnBQ.I vx0 4iP0 xx0 4eN0 Bb0 7ip0 U0n0 A10 1db0 1cN0 1db0 1dd0 1db0 1eN0 1bb0 1e10 1cL0 1c10 1db0 1dd0 1db0 1cN0 1db0 1q10 fAn0 1ep0 1db0 AKq0 TA0 1o00","Africa/Tunis|PMT CET CEST|-9.l -10 -20|0121212121212121212121212121212121|-2nco9.l 18pa9.l 1qM0 DA0 3Tc0 11B0 1ze0 WM0 7z0 3d0 14L0 1cN0 1f90 1ar0 16J0 1gXB0 WM0 1rA0 11c0 nwo0 Ko0 1cM0 1cM0 1rA0 10M0 zuM0 10N0 1aN0 1qM0 WM0 1qM0 11A0 1o00","Africa/Windhoek|SWAT SAST SAST CAT WAT WAST|-1u -20 -30 -20 -10 -20|012134545454545454545454545454545454545454545454545454545454545454545454545454545454545454545|-2GJdu 1Ajdu 1cL0 1SqL0 9NA0 11D0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0","America/Adak|NST NWT NPT BST BDT AHST HST HDT|b0 a0 a0 b0 a0 a0 a0 90|012034343434343434343434343434343456767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-17SX0 8wW0 iB0 Qlb0 52O0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cm0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Anchorage|CAT CAWT CAPT AHST AHDT YST AKST AKDT|a0 90 90 a0 90 90 90 80|012034343434343434343434343434343456767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-17T00 8wX0 iA0 Qlb0 52O0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cm0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Anguilla|LMT AST|46.4 40|01|-2kNvR.U","America/Araguaina|LMT BRT BRST|3c.M 30 20|0121212121212121212121212121212121212121212121212121|-2glwL.c HdKL.c 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 dMN0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 ny10 Lz0","America/Argentina/Buenos_Aires|CMT ART ARST ART ARST|4g.M 40 30 30 20|0121212121212121212121212121212121212121213434343434343234343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wp0 Rb0 1wp0 TX0 g0p0 10M0 j3c0 uL0 1qN0 WL0","America/Argentina/Catamarca|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|0121212121212121212121212121212121212121213434343454343235343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 g0p0 10M0 ako0 7B0 8zb0 uL0","America/Argentina/Cordoba|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|0121212121212121212121212121212121212121213434343454343234343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 g0p0 10M0 j3c0 uL0 1qN0 WL0","America/Argentina/Jujuy|CMT ART ARST ART ARST WART WARST|4g.M 40 30 30 20 40 30|01212121212121212121212121212121212121212134343456543432343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1ze0 TX0 1ld0 WK0 1wp0 TX0 g0p0 10M0 j3c0 uL0","America/Argentina/La_Rioja|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|01212121212121212121212121212121212121212134343434534343235343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Qn0 qO0 16n0 Rb0 1wp0 TX0 g0p0 10M0 ako0 7B0 8zb0 uL0","America/Argentina/Mendoza|CMT ART ARST ART ARST WART WARST|4g.M 40 30 30 20 40 30|0121212121212121212121212121212121212121213434345656543235343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1u20 SL0 1vd0 Tb0 1wp0 TW0 g0p0 10M0 agM0 Op0 7TX0 uL0","America/Argentina/Rio_Gallegos|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|0121212121212121212121212121212121212121213434343434343235343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wp0 Rb0 1wp0 TX0 g0p0 10M0 ako0 7B0 8zb0 uL0","America/Argentina/Salta|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|01212121212121212121212121212121212121212134343434543432343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 g0p0 10M0 j3c0 uL0","America/Argentina/San_Juan|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|01212121212121212121212121212121212121212134343434534343235343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Qn0 qO0 16n0 Rb0 1wp0 TX0 g0p0 10M0 ak00 m10 8lb0 uL0","America/Argentina/San_Luis|CMT ART ARST ART ARST WART WARST|4g.M 40 30 30 20 40 30|01212121212121212121212121212121212121212134343456536353465653|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 XX0 1q20 SL0 AN0 kin0 10M0 ak00 m10 8lb0 8L0 jd0 1qN0 WL0 1qN0","America/Argentina/Tucuman|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|012121212121212121212121212121212121212121343434345434323534343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 g0p0 10M0 ako0 4N0 8BX0 uL0 1qN0 WL0","America/Argentina/Ushuaia|CMT ART ARST ART ARST WART|4g.M 40 30 30 20 40|0121212121212121212121212121212121212121213434343434343235343|-20UHH.c pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wp0 Rb0 1wp0 TX0 g0p0 10M0 ajA0 8p0 8zb0 uL0","America/Aruba|LMT ANT AST|4z.L 4u 40|012|-2kV7o.d 28KLS.d","America/Asuncion|AMT PYT PYT PYST|3O.E 40 30 30|012131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313|-1x589.k 1DKM9.k 3CL0 3Dd0 10L0 1pB0 10n0 1pB0 10n0 1pB0 1cL0 1dd0 1db0 1dd0 1cL0 1dd0 1cL0 1dd0 1cL0 1dd0 1db0 1dd0 1cL0 1dd0 1cL0 1dd0 1cL0 1dd0 1db0 1dd0 1cL0 1lB0 14n0 1dd0 1cL0 1fd0 WL0 1rd0 1aL0 1dB0 Xz0 1qp0 Xb0 1qN0 10L0 1rB0 TX0 1tB0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 WN0 1qL0 11B0 1nX0 1ip0 WL0 1qN0 WL0 1qN0 WL0 1tB0 TX0 1tB0 TX0 1tB0 19X0 1a10 1fz0 1a10 1fz0 1cN0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0","America/Atikokan|CST CDT CWT CPT EST|60 50 50 50 50|0101234|-25TQ0 1in0 Rnb0 3je0 8x30 iw0","America/Bahia|LMT BRT BRST|2y.4 30 20|01212121212121212121212121212121212121212121212121212121212121|-2glxp.U HdLp.U 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 l5B0 Rb0","America/Bahia_Banderas|LMT MST CST PST MDT CDT|71 70 60 80 60 50|0121212131414141414141414141414141414152525252525252525252525252525252525252525252525252525252|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 otX0 gmN0 P2N0 13Vd0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nW0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Barbados|LMT BMT AST ADT|3W.t 3W.t 40 30|01232323232|-1Q0I1.v jsM0 1ODC1.v IL0 1ip0 17b0 1ip0 17b0 1ld0 13b0","America/Belem|LMT BRT BRST|3d.U 30 20|012121212121212121212121212121|-2glwK.4 HdKK.4 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0","America/Belize|LMT CST CHDT CDT|5Q.M 60 5u 50|01212121212121212121212121212121212121212121212121213131|-2kBu7.c fPA7.c Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1f0Mu qn0 lxB0 mn0","America/Blanc-Sablon|AST ADT AWT APT|40 30 30 30|010230|-25TS0 1in0 UGp0 8x50 iu0","America/Boa_Vista|LMT AMT AMST|42.E 40 30|0121212121212121212121212121212121|-2glvV.k HdKV.k 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 smp0 WL0 1tB0 2L0","America/Bogota|BMT COT COST|4U.g 50 40|0121|-2eb73.I 38yo3.I 2en0","America/Boise|PST PDT MST MWT MPT MDT|80 70 70 60 60 60|0101023425252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252|-261q0 1nX0 11B0 1nX0 8C10 JCL0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 Dd0 1Kn0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Cambridge_Bay|zzz MST MWT MPT MDDT MDT CST CDT EST|0 70 60 60 50 60 60 50 50|0123141515151515151515151515151515151515151515678651515151515151515151515151515151515151515151515151515151515151515151515151|-21Jc0 RO90 8x20 ix0 LCL0 1fA0 zgO0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11A0 1nX0 2K0 WQ0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Campo_Grande|LMT AMT AMST|3C.s 40 30|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-2glwl.w HdLl.w 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 1C10 Lz0 1Ip0 HX0 1zd0 On0 1HB0 IL0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1zd0 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1C10 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1C10 Lz0 1C10 Lz0 1C10 Lz0 1C10 On0 1zd0 Rb0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0","America/Cancun|LMT CST EST EDT CDT|5L.4 60 50 40 50|0123232341414141414141414141414141414141412|-1UQG0 2q2o0 yLB0 1lb0 14p0 1lb0 14p0 Lz0 xB0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 Dd0","America/Caracas|CMT VET VET|4r.E 4u 40|0121|-2kV7w.k 28KM2.k 1IwOu","America/Cayenne|LMT GFT GFT|3t.k 40 30|012|-2mrwu.E 2gWou.E","America/Cayman|CMT EST|5j.A 50|01|-2uduE.o","America/Chicago|CST CDT EST CWT CPT|60 50 50 50 50|01010101010101010101010101010101010102010101010103401010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261s0 1nX0 11B0 1nX0 1wp0 TX0 WN0 1qL0 1cN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 11B0 1Hz0 14p0 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 RB0 8x30 iw0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Chihuahua|LMT MST CST CDT MDT|74.k 70 60 50 60|0121212323241414141414141414141414141414141414141414141414141414141414141414141414141414141|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 2zQN0 1lb0 14p0 1lb0 14q0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Costa_Rica|SJMT CST CDT|5A.d 60 50|0121212121|-1Xd6n.L 2lu0n.L Db0 1Kp0 Db0 pRB0 15b0 1kp0 mL0","America/Creston|MST PST|70 80|010|-29DR0 43B0","America/Cuiaba|LMT AMT AMST|3I.k 40 30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-2glwf.E HdLf.E 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 4a10 HX0 1zd0 On0 1HB0 IL0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1zd0 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1C10 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1C10 Lz0 1C10 Lz0 1C10 Lz0 1C10 On0 1zd0 Rb0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0","America/Danmarkshavn|LMT WGT WGST GMT|1e.E 30 20 0|01212121212121212121212121212121213|-2a5WJ.k 2z5fJ.k 19U0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 DC0","America/Dawson|YST YDT YWT YPT YDDT PST PDT|90 80 80 80 70 80 70|0101023040565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-25TN0 1in0 1o10 13V0 Ser0 8x00 iz0 LCL0 1fA0 jrA0 fNd0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Dawson_Creek|PST PDT PWT PPT MST|80 70 70 70 70|0102301010101010101010101010101010101010101010101010101014|-25TO0 1in0 UGp0 8x10 iy0 3NB0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 ML0","America/Denver|MST MDT MWT MPT|70 60 60 60|01010101023010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261r0 1nX0 11B0 1nX0 11B0 1qL0 WN0 mn0 Ord0 8x20 ix0 LCN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Detroit|LMT CST EST EWT EPT EDT|5w.b 60 50 40 40 40|01234252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252|-2Cgir.N peqr.N 156L0 8x40 iv0 6fd0 11z0 Jy10 SL0 dnB0 1cL0 s10 1Vz0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Edmonton|LMT MST MDT MWT MPT|7x.Q 70 60 60 60|01212121212121341212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2yd4q.8 shdq.8 1in0 17d0 hz0 2dB0 1fz0 1a10 11z0 1qN0 WL0 1qN0 11z0 IGN0 8x20 ix0 3NB0 11z0 LFB0 1cL0 3Cp0 1cL0 66N0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Eirunepe|LMT ACT ACST AMT|4D.s 50 40 40|0121212121212121212121212121212131|-2glvk.w HdLk.w 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 dPB0 On0 yTd0 d5X0","America/El_Salvador|LMT CST CDT|5U.M 60 50|012121|-1XiG3.c 2Fvc3.c WL0 1qN0 WL0","America/Ensenada|LMT MST PST PDT PWT PPT|7M.4 70 80 70 70 70|012123245232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1UQE0 4PX0 8mM0 8lc0 SN0 1cL0 pHB0 83r0 zI0 5O10 1Rz0 cOP0 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 BUp0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Fort_Wayne|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|010101023010101010101010101040454545454545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 QI10 Db0 RB0 8x30 iw0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 5Tz0 1o10 qLb0 1cL0 1cN0 1cL0 1qhd0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Fortaleza|LMT BRT BRST|2y 30 20|0121212121212121212121212121212121212121|-2glxq HdLq 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 nsp0 WL0 1tB0 5z0 2mN0 On0","America/Glace_Bay|LMT AST ADT AWT APT|3X.M 40 30 30 30|012134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2IsI0.c CwO0.c 1in0 UGp0 8x50 iu0 iq10 11z0 Jg10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Godthab|LMT WGT WGST|3q.U 30 20|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2a5Ux.4 2z5dx.4 19U0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","America/Goose_Bay|NST NDT NST NDT NWT NPT AST ADT ADDT|3u.Q 2u.Q 3u 2u 2u 2u 40 30 20|010232323232323245232323232323232323232323232323232323232326767676767676767676767676767676767676767676768676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-25TSt.8 1in0 DXb0 2HbX.8 WL0 1qN0 WL0 1qN0 WL0 1tB0 TX0 1tB0 WL0 1qN0 WL0 1qN0 7UHu itu 1tB0 WL0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1tB0 WL0 1ld0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 S10 g0u 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14n1 1lb0 14p0 1nW0 11C0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zcX Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Grand_Turk|KMT EST EDT AST|57.b 50 40 40|0121212121212121212121212121212121212121212121212121212121212121212121212123|-2l1uQ.N 2HHBQ.N 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Guatemala|LMT CST CDT|62.4 60 50|0121212121|-24KhV.U 2efXV.U An0 mtd0 Nz0 ifB0 17b0 zDB0 11z0","America/Guayaquil|QMT ECT|5e 50|01|-1yVSK","America/Guyana|LMT GBGT GYT GYT GYT|3Q.E 3J 3J 30 40|01234|-2dvU7.k 24JzQ.k mlc0 Bxbf","America/Halifax|LMT AST ADT AWT APT|4e.o 40 30 30 30|0121212121212121212121212121212121212121212121212134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2IsHJ.A xzzJ.A 1db0 3I30 1in0 3HX0 IL0 1E10 ML0 1yN0 Pb0 1Bd0 Mn0 1Bd0 Rz0 1w10 Xb0 1w10 LX0 1w10 Xb0 1w10 Lz0 1C10 Jz0 1E10 OL0 1yN0 Un0 1qp0 Xb0 1qp0 11X0 1w10 Lz0 1HB0 LX0 1C10 FX0 1w10 Xb0 1qp0 Xb0 1BB0 LX0 1td0 Xb0 1qp0 Xb0 Rf0 8x50 iu0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 3Qp0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 3Qp0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 6i10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Havana|HMT CST CDT|5t.A 50 40|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1Meuu.o 72zu.o ML0 sld0 An0 1Nd0 Db0 1Nd0 An0 6Ep0 An0 1Nd0 An0 JDd0 Mn0 1Ap0 On0 1fd0 11X0 1qN0 WL0 1wp0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 14n0 1ld0 14L0 1kN0 15b0 1kp0 1cL0 1cN0 1fz0 1a10 1fz0 1fB0 11z0 14p0 1nX0 11B0 1nX0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 1a10 1in0 1a10 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 17c0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 11A0 6i00 Rc0 1wo0 U00 1tA0 Rc0 1wo0 U00 1wo0 U00 1zc0 U00 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0","America/Hermosillo|LMT MST CST PST MDT|7n.Q 70 60 80 60|0121212131414141|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 otX0 gmN0 P2N0 13Vd0 1lb0 14p0 1lb0 14p0 1lb0","America/Indiana/Knox|CST CDT CWT CPT EST|60 50 50 50 50|0101023010101010101010101010101010101040101010101010101010101010101010101010101010101010141010101010101010101010101010101010101010101010101010101010101010|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 3NB0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 11z0 1o10 11z0 1o10 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 3Cn0 8wp0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 z8o0 1o00 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Indiana/Marengo|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|0101023010101010101010104545454545414545454545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 dyN0 11z0 6fd0 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 jrz0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1VA0 LA0 1BX0 1e6p0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Indiana/Petersburg|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|01010230101010101010101010104010101010101010101010141014545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 njX0 WN0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 3Fb0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 19co0 1o00 Rd0 1zb0 Oo0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Indiana/Tell_City|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|01010230101010101010101010101010454541010101010101010101010101010101010101010101010101010101010101010|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 g0p0 11z0 1o10 11z0 1qL0 WN0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 caL0 1cL0 1cN0 1cL0 1qhd0 1o00 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Indiana/Vevay|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|010102304545454545454545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 kPB0 Awn0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1lnd0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Indiana/Vincennes|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|01010230101010101010101010101010454541014545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 g0p0 11z0 1o10 11z0 1qL0 WN0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 caL0 1cL0 1cN0 1cL0 1qhd0 1o00 Rd0 1zb0 Oo0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Indiana/Winamac|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|01010230101010101010101010101010101010454541054545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 jrz0 1cL0 1cN0 1cL0 1qhd0 1o00 Rd0 1za0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Inuvik|zzz PST PDDT MST MDT|0 80 60 70 60|0121343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-FnA0 tWU0 1fA0 wPe0 2pz0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Iqaluit|zzz EWT EPT EST EDDT EDT CST CDT|0 40 40 50 30 40 60 50|01234353535353535353535353535353535353535353567353535353535353535353535353535353535353535353535353535353535353535353535353|-16K00 7nX0 iv0 LCL0 1fA0 zgO0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11C0 1nX0 11A0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Jamaica|KMT EST EDT|57.b 50 40|0121212121212121212121|-2l1uQ.N 2uM1Q.N 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0","America/Juneau|PST PWT PPT PDT YDT YST AKST AKDT|80 70 70 70 80 90 90 80|01203030303030303030303030403030356767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-17T20 8x10 iy0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cM0 1cM0 1cL0 1cN0 1fz0 1a10 1fz0 co0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Kentucky/Louisville|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|0101010102301010101010101010101010101454545454545414545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 3Fd0 Nb0 LPd0 11z0 RB0 8x30 iw0 Bb0 10N0 2bB0 8in0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 xz0 gso0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1VA0 LA0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Kentucky/Monticello|CST CDT CWT CPT EST EDT|60 50 50 50 50 40|0101023010101010101010101010101010101010101010101010101010101010101010101454545454545454545454545454545454545454545454545454545454545454545454545454|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 SWp0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11A0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/La_Paz|CMT BOST BOT|4w.A 3w.A 40|012|-1x37r.o 13b0","America/Lima|LMT PET PEST|58.A 50 40|0121212121212121|-2tyGP.o 1bDzP.o zX0 1aN0 1cL0 1cN0 1cL0 1PrB0 zX0 1O10 zX0 6Gp0 zX0 98p0 zX0","America/Los_Angeles|PST PDT PWT PPT|80 70 70 70|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261q0 1nX0 11B0 1nX0 SgN0 8x10 iy0 5Wp0 1Vb0 3dB0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Maceio|LMT BRT BRST|2m.Q 30 20|012121212121212121212121212121212121212121|-2glxB.8 HdLB.8 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 dMN0 Lz0 8Q10 WL0 1tB0 5z0 2mN0 On0","America/Managua|MMT CST EST CDT|5J.c 60 50 50|0121313121213131|-1quie.M 1yAMe.M 4mn0 9Up0 Dz0 1K10 Dz0 s3F0 1KH0 DB0 9In0 k8p0 19X0 1o30 11y0","America/Manaus|LMT AMT AMST|40.4 40 30|01212121212121212121212121212121|-2glvX.U HdKX.U 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 dPB0 On0","America/Martinique|FFMT AST ADT|44.k 40 30|0121|-2mPTT.E 2LPbT.E 19X0","America/Matamoros|LMT CST CDT|6E 60 50|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1UQG0 2FjC0 1nX0 i6p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Mazatlan|LMT MST CST PST MDT|75.E 70 60 80 60|0121212131414141414141414141414141414141414141414141414141414141414141414141414141414141414141|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 otX0 gmN0 P2N0 13Vd0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Menominee|CST CDT CWT CPT EST|60 50 50 50 50|01010230101041010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 LCN0 1fz0 6410 9Jb0 1cM0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Merida|LMT CST EST CDT|5W.s 60 50 50|0121313131313131313131313131313131313131313131313131313131313131313131313131313131313131|-1UQG0 2q2o0 2hz0 wu30 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Metlakatla|PST PWT PPT PDT|80 70 70 70|0120303030303030303030303030303030|-17T20 8x10 iy0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0","America/Mexico_City|LMT MST CST CDT CWT|6A.A 70 60 50 50|012121232324232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 gEn0 TX0 3xd0 Jb0 6zB0 SL0 e5d0 17b0 1Pff0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Miquelon|LMT AST PMST PMDT|3I.E 40 30 20|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-2mKkf.k 2LTAf.k gQ10 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Moncton|EST AST ADT AWT APT|50 40 30 30 30|012121212121212121212134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2IsH0 CwN0 1in0 zAo0 An0 1Nd0 An0 1Nd0 An0 1Nd0 An0 1Nd0 An0 1Nd0 An0 1K10 Lz0 1zB0 NX0 1u10 Wn0 S20 8x50 iu0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 3Cp0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14n1 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 ReX 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Monterrey|LMT CST CDT|6F.g 60 50|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1UQG0 2FjC0 1nX0 i6p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Montevideo|MMT UYT UYHST UYST UYT UYHST|3I.I 3u 30 20 30 2u|012121212121212121212121213434343434345454543453434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-20UIf.g 8jzJ.g 1cLu 1dcu 1cLu 1dcu 1cLu ircu 11zu 1o0u 11zu 1o0u 11zu 1qMu WLu 1qMu WLu 1qMu WLu 1qMu 11zu 1o0u 11zu NAu 11bu 2iMu zWu Dq10 19X0 pd0 jz0 cm10 19X0 1fB0 1on0 11d0 1oL0 1nB0 1fzu 1aou 1fzu 1aou 1fzu 3nAu Jb0 3MN0 1SLu 4jzu 2PB0 Lb0 3Dd0 1pb0 ixd0 An0 1MN0 An0 1wp0 On0 1wp0 Rb0 1zd0 On0 1wp0 Rb0 s8p0 1fB0 1ip0 11z0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 1ld0 14n0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 1ld0 14n0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10","America/Montreal|EST EDT EWT EPT|50 40 40 40|01010101010101010101010101010101010101010101012301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-25TR0 1in0 11Wu 1nzu 1fD0 WJ0 1wr0 Nb0 1Ap0 On0 1zd0 On0 1wp0 TX0 1tB0 TX0 1tB0 TX0 1tB0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 4kM0 8x40 iv0 1o10 11z0 1nX0 11z0 1o10 11z0 1o10 1qL0 11D0 1nX0 11B0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Nassau|LMT EST EDT|59.u 50 40|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2kNuO.u 26XdO.u 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/New_York|EST EDT EWT EPT|50 40 40 40|01010101010101010101010101010101010101010101010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261t0 1nX0 11B0 1nX0 11B0 1qL0 1a10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 RB0 8x40 iv0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Nipigon|EST EDT EWT EPT|50 40 40 40|010123010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-25TR0 1in0 Rnb0 3je0 8x40 iv0 19yN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Nome|NST NWT NPT BST BDT YST AKST AKDT|b0 a0 a0 b0 a0 90 90 80|012034343434343434343434343434343456767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-17SX0 8wW0 iB0 Qlb0 52O0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cl0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Noronha|LMT FNT FNST|29.E 20 10|0121212121212121212121212121212121212121|-2glxO.k HdKO.k 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 nsp0 WL0 1tB0 2L0 2pB0 On0","America/North_Dakota/Beulah|MST MDT MWT MPT CST CDT|70 60 60 60 60 50|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101014545454545454545454545454545454545454545454545454545454|-261r0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Oo0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/North_Dakota/Center|MST MDT MWT MPT CST CDT|70 60 60 60 60 50|010102301010101010101010101010101010101010101010101010101014545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-261r0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14o0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/North_Dakota/New_Salem|MST MDT MWT MPT CST CDT|70 60 60 60 60 50|010102301010101010101010101010101010101010101010101010101010101010101010101010101454545454545454545454545454545454545454545454545454545454545454545454|-261r0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14o0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Ojinaga|LMT MST CST CDT MDT|6V.E 70 60 50 60|0121212323241414141414141414141414141414141414141414141414141414141414141414141414141414141|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 2zQN0 1lb0 14p0 1lb0 14q0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Pangnirtung|zzz AST AWT APT ADDT ADT EDT EST CST CDT|0 40 30 30 20 30 40 50 60 50|012314151515151515151515151515151515167676767689767676767676767676767676767676767676767676767676767676767676767676767676767|-1XiM0 PnG0 8x50 iu0 LCL0 1fA0 zgO0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1o00 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11C0 1nX0 11A0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Paramaribo|LMT PMT PMT NEGT SRT SRT|3E.E 3E.Q 3E.A 3u 3u 30|012345|-2nDUj.k Wqo0.c qanX.I 1dmLN.o lzc0","America/Phoenix|MST MDT MWT|70 60 60|01010202010|-261r0 1nX0 11B0 1nX0 SgN0 4Al1 Ap0 1db0 SWqX 1cL0","America/Port-au-Prince|PPMT EST EDT|4N 50 40|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-28RHb 2FnMb 19X0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14q0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 i6n0 1nX0 11B0 1nX0 d430 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Porto_Acre|LMT ACT ACST AMT|4v.c 50 40 40|01212121212121212121212121212131|-2glvs.M HdLs.M 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 NBd0 d5X0","America/Porto_Velho|LMT AMT AMST|4f.A 40 30|012121212121212121212121212121|-2glvI.o HdKI.o 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0","America/Puerto_Rico|AST AWT APT|40 30 30|0120|-17lU0 7XT0 iu0","America/Rainy_River|CST CDT CWT CPT|60 50 50 50|010123010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-25TQ0 1in0 Rnb0 3je0 8x30 iw0 19yN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Rankin_Inlet|zzz CST CDDT CDT EST|0 60 40 50 50|012131313131313131313131313131313131313131313431313131313131313131313131313131313131313131313131313131313131313131313131|-vDc0 keu0 1fA0 zgO0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Recife|LMT BRT BRST|2j.A 30 20|0121212121212121212121212121212121212121|-2glxE.o HdLE.o 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 nsp0 WL0 1tB0 2L0 2pB0 On0","America/Regina|LMT MST MDT MWT MPT CST|6W.A 70 60 60 60 60|012121212121212121212121341212121212121212121212121215|-2AD51.o uHe1.o 1in0 s2L0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 66N0 1cL0 1cN0 19X0 1fB0 1cL0 1fB0 1cL0 1cN0 1cL0 M30 8x20 ix0 1ip0 1cL0 1ip0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 3NB0 1cL0 1cN0","America/Resolute|zzz CST CDDT CDT EST|0 60 40 50 50|012131313131313131313131313131313131313131313431313131313431313131313131313131313131313131313131313131313131313131313131|-SnA0 GWS0 1fA0 zgO0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Santa_Isabel|LMT MST PST PDT PWT PPT|7D.s 70 80 70 70 70|012123245232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1UQE0 4PX0 8mM0 8lc0 SN0 1cL0 pHB0 83r0 zI0 5O10 1Rz0 cOP0 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 BUp0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0","America/Santarem|LMT AMT AMST BRT|3C.M 40 30 30|0121212121212121212121212121213|-2glwl.c HdLl.c 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 NBd0","America/Santiago|SMT CLT CLT CLST CLST CLT|4G.K 50 40 40 30 30|01020313131313121242124242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424245|-2q2jh.e fJAh.e 5knG.K 1Vzh.e jRAG.K 1pbh.e 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 nHX0 op0 9Bz0 jb0 1oN0 ko0 Qeo0 WL0 1zd0 On0 1ip0 11z0 1o10 11z0 1qN0 WL0 1ld0 14n0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 1wn0","America/Santo_Domingo|SDMT EST EDT EHDT AST|4E 50 40 4u 40|01213131313131414|-1ttjk 1lJMk Mn0 6sp0 Lbu 1Cou yLu 1RAu wLu 1QMu xzu 1Q0u xXu 1PAu 13jB0 e00","America/Sao_Paulo|LMT BRT BRST|36.s 30 20|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-2glwR.w HdKR.w 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 pTd0 PX0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 1C10 Lz0 1Ip0 HX0 1zd0 On0 1HB0 IL0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1zd0 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1C10 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1C10 Lz0 1C10 Lz0 1C10 Lz0 1C10 On0 1zd0 Rb0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0","America/Scoresbysund|LMT CGT CGST EGST EGT|1r.Q 20 10 0 10|0121343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-2a5Ww.8 2z5ew.8 1a00 1cK0 1cL0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","America/Sitka|PST PWT PPT PDT YST AKST AKDT|80 70 70 70 90 90 80|01203030303030303030303030303030345656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-17T20 8x10 iy0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 co0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/St_Johns|NST NDT NST NDT NWT NPT NDDT|3u.Q 2u.Q 3u 2u 2u 2u 1u|01010101010101010101010101010101010102323232323232324523232323232323232323232323232323232323232323232323232323232323232323232323232323232326232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-28oit.8 14L0 1nB0 1in0 1gm0 Dz0 1JB0 1cL0 1cN0 1cL0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 1cL0 1cN0 1cL0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 1cL0 1fB0 19X0 1fB0 19X0 10O0 eKX.8 19X0 1iq0 WL0 1qN0 WL0 1qN0 WL0 1tB0 TX0 1tB0 WL0 1qN0 WL0 1qN0 7UHu itu 1tB0 WL0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1tB0 WL0 1ld0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14n1 1lb0 14p0 1nW0 11C0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zcX Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Swift_Current|LMT MST MDT MWT MPT CST|7b.k 70 60 60 60 60|012134121212121212121215|-2AD4M.E uHdM.E 1in0 UGp0 8x20 ix0 1o10 17b0 1ip0 11z0 1o10 11z0 1o10 11z0 isN0 1cL0 3Cp0 1cL0 1cN0 11z0 1qN0 WL0 pMp0","America/Tegucigalpa|LMT CST CDT|5M.Q 60 50|01212121|-1WGGb.8 2ETcb.8 WL0 1qN0 WL0 GRd0 AL0","America/Thule|LMT AST ADT|4z.8 40 30|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2a5To.Q 31NBo.Q 1cL0 1cN0 1cL0 1fB0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Thunder_Bay|CST EST EWT EPT EDT|60 50 40 40 40|0123141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141|-2q5S0 1iaN0 8x40 iv0 XNB0 1cL0 1cN0 1fz0 1cN0 1cL0 3Cp0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Vancouver|PST PDT PWT PPT|80 70 70 70|0102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-25TO0 1in0 UGp0 8x10 iy0 1o10 17b0 1ip0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Whitehorse|YST YDT YWT YPT YDDT PST PDT|90 80 80 80 70 80 70|0101023040565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-25TN0 1in0 1o10 13V0 Ser0 8x00 iz0 LCL0 1fA0 3NA0 vrd0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Winnipeg|CST CDT CWT CPT|60 50 50 50|010101023010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aIi0 WL0 3ND0 1in0 Jap0 Rb0 aCN0 8x30 iw0 1tB0 11z0 1ip0 11z0 1o10 11z0 1o10 11z0 1rd0 10L0 1op0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 1cL0 1cN0 11z0 6i10 WL0 6i10 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1a00 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1a00 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Yakutat|YST YWT YPT YDT AKST AKDT|90 80 80 80 90 80|01203030303030303030303030303030304545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-17T10 8x00 iz0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cn0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","America/Yellowknife|zzz MST MWT MPT MDDT MDT|0 70 60 60 50 60|012314151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151|-1pdA0 hix0 8x20 ix0 LCL0 1fA0 zgO0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","Antarctica/Casey|zzz AWST CAST|0 -80 -b0|012121|-2q00 1DjS0 T90 40P0 KL0","Antarctica/Davis|zzz DAVT DAVT|0 -70 -50|01012121|-vyo0 iXt0 alj0 1D7v0 VB0 3Wn0 KN0","Antarctica/DumontDUrville|zzz PMT DDUT|0 -a0 -a0|0102|-U0o0 cfq0 bFm0","Antarctica/Macquarie|AEST AEDT zzz MIST|-a0 -b0 0 -b0|0102010101010101010101010101010101010101010101010101010101010101010101010101010101010101013|-29E80 19X0 4SL0 1ayy0 Lvs0 1cM0 1o00 Rc0 1wo0 Rc0 1wo0 U00 1wo0 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1qM0 WM0 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1wo0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 11A0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 11A0 1o00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1cM0 1cM0 1cM0","Antarctica/Mawson|zzz MAWT MAWT|0 -60 -50|012|-CEo0 2fyk0","Antarctica/McMurdo|NZMT NZST NZST NZDT|-bu -cu -c0 -d0|01020202020202020202020202023232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323|-1GCVu Lz0 1tB0 11zu 1o0u 11zu 1o0u 11zu 1o0u 14nu 1lcu 14nu 1lcu 1lbu 11Au 1nXu 11Au 1nXu 11Au 1nXu 11Au 1nXu 11Au 1qLu WMu 1qLu 11Au 1n1bu IM0 1C00 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1qM0 14o0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1io0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00","Antarctica/Palmer|zzz ARST ART ART ARST CLT CLST CLT|0 30 40 30 20 40 30 30|012121212123435656565656565656565656565656565656565656565656565656565656565656567|-cao0 nD0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 jsN0 14N0 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 1wn0","Antarctica/Rothera|zzz ROTT|0 30|01|gOo0","Antarctica/Syowa|zzz SYOT|0 -30|01|-vs00","Antarctica/Troll|zzz UTC CEST|0 0 -20|01212121212121212121212121212121212121212121212121212121212121212121|1puo0 hd0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Antarctica/Vostok|zzz VOST|0 -60|01|-tjA0","Arctic/Longyearbyen|CET CEST|-10 -20|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2awM0 Qm0 W6o0 5pf0 WM0 1fA0 1cM0 1cM0 1cM0 1cM0 wJc0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1qM0 WM0 zpc0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Asia/Aden|LMT AST|-36.Q -30|01|-TvD6.Q","Asia/Almaty|LMT ALMT ALMT ALMST|-57.M -50 -60 -70|0123232323232323232323232323232323232323232323232|-1Pc57.M eUo7.M 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 3Cl0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0","Asia/Amman|LMT EET EEST|-2n.I -20 -30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1yW2n.I 1HiMn.I KL0 1oN0 11b0 1oN0 11b0 1pd0 1dz0 1cp0 11b0 1op0 11b0 fO10 1db0 1e10 1cL0 1cN0 1cL0 1cN0 1fz0 1pd0 10n0 1ld0 14n0 1hB0 15b0 1ip0 19X0 1cN0 1cL0 1cN0 17b0 1ld0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1So0 y00 1fc0 1dc0 1co0 1dc0 1cM0 1cM0 1cM0 1o00 11A0 1lc0 17c0 1cM0 1cM0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 4bX0 Dd0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0","Asia/Anadyr|LMT ANAT ANAT ANAST ANAST ANAST ANAT|-bN.U -c0 -d0 -e0 -d0 -c0 -b0|01232414141414141414141561414141414141414141414141414141414141561|-1PcbN.U eUnN.U 23CL0 1db0 1cN0 1dc0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qN0 WM0","Asia/Aqtau|LMT FORT FORT SHET SHET SHEST AQTT AQTST AQTST AQTT|-3l.4 -40 -50 -50 -60 -60 -50 -60 -50 -40|012345353535353535353536767676898989898989898989896|-1Pc3l.4 eUnl.4 1jcL0 JDc0 1cL0 1dc0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2UK0 Fz0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cN0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 RW0","Asia/Aqtobe|LMT AKTT AKTT AKTST AKTT AQTT AQTST|-3M.E -40 -50 -60 -60 -50 -60|01234323232323232323232565656565656565656565656565|-1Pc3M.E eUnM.E 23CL0 1db0 1cM0 1dc0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2UK0 Fz0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0","Asia/Ashgabat|LMT ASHT ASHT ASHST ASHST TMT TMT|-3R.w -40 -50 -60 -50 -40 -50|012323232323232323232324156|-1Pc3R.w eUnR.w 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 ba0 xC0","Asia/Baghdad|BMT AST ADT|-2V.A -30 -40|012121212121212121212121212121212121212121212121212121|-26BeV.A 2ACnV.A 11b0 1cp0 1dz0 1dd0 1db0 1cN0 1cp0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1de0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0","Asia/Bahrain|LMT GST AST|-3q.8 -40 -30|012|-21Jfq.8 27BXq.8","Asia/Baku|LMT BAKT BAKT BAKST BAKST AZST AZT AZT AZST|-3j.o -30 -40 -50 -40 -40 -30 -40 -50|0123232323232323232323245657878787878787878787878787878787878787878787878787878787878787878787878787878787878787|-1Pc3j.o 1jUoj.o WCL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 10K0 c30 1cJ0 1cL0 8wu0 1o00 11z0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Asia/Bangkok|BMT ICT|-6G.4 -70|01|-218SG.4","Asia/Beirut|EET EEST|-20 -30|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-21aq0 1on0 1410 1db0 19B0 1in0 1ip0 WL0 1lQp0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 11b0 q6N0 En0 1oN0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 11b0 1op0 11b0 dA10 17b0 1iN0 17b0 1iN0 17b0 1iN0 17b0 1vB0 SL0 1mp0 13z0 1iN0 17b0 1iN0 17b0 1jd0 12n0 1a10 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0","Asia/Bishkek|LMT FRUT FRUT FRUST FRUST KGT KGST KGT|-4W.o -50 -60 -70 -60 -50 -60 -60|01232323232323232323232456565656565656565656565656567|-1Pc4W.o eUnW.o 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 11c0 1tX0 17b0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1cPu 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 T8u","Asia/Brunei|LMT BNT BNT|-7D.E -7u -80|012|-1KITD.E gDc9.E","Asia/Calcutta|HMT BURT IST IST|-5R.k -6u -5u -6u|01232|-18LFR.k 1unn.k HB0 7zX0","Asia/Chita|LMT YAKT YAKT YAKST YAKST YAKT IRKT|-7x.Q -80 -90 -a0 -90 -a0 -80|012323232323232323232324123232323232323232323232323232323232323256|-21Q7x.Q pAnx.Q 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Choibalsan|LMT ULAT ULAT CHOST CHOT CHOT CHOST|-7C -70 -80 -a0 -90 -80 -90|0123434343434343434343434343434343434343434343456565656565656565656565656565656565656565656565|-2APHC 2UkoC cKn0 1da0 1dd0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 6hD0 11z0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 3Db0 h1f0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0","Asia/Chongqing|CST CDT|-80 -90|01010101010101010|-1c1I0 LX0 16p0 1jz0 1Myp0 Rb0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0","Asia/Colombo|MMT IST IHST IST LKT LKT|-5j.w -5u -60 -6u -6u -60|01231451|-2zOtj.w 1rFbN.w 1zzu 7Apu 23dz0 11zu n3cu","Asia/Dacca|HMT BURT IST DACT BDT BDST|-5R.k -6u -5u -60 -60 -70|01213454|-18LFR.k 1unn.k HB0 m6n0 LqMu 1x6n0 1i00","Asia/Damascus|LMT EET EEST|-2p.c -20 -30|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-21Jep.c Hep.c 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1xRB0 11X0 1oN0 10L0 1pB0 11b0 1oN0 10L0 1mp0 13X0 1oN0 11b0 1pd0 11b0 1oN0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 Nb0 1AN0 Nb0 bcp0 19X0 1gp0 19X0 3ld0 1xX0 Vd0 1Bz0 Sp0 1vX0 10p0 1dz0 1cN0 1cL0 1db0 1db0 1g10 1an0 1ap0 1db0 1fd0 1db0 1cN0 1db0 1dd0 1db0 1cp0 1dz0 1c10 1dX0 1cN0 1db0 1dd0 1db0 1cN0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1db0 1cN0 1db0 1cN0 19z0 1fB0 1qL0 11B0 1on0 Wp0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0","Asia/Dili|LMT TLT JST TLT WITA|-8m.k -80 -90 -90 -80|012343|-2le8m.k 1dnXm.k 8HA0 1ew00 Xld0","Asia/Dubai|LMT GST|-3F.c -40|01|-21JfF.c","Asia/Dushanbe|LMT DUST DUST DUSST DUSST TJT|-4z.c -50 -60 -70 -60 -50|0123232323232323232323245|-1Pc4z.c eUnz.c 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 14N0","Asia/Gaza|EET EET EEST IST IDT|-20 -30 -30 -20 -30|010101010102020202020202020202023434343434343434343434343430202020202020202020202020202020202020202020202020202020202020202020202020202020202020|-1c2q0 5Rb0 10r0 1px0 10N0 1pz0 16p0 1jB0 16p0 1jx0 pBd0 Vz0 1oN0 11b0 1oO0 10N0 1pz0 10N0 1pb0 10N0 1pb0 10N0 1pb0 10N0 1pz0 10N0 1pb0 10N0 1pb0 11d0 1oL0 dW0 hfB0 Db0 1fB0 Rb0 npB0 11z0 1C10 IL0 1s10 10n0 1o10 WL0 1zd0 On0 1ld0 11z0 1o10 14n0 1o10 14n0 1nd0 12n0 1nd0 Xz0 1q10 12n0 M10 C00 17c0 1io0 17c0 1io0 17c0 1o00 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 17c0 1io0 18N0 1bz0 19z0 1gp0 1610 1iL0 11z0 1o10 14o0 1lA1 SKX 1xd1 MKX 1AN0 1a00 1fA0 1cL0 1cN0 1nX0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0","Asia/Hebron|EET EET EEST IST IDT|-20 -30 -30 -20 -30|01010101010202020202020202020202343434343434343434343434343020202020202020202020202020202020202020202020202020202020202020202020202020202020202020|-1c2q0 5Rb0 10r0 1px0 10N0 1pz0 16p0 1jB0 16p0 1jx0 pBd0 Vz0 1oN0 11b0 1oO0 10N0 1pz0 10N0 1pb0 10N0 1pb0 10N0 1pb0 10N0 1pz0 10N0 1pb0 10N0 1pb0 11d0 1oL0 dW0 hfB0 Db0 1fB0 Rb0 npB0 11z0 1C10 IL0 1s10 10n0 1o10 WL0 1zd0 On0 1ld0 11z0 1o10 14n0 1o10 14n0 1nd0 12n0 1nd0 Xz0 1q10 12n0 M10 C00 17c0 1io0 17c0 1io0 17c0 1o00 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 17c0 1io0 18N0 1bz0 19z0 1gp0 1610 1iL0 12L0 1mN0 14o0 1lc0 Tb0 1xd1 MKX bB0 cn0 1cN0 1a00 1fA0 1cL0 1cN0 1nX0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 1210 1nz0 14N0 1nz0 1210 1nz0 1210 1nz0 1210 1nz0","Asia/Ho_Chi_Minh|LMT PLMT ICT IDT JST|-76.E -76.u -70 -80 -90|0123423232|-2yC76.E bK00.a 1h7b6.u 5lz0 18o0 3Oq0 k5b0 aW00 BAM0","Asia/Hong_Kong|LMT HKT HKST JST|-7A.G -80 -90 -90|0121312121212121212121212121212121212121212121212121212121212121212121|-2CFHA.G 1sEP6.G 1cL0 ylu 93X0 1qQu 1tX0 Rd0 1In0 NB0 1cL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1kL0 14N0 1nX0 U10 1tz0 U10 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 U10 1tz0 U10 1wn0 Rd0 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 17d0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 s10 1Vz0 1cN0 1cL0 1cN0 1cL0 6fd0 14n0","Asia/Hovd|LMT HOVT HOVT HOVST|-66.A -60 -70 -80|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-2APG6.A 2Uko6.A cKn0 1db0 1dd0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 6hD0 11z0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 kEp0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0","Asia/Irkutsk|IMT IRKT IRKT IRKST IRKST IRKT|-6V.5 -70 -80 -90 -80 -90|012323232323232323232324123232323232323232323232323232323232323252|-21zGV.5 pjXV.5 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Istanbul|IMT EET EEST TRST TRT|-1U.U -20 -30 -40 -30|012121212121212121212121212121212121212121212121212121234343434342121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2ogNU.U dzzU.U 11b0 8tB0 1on0 1410 1db0 19B0 1in0 3Rd0 Un0 1oN0 11b0 zSp0 CL0 mN0 1Vz0 1gN0 1pz0 5Rd0 1fz0 1yp0 ML0 1kp0 17b0 1ip0 17b0 1fB0 19X0 1jB0 18L0 1ip0 17z0 qdd0 xX0 3S10 Tz0 dA10 11z0 1o10 11z0 1qN0 11z0 1ze0 11B0 WM0 1qO0 WI0 1nX0 1rB0 10L0 11B0 1in0 17d0 1in0 2pX0 19E0 1fU0 16Q0 1iI0 16Q0 1iI0 1Vd0 pb0 3Kp0 14o0 1df0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cL0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WO0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 Xc0 1qo0 WM0 1qM0 11A0 1o00 1200 1nA0 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Asia/Jakarta|BMT JAVT WIB JST WIB WIB|-77.c -7k -7u -90 -80 -70|01232425|-1Q0Tk luM0 mPzO 8vWu 6kpu 4PXu xhcu","Asia/Jayapura|LMT WIT ACST|-9m.M -90 -9u|0121|-1uu9m.M sMMm.M L4nu","Asia/Jerusalem|JMT IST IDT IDDT|-2k.E -20 -30 -40|01212121212132121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-26Bek.E SyMk.E 5Rb0 10r0 1px0 10N0 1pz0 16p0 1jB0 16p0 1jx0 3LB0 Em0 or0 1cn0 1dB0 16n0 10O0 1ja0 1tC0 14o0 1cM0 1a00 11A0 1Na0 An0 1MP0 AJ0 1Kp0 LC0 1oo0 Wl0 EQN0 Db0 1fB0 Rb0 npB0 11z0 1C10 IL0 1s10 10n0 1o10 WL0 1zd0 On0 1ld0 11z0 1o10 14n0 1o10 14n0 1nd0 12n0 1nd0 Xz0 1q10 12n0 1hB0 1dX0 1ep0 1aL0 1eN0 17X0 1nf0 11z0 1tB0 19W0 1e10 17b0 1ep0 1gL0 18N0 1fz0 1eN0 17b0 1gq0 1gn0 19d0 1dz0 1c10 17X0 1hB0 1gn0 19d0 1dz0 1c10 17X0 1kp0 1dz0 1c10 1aL0 1eN0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0","Asia/Kabul|AFT AFT|-40 -4u|01|-10Qs0","Asia/Kamchatka|LMT PETT PETT PETST PETST|-ay.A -b0 -c0 -d0 -c0|01232323232323232323232412323232323232323232323232323232323232412|-1SLKy.A ivXy.A 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qN0 WM0","Asia/Karachi|LMT IST IST KART PKT PKST|-4s.c -5u -6u -50 -50 -60|012134545454|-2xoss.c 1qOKW.c 7zX0 eup0 LqMu 1fy01 1cL0 dK0X 11b0 1610 1jX0","Asia/Kashgar|LMT XJT|-5O.k -60|01|-1GgtO.k","Asia/Kathmandu|LMT IST NPT|-5F.g -5u -5J|012|-21JhF.g 2EGMb.g","Asia/Khandyga|LMT YAKT YAKT YAKST YAKST VLAT VLAST VLAT YAKT|-92.d -80 -90 -a0 -90 -a0 -b0 -b0 -a0|01232323232323232323232412323232323232323232323232565656565656565782|-21Q92.d pAp2.d 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 qK0 yN0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 17V0 7zD0","Asia/Krasnoyarsk|LMT KRAT KRAT KRAST KRAST KRAT|-6b.q -60 -70 -80 -70 -80|012323232323232323232324123232323232323232323232323232323232323252|-21Hib.q prAb.q 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Kuala_Lumpur|SMT MALT MALST MALT MALT JST MYT|-6T.p -70 -7k -7k -7u -90 -80|01234546|-2Bg6T.p 17anT.p 7hXE dM00 17bO 8Fyu 1so1u","Asia/Kuching|LMT BORT BORT BORTST JST MYT|-7l.k -7u -80 -8k -90 -80|01232323232323232425|-1KITl.k gDbP.k 6ynu AnE 1O0k AnE 1NAk AnE 1NAk AnE 1NAk AnE 1O0k AnE 1NAk AnE pAk 8Fz0 1so10","Asia/Macao|LMT MOT MOST CST|-7y.k -80 -90 -80|0121212121212121212121212121212121212121213|-2le7y.k 1XO34.k 1wn0 Rd0 1wn0 R9u 1wqu U10 1tz0 TVu 1tz0 17gu 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cJu 1cL0 1cN0 1fz0 1cN0 1cOu 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cJu 1cL0 1cN0 1fz0 1cN0 1cL0 KEp0","Asia/Magadan|LMT MAGT MAGT MAGST MAGST MAGT|-a3.c -a0 -b0 -c0 -b0 -c0|012323232323232323232324123232323232323232323232323232323232323251|-1Pca3.c eUo3.c 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Makassar|LMT MMT WITA JST|-7V.A -7V.A -80 -90|01232|-21JjV.A vfc0 myLV.A 8ML0","Asia/Manila|PHT PHST JST|-80 -90 -90|010201010|-1kJI0 AL0 cK10 65X0 mXB0 vX0 VK10 1db0","Asia/Nicosia|LMT EET EEST|-2d.s -20 -30|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1Vc2d.s 2a3cd.s 1cL0 1qp0 Xz0 19B0 19X0 1fB0 1db0 1cp0 1cL0 1fB0 19X0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1o30 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Asia/Novokuznetsk|LMT KRAT KRAT KRAST KRAST NOVST NOVT NOVT|-5M.M -60 -70 -80 -70 -70 -60 -70|012323232323232323232324123232323232323232323232323232323232325672|-1PctM.M eULM.M 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qN0 WM0 8Hz0","Asia/Novosibirsk|LMT NOVT NOVT NOVST NOVST|-5v.E -60 -70 -80 -70|0123232323232323232323241232341414141414141414141414141414141414121|-21Qnv.E pAFv.E 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 ml0 Os0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Omsk|LMT OMST OMST OMSST OMSST OMST|-4R.u -50 -60 -70 -60 -70|012323232323232323232324123232323232323232323232323232323232323252|-224sR.u pMLR.u 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Oral|LMT URAT URAT URAST URAT URAST ORAT ORAST ORAT|-3p.o -40 -50 -60 -60 -50 -40 -50 -50|012343232323232323251516767676767676767676767676768|-1Pc3p.o eUnp.o 23CL0 1db0 1cM0 1dc0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 2UK0 Fz0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 RW0","Asia/Pontianak|LMT PMT WIB JST WIB WITA WIB|-7h.k -7h.k -7u -90 -80 -80 -70|012324256|-2ua7h.k XE00 munL.k 8Rau 6kpu 4PXu xhcu Wqnu","Asia/Pyongyang|LMT KST JCST JST KST|-8n -8u -90 -90 -90|01234|-2um8n 97XR 12FXu jdA0","Asia/Qyzylorda|LMT KIZT KIZT KIZST KIZT QYZT QYZT QYZST|-4l.Q -40 -50 -60 -60 -50 -60 -70|012343232323232323232325676767676767676767676767676|-1Pc4l.Q eUol.Q 23CL0 1db0 1cM0 1dc0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2UK0 dC0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0","Asia/Rangoon|RMT BURT JST MMT|-6o.E -6u -90 -6u|0123|-21Jio.E SmnS.E 7j9u","Asia/Sakhalin|LMT JCST JST SAKT SAKST SAKST SAKT|-9u.M -90 -90 -b0 -c0 -b0 -a0|0123434343434343434343435634343434343565656565656565656565656565636|-2AGVu.M 1iaMu.M je00 1qFa0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o10 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Samarkand|LMT SAMT SAMT SAMST TAST UZST UZT|-4r.R -40 -50 -60 -60 -60 -50|01234323232323232323232356|-1Pc4r.R eUor.R 23CL0 1db0 1cM0 1dc0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 11x0 bf0","Asia/Seoul|LMT KST JCST JST KST KDT KDT|-8r.Q -8u -90 -90 -90 -9u -a0|01234151515151515146464|-2um8r.Q 97XV.Q 12FXu jjA0 kKo0 2I0u OL0 1FB0 Rb0 1qN0 TX0 1tB0 TX0 1tB0 TX0 1tB0 TX0 2ap0 12FBu 11A0 1o00 11A0","Asia/Singapore|SMT MALT MALST MALT MALT JST SGT SGT|-6T.p -70 -7k -7k -7u -90 -7u -80|012345467|-2Bg6T.p 17anT.p 7hXE dM00 17bO 8Fyu Mspu DTA0","Asia/Srednekolymsk|LMT MAGT MAGT MAGST MAGST MAGT SRET|-ae.Q -a0 -b0 -c0 -b0 -c0 -b0|012323232323232323232324123232323232323232323232323232323232323256|-1Pcae.Q eUoe.Q 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Taipei|JWST JST CST CDT|-80 -90 -80 -90|01232323232323232323232323232323232323232|-1iw80 joM0 1yo0 Tz0 1ip0 1jX0 1cN0 11b0 1oN0 11b0 1oN0 11b0 1oN0 11b0 10N0 1BX0 10p0 1pz0 10p0 1pz0 10p0 1db0 1dd0 1db0 1cN0 1db0 1cN0 1db0 1cN0 1db0 1BB0 ML0 1Bd0 ML0 uq10 1db0 1cN0 1db0 97B0 AL0","Asia/Tashkent|LMT TAST TAST TASST TASST UZST UZT|-4B.b -50 -60 -70 -60 -60 -50|01232323232323232323232456|-1Pc4B.b eUnB.b 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 11y0 bf0","Asia/Tbilisi|TBMT TBIT TBIT TBIST TBIST GEST GET GET GEST|-2X.b -30 -40 -50 -40 -40 -30 -40 -50|0123232323232323232323245656565787878787878787878567|-1Pc2X.b 1jUnX.b WCL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 3y0 19f0 1cK0 1cL0 1cN0 1cL0 1cN0 1cL0 1cM0 1cL0 1fB0 3Nz0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 An0 Os0 WM0","Asia/Tehran|LMT TMT IRST IRST IRDT IRDT|-3p.I -3p.I -3u -40 -50 -4u|01234325252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252|-2btDp.I 1d3c0 1huLT.I TXu 1pz0 sN0 vAu 1cL0 1dB0 1en0 pNB0 UL0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 64p0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0","Asia/Thimbu|LMT IST BTT|-5W.A -5u -60|012|-Su5W.A 1BGMs.A","Asia/Tokyo|JCST JST JDT|-90 -90 -a0|0121212121|-1iw90 pKq0 QL0 1lB0 13X0 1zB0 NX0 1zB0 NX0","Asia/Ulaanbaatar|LMT ULAT ULAT ULAST|-77.w -70 -80 -90|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-2APH7.w 2Uko7.w cKn0 1db0 1dd0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 6hD0 11z0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 kEp0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1fx0 1cP0 1cJ0 1cP0 1cJ0 1cP0 1cJ0","Asia/Ust-Nera|LMT YAKT YAKT MAGST MAGT MAGST MAGT MAGT VLAT VLAT|-9w.S -80 -90 -c0 -b0 -b0 -a0 -c0 -b0 -a0|0123434343434343434343456434343434343434343434343434343434343434789|-21Q9w.S pApw.S 23CL0 1d90 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 17V0 7zD0","Asia/Vladivostok|LMT VLAT VLAT VLAST VLAST VLAT|-8L.v -90 -a0 -b0 -a0 -b0|012323232323232323232324123232323232323232323232323232323232323252|-1SJIL.v itXL.v 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Yakutsk|LMT YAKT YAKT YAKST YAKST YAKT|-8C.W -80 -90 -a0 -90 -a0|012323232323232323232324123232323232323232323232323232323232323252|-21Q8C.W pAoC.W 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Yekaterinburg|LMT PMT SVET SVET SVEST SVEST YEKT YEKST YEKT|-42.x -3J.5 -40 -50 -60 -50 -50 -60 -60|0123434343434343434343435267676767676767676767676767676767676767686|-2ag42.x 7mQh.s qBvJ.5 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Asia/Yerevan|LMT YERT YERT YERST YERST AMST AMT AMT AMST|-2W -30 -40 -50 -40 -40 -30 -40 -50|0123232323232323232323245656565657878787878787878787878787878787|-1Pc2W 1jUnW WCL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1am0 2r0 1cJ0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 3Fb0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0","Atlantic/Azores|HMT AZOT AZOST AZOMT AZOT AZOST WET|1S.w 20 10 0 10 0 0|01212121212121212121212121212121212121212121232123212321232121212121212121212121212121212121212121454545454545454545454545454545456545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-2ldW5.s aPX5.s Sp0 LX0 1vc0 Tc0 1uM0 SM0 1vc0 Tc0 1vc0 SM0 1vc0 6600 1co0 3E00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 3I00 17c0 1cM0 1cM0 3Fc0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 1tA0 1cM0 1dc0 1400 gL0 IM0 s10 U00 dX0 Rc0 pd0 Rc0 gL0 Oo0 pd0 Rc0 gL0 Oo0 pd0 14o0 1cM0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 3Co0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 qIl0 1cM0 1fA0 1cM0 1cM0 1cN0 1cL0 1cN0 1cM0 1cM0 1cM0 1cM0 1cN0 1cL0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cL0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Atlantic/Bermuda|LMT AST ADT|4j.i 40 30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1BnRE.G 1LTbE.G 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","Atlantic/Canary|LMT CANT WET WEST|11.A 10 0 -10|01232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1UtaW.o XPAW.o 1lAK0 1a10 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Atlantic/Cape_Verde|LMT CVT CVST CVT|1y.4 20 10 10|01213|-2xomp.U 1qOMp.U 7zX0 1djf0","Atlantic/Faeroe|LMT WET WEST|r.4 0 -10|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2uSnw.U 2Wgow.U 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Atlantic/Madeira|FMT MADT MADST MADMT WET WEST|17.A 10 0 -10 0 -10|01212121212121212121212121212121212121212121232123212321232121212121212121212121212121212121212121454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-2ldWQ.o aPWQ.o Sp0 LX0 1vc0 Tc0 1uM0 SM0 1vc0 Tc0 1vc0 SM0 1vc0 6600 1co0 3E00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 3I00 17c0 1cM0 1cM0 3Fc0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 1tA0 1cM0 1dc0 1400 gL0 IM0 s10 U00 dX0 Rc0 pd0 Rc0 gL0 Oo0 pd0 Rc0 gL0 Oo0 pd0 14o0 1cM0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 3Co0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 qIl0 1cM0 1fA0 1cM0 1cM0 1cN0 1cL0 1cN0 1cM0 1cM0 1cM0 1cM0 1cN0 1cL0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Atlantic/Reykjavik|LMT IST ISST GMT|1s 10 0 0|012121212121212121212121212121212121212121212121212121212121212121213|-2uWmw mfaw 1Bd0 ML0 1LB0 Cn0 1LB0 3fX0 C10 HrX0 1cO0 LB0 1EL0 LA0 1C00 Oo0 1wo0 Rc0 1wo0 Rc0 1wo0 Rc0 1zc0 Oo0 1zc0 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1lc0 14o0 1o00 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1o00 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1o00 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1o00 14o0","Atlantic/South_Georgia|GST|20|0|","Atlantic/Stanley|SMT FKT FKST FKT FKST|3P.o 40 30 30 20|0121212121212134343212121212121212121212121212121212121212121212121212|-2kJw8.A 12bA8.A 19X0 1fB0 19X0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 Cn0 1Cc10 WL0 1qL0 U10 1tz0 U10 1qM0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1tz0 U10 1tz0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1tz0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qN0 U10 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 U10 1tz0 U10 1tz0 U10 1wn0 U10 1tz0 U10 1tz0 U10","Australia/ACT|AEST AEDT|-a0 -b0|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101|-293lX xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 14o0 1o00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1tA0 WM0 1tA0 U00 1tA0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 11A0 1o00 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0","Australia/Adelaide|ACST ACDT|-9u -au|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101|-293lt xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 Oo0 1zc0 WM0 1qM0 Rc0 1zc0 U00 1tA0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0","Australia/Brisbane|AEST AEDT|-a0 -b0|01010101010101010|-293lX xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 H1A0 Oo0 1zc0 Oo0 1zc0 Oo0","Australia/Broken_Hill|ACST ACDT|-9u -au|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101|-293lt xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 14o0 1o00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1tA0 WM0 1tA0 U00 1tA0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0","Australia/Currie|AEST AEDT|-a0 -b0|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101|-29E80 19X0 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1qM0 WM0 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1wo0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 11A0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 11A0 1o00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0","Australia/Darwin|ACST ACDT|-9u -au|010101010|-293lt xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0","Australia/Eucla|ACWST ACWDT|-8J -9J|0101010101010101010|-293kI xcX 10jd0 yL0 1cN0 1cL0 1gSp0 Oo0 l5A0 Oo0 iJA0 G00 zU00 IM0 1qM0 11A0 1o00 11A0","Australia/Hobart|AEST AEDT|-a0 -b0|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101|-29E80 19X0 10jd0 yL0 1cN0 1cL0 1fB0 19X0 VfB0 1cM0 1o00 Rc0 1wo0 Rc0 1wo0 U00 1wo0 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1qM0 WM0 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1wo0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 11A0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 11A0 1o00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0","Australia/LHI|AEST LHST LHDT LHDT|-a0 -au -bu -b0|0121212121313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313|raC0 1zdu Rb0 1zd0 On0 1zd0 On0 1zd0 On0 1zd0 TXu 1qMu WLu 1tAu WLu 1tAu TXu 1tAu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1zcu Onu 1zcu Onu 1zcu 11zu 1o0u 11zu 1o0u 11zu 1o0u 11zu 1qMu WLu 11Au 1nXu 1qMu 11zu 1o0u 11zu 1o0u 11zu 1qMu WLu 1qMu 11zu 1o0u WLu 1qMu 14nu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu","Australia/Lindeman|AEST AEDT|-a0 -b0|010101010101010101010|-293lX xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 H1A0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0","Australia/Melbourne|AEST AEDT|-a0 -b0|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101|-293lX xcX 10jd0 yL0 1cN0 1cL0 1fB0 19X0 17c10 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1qM0 11A0 1tA0 U00 1tA0 U00 1tA0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 11A0 1o00 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0","Australia/Perth|AWST AWDT|-80 -90|0101010101010101010|-293jX xcX 10jd0 yL0 1cN0 1cL0 1gSp0 Oo0 l5A0 Oo0 iJA0 G00 zU00 IM0 1qM0 11A0 1o00 11A0","CET|CET CEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 16M0 1gMM0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","CST6CDT|CST CDT CWT CPT|60 50 50 50|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","Chile/EasterIsland|EMT EAST EASST EAST EASST EAST|7h.s 70 60 60 50 50|012121212121212121212121212123434343434343434343434343434343434343434343434343434343434343434345|-1uSgG.w 1s4IG.w WL0 1zd0 On0 1ip0 11z0 1o10 11z0 1qN0 WL0 1ld0 14n0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 1wn0","EET|EET EEST|-20 -30|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|hDB0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","EST|EST|50|0|","EST5EDT|EST EDT EWT EPT|50 40 40 40|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261t0 1nX0 11B0 1nX0 SgN0 8x40 iv0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","Eire|DMT IST GMT BST IST|p.l -y.D 0 -10 -10|01232323232324242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242|-2ax9y.D Rc0 1fzy.D 14M0 1fc0 1g00 1co0 1dc0 1co0 1oo0 1400 1dc0 19A0 1io0 1io0 WM0 1o00 14o0 1o00 17c0 1io0 17c0 1fA0 1a00 1lc0 17c0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1cM0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1io0 1qM0 Dc0 g5X0 14p0 1wn0 17d0 1io0 11A0 1o00 17c0 1fA0 1a00 1fA0 1cM0 1fA0 1a00 17c0 1fA0 1a00 1io0 17c0 1lc0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1a00 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1tA0 IM0 90o0 U00 1tA0 U00 1tA0 U00 1tA0 U00 1tA0 WM0 1qM0 WM0 1qM0 WM0 1tA0 U00 1tA0 U00 1tA0 11z0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 14o0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Etc/GMT+0|GMT|0|0|","Etc/GMT+1|GMT+1|10|0|","Etc/GMT+10|GMT+10|a0|0|","Etc/GMT+11|GMT+11|b0|0|","Etc/GMT+12|GMT+12|c0|0|","Etc/GMT+2|GMT+2|20|0|","Etc/GMT+3|GMT+3|30|0|","Etc/GMT+4|GMT+4|40|0|","Etc/GMT+5|GMT+5|50|0|","Etc/GMT+6|GMT+6|60|0|","Etc/GMT+7|GMT+7|70|0|","Etc/GMT+8|GMT+8|80|0|","Etc/GMT+9|GMT+9|90|0|","Etc/GMT-1|GMT-1|-10|0|","Etc/GMT-10|GMT-10|-a0|0|","Etc/GMT-11|GMT-11|-b0|0|","Etc/GMT-12|GMT-12|-c0|0|","Etc/GMT-13|GMT-13|-d0|0|","Etc/GMT-14|GMT-14|-e0|0|","Etc/GMT-2|GMT-2|-20|0|","Etc/GMT-3|GMT-3|-30|0|","Etc/GMT-4|GMT-4|-40|0|","Etc/GMT-5|GMT-5|-50|0|","Etc/GMT-6|GMT-6|-60|0|","Etc/GMT-7|GMT-7|-70|0|","Etc/GMT-8|GMT-8|-80|0|","Etc/GMT-9|GMT-9|-90|0|","Etc/UCT|UCT|0|0|","Etc/UTC|UTC|0|0|","Europe/Amsterdam|AMT NST NEST NET CEST CET|-j.w -1j.w -1k -k -20 -10|010101010101010101010101010101010101010101012323234545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545|-2aFcj.w 11b0 1iP0 11A0 1io0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1co0 1io0 1yo0 Pc0 1a00 1fA0 1Bc0 Mo0 1tc0 Uo0 1tA0 U00 1uo0 W00 1s00 VA0 1so0 Vc0 1sM0 UM0 1wo0 Rc0 1u00 Wo0 1rA0 W00 1s00 VA0 1sM0 UM0 1w00 fV0 BCX.w 1tA0 U00 1u00 Wo0 1sm0 601k WM0 1fA0 1cM0 1cM0 1cM0 16M0 1gMM0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Andorra|WET CET CEST|0 -10 -20|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-UBA0 1xIN0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Athens|AMT EET EEST CEST CET|-1y.Q -20 -30 -20 -10|012123434121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2a61x.Q CNbx.Q mn0 kU10 9b0 3Es0 Xa0 1fb0 1dd0 k3X0 Nz0 SCp0 1vc0 SO0 1cM0 1a00 1ao0 1fc0 1a10 1fG0 1cg0 1dX0 1bX0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Belfast|GMT BST BDST|0 -10 -20|0101010101010101010101010101010101010101010101010121212121210101210101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2axa0 Rc0 1fA0 14M0 1fc0 1g00 1co0 1dc0 1co0 1oo0 1400 1dc0 19A0 1io0 1io0 WM0 1o00 14o0 1o00 17c0 1io0 17c0 1fA0 1a00 1lc0 17c0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1cM0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1io0 1qM0 Dc0 2Rz0 Dc0 1zc0 Oo0 1zc0 Rc0 1wo0 17c0 1iM0 FA0 xB0 1fA0 1a00 14o0 bb0 LA0 xB0 Rc0 1wo0 11A0 1o00 17c0 1fA0 1a00 1fA0 1cM0 1fA0 1a00 17c0 1fA0 1a00 1io0 17c0 1lc0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1a00 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1tA0 IM0 90o0 U00 1tA0 U00 1tA0 U00 1tA0 U00 1tA0 WM0 1qM0 WM0 1qM0 WM0 1tA0 U00 1tA0 U00 1tA0 11z0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 14o0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Belgrade|CET CEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-19RC0 3IP0 WM0 1fA0 1cM0 1cM0 1rc0 Qo0 1vmo0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Berlin|CET CEST CEMT|-10 -20 -30|01010101010101210101210101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 kL0 Nc0 m10 WM0 1ao0 1cp0 dX0 jz0 Dd0 1io0 17c0 1fA0 1a00 1ehA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Bratislava|CET CEST|-10 -20|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 16M0 1lc0 1tA0 17A0 11c0 1io0 17c0 1io0 17c0 1fc0 1ao0 1bNc0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Brussels|WET CET CEST WEST|0 -10 -20 -10|0121212103030303030303030303030303030303030303030303212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2ehc0 3zX0 11c0 1iO0 11A0 1o00 11A0 my0 Ic0 1qM0 Rc0 1EM0 UM0 1u00 10o0 1io0 1io0 17c0 1a00 1fA0 1cM0 1cM0 1io0 17c0 1fA0 1a00 1io0 1a30 1io0 17c0 1fA0 1a00 1io0 17c0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 y00 5Wn0 WM0 1fA0 1cM0 16M0 1iM0 16M0 1C00 Uo0 1eeo0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Bucharest|BMT EET EEST|-1I.o -20 -30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1xApI.o 20LI.o RA0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1Axc0 On0 1fA0 1a10 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cK0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cL0 1cN0 1cL0 1fB0 1nX0 11E0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Budapest|CET CEST|-10 -20|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1ip0 17b0 1op0 1tb0 Q2m0 3Ne0 WM0 1fA0 1cM0 1cM0 1oJ0 1dc0 1030 1fA0 1cM0 1cM0 1cM0 1cM0 1fA0 1a00 1iM0 1fA0 8Ha0 Rb0 1wN0 Rb0 1BB0 Lz0 1C20 LB0 SNX0 1a10 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Busingen|CET CEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-19Lc0 11A0 1o00 11A0 1xG10 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Chisinau|CMT BMT EET EEST CEST CET MSK MSD|-1T -1I.o -20 -30 -20 -10 -30 -40|0123232323232323232345454676767676767676767623232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-26jdT wGMa.A 20LI.o RA0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 27A0 2en0 39g0 WM0 1fA0 1cM0 V90 1t7z0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1ty0 2bD0 1cM0 1cK0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1nX0 11E0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Copenhagen|CET CEST|-10 -20|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2azC0 Tz0 VuO0 60q0 WM0 1fA0 1cM0 1cM0 1cM0 S00 1HA0 Nc0 1C00 Dc0 1Nc0 Ao0 1h5A0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Gibraltar|GMT BST BDST CET CEST|0 -10 -20 -10 -20|010101010101010101010101010101010101010101010101012121212121010121010101010101010101034343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-2axa0 Rc0 1fA0 14M0 1fc0 1g00 1co0 1dc0 1co0 1oo0 1400 1dc0 19A0 1io0 1io0 WM0 1o00 14o0 1o00 17c0 1io0 17c0 1fA0 1a00 1lc0 17c0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1cM0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1io0 1qM0 Dc0 2Rz0 Dc0 1zc0 Oo0 1zc0 Rc0 1wo0 17c0 1iM0 FA0 xB0 1fA0 1a00 14o0 bb0 LA0 xB0 Rc0 1wo0 11A0 1o00 17c0 1fA0 1a00 1fA0 1cM0 1fA0 1a00 17c0 1fA0 1a00 1io0 17c0 1lc0 17c0 1fA0 10Jz0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Helsinki|HMT EET EEST|-1D.N -20 -30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1WuND.N OULD.N 1dA0 1xGq0 1cM0 1cM0 1cM0 1cN0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Kaliningrad|CET CEST CET CEST MSK MSD EEST EET FET|-10 -20 -20 -30 -30 -40 -30 -20 -30|0101010101010232454545454545454545454676767676767676767676767676767676767676787|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 Am0 Lb0 1en0 op0 1pNz0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 1cJ0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Europe/Kiev|KMT EET MSK CEST CET MSD EEST|-22.4 -20 -30 -20 -10 -40 -30|0123434252525252525252525256161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161|-1Pc22.4 eUo2.4 rnz0 2Hg0 WM0 1fA0 da0 1v4m0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 Db0 3220 1cK0 1cL0 1cN0 1cL0 1cN0 1cL0 1cQ0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Lisbon|LMT WET WEST WEMT CET CEST|A.J 0 -10 -20 -10 -20|012121212121212121212121212121212121212121212321232123212321212121212121212121212121212121212121214121212121212121212121212121212124545454212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2ldXn.f aPWn.f Sp0 LX0 1vc0 Tc0 1uM0 SM0 1vc0 Tc0 1vc0 SM0 1vc0 6600 1co0 3E00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 3I00 17c0 1cM0 1cM0 3Fc0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 1tA0 1cM0 1dc0 1400 gL0 IM0 s10 U00 dX0 Rc0 pd0 Rc0 gL0 Oo0 pd0 Rc0 gL0 Oo0 pd0 14o0 1cM0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 3Co0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 pvy0 1cM0 1cM0 1fA0 1cM0 1cM0 1cN0 1cL0 1cN0 1cM0 1cM0 1cM0 1cM0 1cN0 1cL0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Luxembourg|LMT CET CEST WET WEST WEST WET|-o.A -10 -20 0 -10 -20 -10|0121212134343434343434343434343434343434343434343434565651212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2DG0o.A t6mo.A TB0 1nX0 Up0 1o20 11A0 rW0 CM0 1qP0 R90 1EO0 UK0 1u20 10m0 1ip0 1in0 17e0 19W0 1fB0 1db0 1cp0 1in0 17d0 1fz0 1a10 1in0 1a10 1in0 17f0 1fA0 1a00 1io0 17c0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 vA0 60L0 WM0 1fA0 1cM0 17c0 1io0 16M0 1C00 Uo0 1eeo0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Madrid|WET WEST WEMT CET CEST|0 -10 -20 -10 -20|01010101010101010101010121212121234343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-28dd0 11A0 1go0 19A0 1co0 1dA0 b1A0 18o0 3I00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 iyo0 Rc0 18o0 1hc0 1io0 1a00 14o0 5aL0 MM0 1vc0 17A0 1i00 1bc0 1eo0 17d0 1in0 17A0 6hA0 10N0 XIL0 1a10 1in0 17d0 19X0 1cN0 1fz0 1a10 1fX0 1cp0 1cO0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Malta|CET CEST|-10 -20|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2as10 M00 1cM0 1cM0 14o0 1o00 WM0 1qM0 17c0 1cM0 M3A0 5M20 WM0 1fA0 1cM0 1cM0 1cM0 16m0 1de0 1lc0 14m0 1lc0 WO0 1qM0 GTW0 On0 1C10 Lz0 1C10 Lz0 1EN0 Lz0 1C10 Lz0 1zd0 Oo0 1C00 On0 1cp0 1cM0 1lA0 Xc0 1qq0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1iN0 19z0 1fB0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Minsk|MMT EET MSK CEST CET MSD EEST FET|-1O -20 -30 -20 -10 -40 -30 -30|012343432525252525252525252616161616161616161616161616161616161616172|-1Pc1O eUnO qNX0 3gQ0 WM0 1fA0 1cM0 Al0 1tsn0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 3Fc0 1cN0 1cK0 1cM0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hy0","Europe/Monaco|PMT WET WEST WEMT CET CEST|-9.l 0 -10 -20 -10 -20|01212121212121212121212121212121212121212121212121232323232345454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-2nco9.l cNb9.l HA0 19A0 1iM0 11c0 1oo0 Wo0 1rc0 QM0 1EM0 UM0 1u00 10o0 1io0 1wo0 Rc0 1a00 1fA0 1cM0 1cM0 1io0 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 1fA0 1a00 1io0 17c0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Df0 2RV0 11z0 11B0 1ze0 WM0 1fA0 1cM0 1fa0 1aq0 16M0 1ekn0 1cL0 1fC0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Moscow|MMT MMT MST MDST MSD MSK MSM EET EEST MSK|-2u.h -2v.j -3v.j -4v.j -40 -30 -50 -20 -30 -40|012132345464575454545454545454545458754545454545454545454545454545454545454595|-2ag2u.h 2pyW.W 1bA0 11X0 GN0 1Hb0 c20 imv.j 3DA0 dz0 15A0 c10 2q10 iM10 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rU0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Europe/Paris|PMT WET WEST CEST CET WEMT|-9.l 0 -10 -20 -10 -20|0121212121212121212121212121212121212121212121212123434352543434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-2nco8.l cNb8.l HA0 19A0 1iM0 11c0 1oo0 Wo0 1rc0 QM0 1EM0 UM0 1u00 10o0 1io0 1wo0 Rc0 1a00 1fA0 1cM0 1cM0 1io0 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 1fA0 1a00 1io0 17c0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Df0 Ik0 5M30 WM0 1fA0 1cM0 Vx0 hB0 1aq0 16M0 1ekn0 1cL0 1fC0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Riga|RMT LST EET MSK CEST CET MSD EEST|-1A.y -2A.y -20 -30 -20 -10 -40 -30|010102345454536363636363636363727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272|-25TzA.y 11A0 1iM0 ko0 gWm0 yDXA.y 2bX0 3fE0 WM0 1fA0 1cM0 1cM0 4m0 1sLy0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 1o00 11A0 1o00 11A0 1qM0 3oo0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Rome|CET CEST|-10 -20|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2as10 M00 1cM0 1cM0 14o0 1o00 WM0 1qM0 17c0 1cM0 M3A0 5M20 WM0 1fA0 1cM0 16K0 1iO0 16m0 1de0 1lc0 14m0 1lc0 WO0 1qM0 GTW0 On0 1C10 Lz0 1C10 Lz0 1EN0 Lz0 1C10 Lz0 1zd0 Oo0 1C00 On0 1C10 Lz0 1zd0 On0 1C10 LA0 1C00 LA0 1zc0 Oo0 1C00 Oo0 1zc0 Oo0 1fC0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Samara|LMT SAMT SAMT KUYT KUYST MSD MSK EEST KUYT SAMST SAMST|-3k.k -30 -40 -40 -50 -40 -30 -30 -30 -50 -40|012343434343434343435656782929292929292929292929292929292929292a12|-22WNk.k qHak.k bcn0 1Qqo0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cN0 8o0 14j0 1cL0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qN0 WM0","Europe/Simferopol|SMT EET MSK CEST CET MSD EEST MSK|-2g -20 -30 -20 -10 -40 -30 -40|012343432525252525252525252161616525252616161616161616161616161616161616172|-1Pc2g eUog rEn0 2qs0 WM0 1fA0 1cM0 3V0 1u0L0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1Q00 4eL0 1cL0 1cN0 1cL0 1cN0 dX0 WL0 1cN0 1cL0 1fB0 1o30 11B0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11z0 1nW0","Europe/Sofia|EET CET CEST EEST|-20 -10 -20 -30|01212103030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030|-168L0 WM0 1fA0 1cM0 1cM0 1cN0 1mKH0 1dd0 1fb0 1ap0 1fb0 1a20 1fy0 1a30 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cK0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1nX0 11E0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Stockholm|CET CEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2azC0 TB0 2yDe0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Tallinn|TMT CET CEST EET MSK MSD EEST|-1D -10 -20 -20 -30 -40 -30|012103421212454545454545454546363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363|-26oND teD 11A0 1Ta0 4rXl KSLD 2FX0 2Jg0 WM0 1fA0 1cM0 18J0 1sTX0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o10 11A0 1qM0 5QM0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Tirane|LMT CET CEST|-1j.k -10 -20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2glBj.k 14pcj.k 5LC0 WM0 4M0 1fCK0 10n0 1op0 11z0 1pd0 11z0 1qN0 WL0 1qp0 Xb0 1qp0 Xb0 1qp0 11z0 1lB0 11z0 1qN0 11z0 1iN0 16n0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Uzhgorod|CET CEST MSK MSD EET EEST|-10 -20 -30 -40 -20 -30|010101023232323232323232320454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-1cqL0 6i00 WM0 1fA0 1cM0 1ml0 1Cp0 1r3W0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1Q00 1Nf0 2pw0 1cL0 1cN0 1cL0 1cN0 1cL0 1cQ0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Vienna|CET CEST|-10 -20|0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 3KM0 14o0 LA00 6i00 WM0 1fA0 1cM0 1cM0 1cM0 400 2qM0 1a00 1cM0 1cM0 1io0 17c0 1gHa0 19X0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Vilnius|WMT KMT CET EET MSK CEST MSD EEST|-1o -1z.A -10 -20 -30 -20 -40 -30|012324525254646464646464646464647373737373737352537373737373737373737373737373737373737373737373737373737373737373737373|-293do 6ILM.o 1Ooz.A zz0 Mfd0 29W0 3is0 WM0 1fA0 1cM0 LV0 1tgL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11B0 1o00 11A0 1qM0 8io0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Volgograd|LMT TSAT STAT STAT VOLT VOLST VOLST VOLT MSD MSK MSK|-2V.E -30 -30 -40 -40 -50 -40 -30 -40 -30 -40|0123454545454545454546767489898989898989898989898989898989898989a9|-21IqV.E cLXV.E cEM0 1gqn0 Lco0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 2pz0 1cJ0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0","Europe/Warsaw|WMT CET CEST EET EEST|-1o -10 -20 -20 -30|012121234312121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2ctdo 1LXo 11d0 1iO0 11A0 1o00 11A0 1on0 11A0 6zy0 HWP0 5IM0 WM0 1fA0 1cM0 1dz0 1mL0 1en0 15B0 1aq0 1nA0 11A0 1io0 17c0 1fA0 1a00 iDX0 LA0 1cM0 1cM0 1C00 Oo0 1cM0 1cM0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1C00 LA0 uso0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","Europe/Zaporozhye|CUT EET MSK CEST CET MSD EEST|-2k -20 -30 -20 -10 -40 -30|01234342525252525252525252526161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161|-1Pc2k eUok rdb0 2RE0 WM0 1fA0 8m0 1v9a0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cK0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cQ0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","HST|HST|a0|0|","Indian/Chagos|LMT IOT IOT|-4N.E -50 -60|012|-2xosN.E 3AGLN.E","Indian/Christmas|CXT|-70|0|","Indian/Cocos|CCT|-6u|0|","Indian/Kerguelen|zzz TFT|0 -50|01|-MG00","Indian/Mahe|LMT SCT|-3F.M -40|01|-2yO3F.M","Indian/Maldives|MMT MVT|-4S -50|01|-olgS","Indian/Mauritius|LMT MUT MUST|-3O -40 -50|012121|-2xorO 34unO 14L0 12kr0 11z0","Indian/Reunion|LMT RET|-3F.Q -40|01|-2mDDF.Q","Kwajalein|MHT KWAT MHT|-b0 c0 -c0|012|-AX0 W9X0","MET|MET MEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 16M0 1gMM0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00","MST|MST|70|0|","MST7MDT|MST MDT MWT MPT|70 60 60 60|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261r0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","NZ-CHAT|CHAST CHAST CHADT|-cf -cJ -dJ|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-WqAf 1adef IM0 1C00 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1qM0 14o0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1io0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00","PST8PDT|PST PDT PWT PPT|80 70 70 70|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261q0 1nX0 11B0 1nX0 SgN0 8x10 iy0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0","Pacific/Apia|LMT WSST SST SDT WSDT WSST|bq.U bu b0 a0 -e0 -d0|01232345454545454545454545454545454545454545454545454545454|-2nDMx.4 1yW03.4 2rRbu 1ff0 1a00 CI0 AQ0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00","Pacific/Bougainville|PGT JST BST|-a0 -90 -b0|0102|-16Wy0 7CN0 2MQp0","Pacific/Chuuk|CHUT|-a0|0|","Pacific/Efate|LMT VUT VUST|-bd.g -b0 -c0|0121212121212121212121|-2l9nd.g 2Szcd.g 1cL0 1oN0 10L0 1fB0 19X0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 Lz0 1Nd0 An0","Pacific/Enderbury|PHOT PHOT PHOT|c0 b0 -d0|012|nIc0 B8n0","Pacific/Fakaofo|TKT TKT|b0 -d0|01|1Gfn0","Pacific/Fiji|LMT FJT FJST|-bT.I -c0 -d0|012121212121212121212121212121212121212121212121212121212121212|-2bUzT.I 3m8NT.I LA0 1EM0 IM0 nJc0 LA0 1o00 Rc0 1wo0 Ao0 1Nc0 Ao0 1Q00 xz0 1SN0 uM0 1SM0 xA0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 xA0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 xA0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1VA0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0 uM0 1SM0","Pacific/Funafuti|TVT|-c0|0|","Pacific/Galapagos|LMT ECT GALT|5W.o 50 60|012|-1yVS1.A 2dTz1.A","Pacific/Gambier|LMT GAMT|8X.M 90|01|-2jof0.c","Pacific/Guadalcanal|LMT SBT|-aD.M -b0|01|-2joyD.M","Pacific/Guam|GST ChST|-a0 -a0|01|1fpq0","Pacific/Honolulu|HST HDT HST|au 9u a0|010102|-1thLu 8x0 lef0 8Pz0 46p0","Pacific/Kiritimati|LINT LINT LINT|aE a0 -e0|012|nIaE B8nk","Pacific/Kosrae|KOST KOST|-b0 -c0|010|-AX0 1bdz0","Pacific/Majuro|MHT MHT|-b0 -c0|01|-AX0","Pacific/Marquesas|LMT MART|9i 9u|01|-2joeG","Pacific/Midway|LMT NST BST SST|bm.M b0 b0 b0|0123|-2nDMB.c 2gVzB.c EyM0","Pacific/Nauru|LMT NRT JST NRT|-b7.E -bu -90 -c0|01213|-1Xdn7.E PvzB.E 5RCu 1ouJu","Pacific/Niue|NUT NUT NUT|bk bu b0|012|-KfME 17y0a","Pacific/Norfolk|NMT NFT|-bc -bu|01|-Kgbc","Pacific/Noumea|LMT NCT NCST|-b5.M -b0 -c0|01212121|-2l9n5.M 2EqM5.M xX0 1PB0 yn0 HeP0 Ao0","Pacific/Palau|PWT|-90|0|","Pacific/Pitcairn|PNT PST|8u 80|01|18Vku","Pacific/Pohnpei|PONT|-b0|0|","Pacific/Port_Moresby|PGT|-a0|0|","Pacific/Rarotonga|CKT CKHST CKT|au 9u a0|012121212121212121212121212|lyWu IL0 1zcu Onu 1zcu Onu 1zcu Rbu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1zcu Onu 1zcu Onu 1zcu Onu","Pacific/Tahiti|LMT TAHT|9W.g a0|01|-2joe1.I","Pacific/Tarawa|GILT|-c0|0|","Pacific/Tongatapu|TOT TOT TOST|-ck -d0 -e0|01212121|-1aB0k 2n5dk 15A0 1wo0 xz0 1Q10 xz0","Pacific/Wake|WAKT|-c0|0|","Pacific/Wallis|WFT|-c0|0|","WET|WET WEST|0 -10|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|hDB0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00"], -links:["Africa/Abidjan|Africa/Bamako","Africa/Abidjan|Africa/Banjul","Africa/Abidjan|Africa/Conakry","Africa/Abidjan|Africa/Dakar","Africa/Abidjan|Africa/Freetown","Africa/Abidjan|Africa/Lome","Africa/Abidjan|Africa/Nouakchott","Africa/Abidjan|Africa/Ouagadougou","Africa/Abidjan|Africa/Sao_Tome","Africa/Abidjan|Africa/Timbuktu","Africa/Abidjan|Atlantic/St_Helena","Africa/Addis_Ababa|Africa/Asmara","Africa/Addis_Ababa|Africa/Asmera","Africa/Addis_Ababa|Africa/Dar_es_Salaam","Africa/Addis_Ababa|Africa/Djibouti","Africa/Addis_Ababa|Africa/Kampala","Africa/Addis_Ababa|Africa/Mogadishu","Africa/Addis_Ababa|Africa/Nairobi","Africa/Addis_Ababa|Indian/Antananarivo","Africa/Addis_Ababa|Indian/Comoro","Africa/Addis_Ababa|Indian/Mayotte","Africa/Bangui|Africa/Brazzaville","Africa/Bangui|Africa/Douala","Africa/Bangui|Africa/Kinshasa","Africa/Bangui|Africa/Lagos","Africa/Bangui|Africa/Libreville","Africa/Bangui|Africa/Luanda","Africa/Bangui|Africa/Malabo","Africa/Bangui|Africa/Niamey","Africa/Bangui|Africa/Porto-Novo","Africa/Blantyre|Africa/Bujumbura","Africa/Blantyre|Africa/Gaborone","Africa/Blantyre|Africa/Harare","Africa/Blantyre|Africa/Kigali","Africa/Blantyre|Africa/Lubumbashi","Africa/Blantyre|Africa/Lusaka","Africa/Blantyre|Africa/Maputo","Africa/Cairo|Egypt","Africa/Johannesburg|Africa/Maseru","Africa/Johannesburg|Africa/Mbabane","Africa/Juba|Africa/Khartoum","Africa/Tripoli|Libya","America/Adak|America/Atka","America/Adak|US/Aleutian","America/Anchorage|US/Alaska","America/Anguilla|America/Antigua","America/Anguilla|America/Dominica","America/Anguilla|America/Grenada","America/Anguilla|America/Guadeloupe","America/Anguilla|America/Marigot","America/Anguilla|America/Montserrat","America/Anguilla|America/Port_of_Spain","America/Anguilla|America/St_Barthelemy","America/Anguilla|America/St_Kitts","America/Anguilla|America/St_Lucia","America/Anguilla|America/St_Thomas","America/Anguilla|America/St_Vincent","America/Anguilla|America/Tortola","America/Anguilla|America/Virgin","America/Argentina/Buenos_Aires|America/Buenos_Aires","America/Argentina/Catamarca|America/Argentina/ComodRivadavia","America/Argentina/Catamarca|America/Catamarca","America/Argentina/Cordoba|America/Cordoba","America/Argentina/Cordoba|America/Rosario","America/Argentina/Jujuy|America/Jujuy","America/Argentina/Mendoza|America/Mendoza","America/Aruba|America/Curacao","America/Aruba|America/Kralendijk","America/Aruba|America/Lower_Princes","America/Atikokan|America/Coral_Harbour","America/Cayman|America/Panama","America/Chicago|US/Central","America/Denver|America/Shiprock","America/Denver|Navajo","America/Denver|US/Mountain","America/Detroit|US/Michigan","America/Edmonton|Canada/Mountain","America/Ensenada|America/Tijuana","America/Ensenada|Mexico/BajaNorte","America/Fort_Wayne|America/Indiana/Indianapolis","America/Fort_Wayne|America/Indianapolis","America/Fort_Wayne|US/East-Indiana","America/Halifax|Canada/Atlantic","America/Havana|Cuba","America/Indiana/Knox|America/Knox_IN","America/Indiana/Knox|US/Indiana-Starke","America/Jamaica|Jamaica","America/Kentucky/Louisville|America/Louisville","America/Los_Angeles|US/Pacific","America/Los_Angeles|US/Pacific-New","America/Manaus|Brazil/West","America/Mazatlan|Mexico/BajaSur","America/Mexico_City|Mexico/General","America/Montreal|America/Toronto","America/Montreal|Canada/Eastern","America/New_York|US/Eastern","America/Noronha|Brazil/DeNoronha","America/Phoenix|US/Arizona","America/Porto_Acre|America/Rio_Branco","America/Porto_Acre|Brazil/Acre","America/Regina|Canada/East-Saskatchewan","America/Regina|Canada/Saskatchewan","America/Santiago|Chile/Continental","America/Sao_Paulo|Brazil/East","America/St_Johns|Canada/Newfoundland","America/Vancouver|Canada/Pacific","America/Whitehorse|Canada/Yukon","America/Winnipeg|Canada/Central","Antarctica/McMurdo|Antarctica/South_Pole","Antarctica/McMurdo|NZ","Antarctica/McMurdo|Pacific/Auckland","Arctic/Longyearbyen|Atlantic/Jan_Mayen","Arctic/Longyearbyen|Europe/Oslo","Asia/Aden|Asia/Kuwait","Asia/Aden|Asia/Riyadh","Asia/Ashgabat|Asia/Ashkhabad","Asia/Bahrain|Asia/Qatar","Asia/Bangkok|Asia/Phnom_Penh","Asia/Bangkok|Asia/Vientiane","Asia/Calcutta|Asia/Kolkata","Asia/Chongqing|Asia/Chungking","Asia/Chongqing|Asia/Harbin","Asia/Chongqing|Asia/Shanghai","Asia/Chongqing|PRC","Asia/Dacca|Asia/Dhaka","Asia/Dubai|Asia/Muscat","Asia/Ho_Chi_Minh|Asia/Saigon","Asia/Hong_Kong|Hongkong","Asia/Istanbul|Europe/Istanbul","Asia/Istanbul|Turkey","Asia/Jerusalem|Asia/Tel_Aviv","Asia/Jerusalem|Israel","Asia/Kashgar|Asia/Urumqi","Asia/Kathmandu|Asia/Katmandu","Asia/Macao|Asia/Macau","Asia/Makassar|Asia/Ujung_Pandang","Asia/Nicosia|Europe/Nicosia","Asia/Seoul|ROK","Asia/Singapore|Singapore","Asia/Taipei|ROC","Asia/Tehran|Iran","Asia/Thimbu|Asia/Thimphu","Asia/Tokyo|Japan","Asia/Ulaanbaatar|Asia/Ulan_Bator","Atlantic/Faeroe|Atlantic/Faroe","Atlantic/Reykjavik|Iceland","Australia/ACT|Australia/Canberra","Australia/ACT|Australia/NSW","Australia/ACT|Australia/Sydney","Australia/Adelaide|Australia/South","Australia/Brisbane|Australia/Queensland","Australia/Broken_Hill|Australia/Yancowinna","Australia/Darwin|Australia/North","Australia/Hobart|Australia/Tasmania","Australia/LHI|Australia/Lord_Howe","Australia/Melbourne|Australia/Victoria","Australia/Perth|Australia/West","Chile/EasterIsland|Pacific/Easter","Eire|Europe/Dublin","Etc/GMT+0|Etc/GMT","Etc/GMT+0|Etc/GMT-0","Etc/GMT+0|Etc/GMT0","Etc/GMT+0|Etc/Greenwich","Etc/GMT+0|GMT","Etc/GMT+0|GMT+0","Etc/GMT+0|GMT-0","Etc/GMT+0|GMT0","Etc/GMT+0|Greenwich","Etc/UCT|UCT","Etc/UTC|Etc/Universal","Etc/UTC|Etc/Zulu","Etc/UTC|UTC","Etc/UTC|Universal","Etc/UTC|Zulu","Europe/Belfast|Europe/Guernsey","Europe/Belfast|Europe/Isle_of_Man","Europe/Belfast|Europe/Jersey","Europe/Belfast|Europe/London","Europe/Belfast|GB","Europe/Belfast|GB-Eire","Europe/Belgrade|Europe/Ljubljana","Europe/Belgrade|Europe/Podgorica","Europe/Belgrade|Europe/Sarajevo","Europe/Belgrade|Europe/Skopje","Europe/Belgrade|Europe/Zagreb","Europe/Bratislava|Europe/Prague","Europe/Busingen|Europe/Vaduz","Europe/Busingen|Europe/Zurich","Europe/Chisinau|Europe/Tiraspol","Europe/Helsinki|Europe/Mariehamn","Europe/Lisbon|Portugal","Europe/Moscow|W-SU","Europe/Rome|Europe/San_Marino","Europe/Rome|Europe/Vatican","Europe/Warsaw|Poland","Kwajalein|Pacific/Kwajalein","NZ-CHAT|Pacific/Chatham","Pacific/Chuuk|Pacific/Truk","Pacific/Chuuk|Pacific/Yap","Pacific/Guam|Pacific/Saipan","Pacific/Honolulu|Pacific/Johnston","Pacific/Honolulu|US/Hawaii","Pacific/Midway|Pacific/Pago_Pago","Pacific/Midway|Pacific/Samoa","Pacific/Midway|US/Samoa","Pacific/Pohnpei|Pacific/Ponape"]}),a}); \ No newline at end of file +!function(M,z){"use strict";"object"==typeof module&&module.exports?module.exports=z(require("moment")):"function"==typeof define&&define.amd?define(["moment"],z):z(M.moment)}(this,function(O){"use strict";void 0===O.version&&O.default&&(O=O.default);var z,W={},A={},c={},d={},R={},M=(O&&"string"==typeof O.version||C("Moment Timezone requires Moment.js. See https://momentjs.com/timezone/docs/#/use-it/browser/"),O.version.split(".")),b=+M[0],p=+M[1];function q(M){return 96= 2.6.0. You are using Moment.js "+O.version+". See momentjs.com"),f.prototype={_set:function(M){this.name=M.name,this.abbrs=M.abbrs,this.untils=M.untils,this.offsets=M.offsets,this.population=M.population},_index:function(M){M=function(M,z){var b,p=z.length;if(M=z[p-2])return p-1;if(M>=z[p-1])return-1;for(var O=0,A=p-1;1= 2.9.0. You are using Moment.js "+O.version+"."),O.defaultZone=M?t(M):null,O};M=O.momentProperties;return"[object Array]"===Object.prototype.toString.call(M)?(M.push("_z"),M.push("_a")):M&&(M._z=null),s({version:"2023d",zones:["Africa/Abidjan|LMT GMT|g.8 0|01|-2ldXH.Q|48e5","Africa/Nairobi|LMT +0230 EAT +0245|-2r.g -2u -30 -2J|012132|-2ua2r.g N6nV.g 3Fbu h1cu dzbJ|47e5","Africa/Algiers|LMT PMT WET WEST CET CEST|-c.c -9.l 0 -10 -10 -20|01232323232323232454542423234542324|-3bQ0c.c MDA2.P cNb9.l HA0 19A0 1iM0 11c0 1oo0 Wo0 1rc0 QM0 1EM0 UM0 DA0 Imo0 rd0 De0 9Xz0 1fb0 1ap0 16K0 2yo0 mEp0 hwL0 jxA0 11A0 dDd0 17b0 11B0 1cN0 2Dy0 1cN0 1fB0 1cL0|26e5","Africa/Lagos|LMT GMT +0030 WAT|-d.z 0 -u -10|01023|-2B40d.z 7iod.z dnXK.p dLzH.z|17e6","Africa/Bissau|LMT -01 GMT|12.k 10 0|012|-2ldX0 2xoo0|39e4","Africa/Maputo|LMT CAT|-2a.k -20|01|-2GJea.k|26e5","Africa/Cairo|LMT EET EEST|-25.9 -20 -30|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2MBC5.9 1AQM5.9 vb0 1ip0 11z0 1iN0 1nz0 12p0 1pz0 10N0 1pz0 16p0 1jz0 s3d0 Vz0 1oN0 11b0 1oO0 10N0 1pz0 10N0 1pb0 10N0 1pb0 10N0 1pb0 10N0 1pz0 10N0 1pb0 10N0 1pb0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1WL0 rd0 1Rz0 wp0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1qL0 Xd0 1oL0 11d0 1oL0 11d0 1pb0 11d0 1oL0 11d0 1oL0 11d0 1ny0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 WL0 1qN0 Rb0 1wp0 On0 1zd0 Lz0 1EN0 Fb0 c10 8n0 8Nd0 gL0 e10 mn0 kSp0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1a10 1fz0|15e6","Africa/Casablanca|LMT +00 +01|u.k 0 -10|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-2gMnt.E 130Lt.E rb0 Dd0 dVb0 b6p0 TX0 EoB0 LL0 gnd0 rz0 43d0 AL0 1Nd0 XX0 1Cp0 pz0 dEp0 4mn0 SyN0 AL0 1Nd0 wn0 1FB0 Db0 1zd0 Lz0 1Nf0 wM0 co0 go0 1o00 s00 dA0 vc0 11A0 A00 e00 y00 11A0 uM0 e00 Dc0 11A0 s00 e00 IM0 WM0 mo0 gM0 LA0 WM0 jA0 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0|32e5","Africa/Ceuta|LMT WET WEST CET CEST|l.g 0 -10 -10 -20|0121212121212121212121343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-2M0M0 GdX0 11z0 drd0 18p0 3HX0 17d0 1fz0 1a10 1io0 1a00 1y7o0 LL0 gnd0 rz0 43d0 AL0 1Nd0 XX0 1Cp0 pz0 dEp0 4VB0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|85e3","Africa/El_Aaiun|LMT -01 +00 +01|Q.M 10 0 -10|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323|-1rDz7.c 1GVA7.c 6L0 AL0 1Nd0 XX0 1Cp0 pz0 1cBB0 AL0 1Nd0 wn0 1FB0 Db0 1zd0 Lz0 1Nf0 wM0 co0 go0 1o00 s00 dA0 vc0 11A0 A00 e00 y00 11A0 uM0 e00 Dc0 11A0 s00 e00 IM0 WM0 mo0 gM0 LA0 WM0 jA0 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0 2600 e00 28M0 e00 2600 gM0|20e4","Africa/Johannesburg|LMT SAST SAST SAST|-1Q -1u -20 -30|0123232|-39EpQ qTcm 1Ajdu 1cL0 1cN0 1cL0|84e5","Africa/Juba|LMT CAT CAST EAT|-26.s -20 -30 -30|012121212121212121212121212121212131|-1yW26.s 1zK06.s 16L0 1iN0 17b0 1jd0 17b0 1ip0 17z0 1i10 17X0 1hB0 18n0 1hd0 19b0 1gp0 19z0 1iN0 17b0 1ip0 17z0 1i10 18n0 1hd0 18L0 1gN0 19b0 1gp0 19z0 1iN0 17z0 1i10 17X0 yGd0 PeX0|","Africa/Khartoum|LMT CAT CAST EAT|-2a.8 -20 -30 -30|012121212121212121212121212121212131|-1yW2a.8 1zK0a.8 16L0 1iN0 17b0 1jd0 17b0 1ip0 17z0 1i10 17X0 1hB0 18n0 1hd0 19b0 1gp0 19z0 1iN0 17b0 1ip0 17z0 1i10 18n0 1hd0 18L0 1gN0 19b0 1gp0 19z0 1iN0 17z0 1i10 17X0 yGd0 HjL0|51e5","Africa/Monrovia|LMT MMT MMT GMT|H.8 H.8 I.u 0|0123|-3ygng.Q 1usM0 28G01.m|11e5","Africa/Ndjamena|LMT WAT WAST|-10.c -10 -20|0121|-2le10.c 2J3c0.c Wn0|13e5","Africa/Sao_Tome|LMT LMT GMT WAT|-q.U A.J 0 -10|01232|-3tooq.U 18aoq.U 4i6N0 2q00|","Africa/Tripoli|LMT CET CEST EET|-Q.I -10 -20 -20|012121213121212121212121213123123|-21JcQ.I 1hnBQ.I vx0 4iP0 xx0 4eN0 Bb0 7ip0 U0n0 A10 1db0 1cN0 1db0 1dd0 1db0 1eN0 1bb0 1e10 1cL0 1c10 1db0 1dd0 1db0 1cN0 1db0 1q10 fAn0 1ep0 1db0 AKq0 TA0 1o00|11e5","Africa/Tunis|LMT PMT CET CEST|-E.I -9.l -10 -20|01232323232323232323232323232323232|-3zO0E.I 1cBAv.n 18pa9.l 1qM0 DA0 3Tc0 11B0 1ze0 WM0 7z0 3d0 14L0 1cN0 1f90 1ar0 16J0 1gXB0 WM0 1rA0 11c0 nwo0 Ko0 1cM0 1cM0 1rA0 10M0 zuM0 10N0 1aN0 1qM0 WM0 1qM0 11A0 1o00|20e5","Africa/Windhoek|LMT +0130 SAST SAST CAT WAT|-18.o -1u -20 -30 -20 -10|012324545454545454545454545454545454545454545454545454|-39Ep8.o qTbC.o 1Ajdu 1cL0 1SqL0 9Io0 16P0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0|32e4","America/Adak|LMT LMT NST NWT NPT BST BDT AHST HST HDT|-cd.m bK.C b0 a0 a0 b0 a0 a0 a0 90|01234256565656565656565656565656565678989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898|-48Pzs.L 1jVzf.p 1EX1d.m 8wW0 iB0 Qlb0 52O0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cm0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|326","America/Anchorage|LMT LMT AST AWT APT AHST AHDT YST AKST AKDT|-e0.o 9X.A a0 90 90 a0 90 90 90 80|01234256565656565656565656565656565678989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898|-48Pzs.L 1jVxs.n 1EX20.o 8wX0 iA0 Qlb0 52O0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cm0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|30e4","America/Puerto_Rico|LMT AST AWT APT|4o.p 40 30 30|01231|-2Qi7z.z 1IUbz.z 7XT0 iu0|24e5","America/Araguaina|LMT -03 -02|3c.M 30 20|0121212121212121212121212121212121212121212121212121|-2glwL.c HdKL.c 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 dMN0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 ny10 Lz0|14e4","America/Argentina/Buenos_Aires|LMT CMT -04 -03 -02|3R.M 4g.M 40 30 20|012323232323232323232323232323232323232323234343434343434343|-331U6.c 125cn pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wp0 Rb0 1wp0 TX0 A4p0 uL0 1qN0 WL0|","America/Argentina/Catamarca|LMT CMT -04 -03 -02|4n.8 4g.M 40 30 20|012323232323232323232323232323232323232323234343434243432343|-331TA.Q 125bR.E pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 rlB0 7B0 8zb0 uL0|","America/Argentina/Cordoba|LMT CMT -04 -03 -02|4g.M 4g.M 40 30 20|012323232323232323232323232323232323232323234343434243434343|-331TH.c 125c0 pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 A4p0 uL0 1qN0 WL0|","America/Argentina/Jujuy|LMT CMT -04 -03 -02|4l.c 4g.M 40 30 20|0123232323232323232323232323232323232323232343434232434343|-331TC.M 125bT.A pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1ze0 TX0 1ld0 WK0 1wp0 TX0 A4p0 uL0|","America/Argentina/La_Rioja|LMT CMT -04 -03 -02|4r.o 4g.M 40 30 20|0123232323232323232323232323232323232323232343434342343432343|-331Tw.A 125bN.o pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Qn0 qO0 16n0 Rb0 1wp0 TX0 rlB0 7B0 8zb0 uL0|","America/Argentina/Mendoza|LMT CMT -04 -03 -02|4z.g 4g.M 40 30 20|012323232323232323232323232323232323232323234343423232432343|-331To.I 125bF.w pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1u20 SL0 1vd0 Tb0 1wp0 TW0 ri10 Op0 7TX0 uL0|","America/Argentina/Rio_Gallegos|LMT CMT -04 -03 -02|4A.Q 4g.M 40 30 20|012323232323232323232323232323232323232323234343434343432343|-331Tn.8 125bD.U pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wp0 Rb0 1wp0 TX0 rlB0 7B0 8zb0 uL0|","America/Argentina/Salta|LMT CMT -04 -03 -02|4l.E 4g.M 40 30 20|0123232323232323232323232323232323232323232343434342434343|-331TC.k 125bT.8 pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 A4p0 uL0|","America/Argentina/San_Juan|LMT CMT -04 -03 -02|4y.4 4g.M 40 30 20|0123232323232323232323232323232323232323232343434342343432343|-331Tp.U 125bG.I pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Qn0 qO0 16n0 Rb0 1wp0 TX0 rld0 m10 8lb0 uL0|","America/Argentina/San_Luis|LMT CMT -04 -03 -02|4p.o 4g.M 40 30 20|0123232323232323232323232323232323232323232343434232323432323|-331Ty.A 125bP.o pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 XX0 1q20 SL0 AN0 vDb0 m10 8lb0 8L0 jd0 1qN0 WL0 1qN0|","America/Argentina/Tucuman|LMT CMT -04 -03 -02|4k.Q 4g.M 40 30 20|01232323232323232323232323232323232323232323434343424343234343|-331TD.8 125bT.U pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wq0 Ra0 1wp0 TX0 rlB0 4N0 8BX0 uL0 1qN0 WL0|","America/Argentina/Ushuaia|LMT CMT -04 -03 -02|4x.c 4g.M 40 30 20|012323232323232323232323232323232323232323234343434343432343|-331Tq.M 125bH.A pKnH.c Mn0 1iN0 Tb0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 1C10 LX0 1C10 LX0 1C10 LX0 1C10 Mn0 MN0 2jz0 MN0 4lX0 u10 5Lb0 1pB0 Fnz0 u10 uL0 1vd0 SL0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 zvd0 Bz0 1tB0 TX0 1wp0 Rb0 1wp0 Rb0 1wp0 TX0 rkN0 8p0 8zb0 uL0|","America/Asuncion|LMT AMT -04 -03|3O.E 3O.E 40 30|0123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323|-3eLw9.k 1FGo0 1DKM9.k 3CL0 3Dd0 10L0 1pB0 10n0 1pB0 10n0 1pB0 1cL0 1dd0 1db0 1dd0 1cL0 1dd0 1cL0 1dd0 1cL0 1dd0 1db0 1dd0 1cL0 1dd0 1cL0 1dd0 1cL0 1dd0 1db0 1dd0 1cL0 1lB0 14n0 1dd0 1cL0 1fd0 WL0 1rd0 1aL0 1dB0 Xz0 1qp0 Xb0 1qN0 10L0 1rB0 TX0 1tB0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 WN0 1qL0 11B0 1nX0 1ip0 WL0 1qN0 WL0 1qN0 WL0 1tB0 TX0 1tB0 TX0 1tB0 19X0 1a10 1fz0 1a10 1fz0 1cN0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0|28e5","America/Panama|LMT CMT EST|5i.8 5j.A 50|012|-3eLuF.Q Iy01.s|15e5","America/Bahia_Banderas|LMT MST CST MDT PST CDT|71 70 60 60 80 50|0121312141313131313131313131313131313152525252525252525252525252|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 otX0 gmN0 P2N0 13Vd0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nW0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0|84e3","America/Bahia|LMT -03 -02|2y.4 30 20|01212121212121212121212121212121212121212121212121212121212121|-2glxp.U HdLp.U 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 l5B0 Rb0|27e5","America/Barbados|LMT AST ADT -0330|3W.t 40 30 3u|0121213121212121|-2m4k1.v 1eAN1.v RB0 1Bz0 Op0 1rb0 11d0 1jJc0 IL0 1ip0 17b0 1ip0 17b0 1ld0 13b0|28e4","America/Belem|LMT -03 -02|3d.U 30 20|012121212121212121212121212121|-2glwK.4 HdKK.4 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0|20e5","America/Belize|LMT CST -0530 CWT CPT CDT|5Q.M 60 5u 50 50 50|012121212121212121212121212121212121212121212121213412121212121212121212121212121212121212121215151|-2kBu7.c fPA7.c Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu Rcu 7Bt0 Ni0 4nd0 Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1wou Rbu 1wou Rbu 1zcu Onu e9Au qn0 lxB0 mn0|57e3","America/Boa_Vista|LMT -04 -03|42.E 40 30|0121212121212121212121212121212121|-2glvV.k HdKV.k 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 smp0 WL0 1tB0 2L0|62e2","America/Bogota|LMT BMT -05 -04|4U.g 4U.g 50 40|01232|-3sTv3.I 1eIo0 38yo3.I 1PX0|90e5","America/Boise|LMT PST PDT MST MWT MPT MDT|7I.N 80 70 70 60 60 60|01212134536363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363636363|-3tFE0 1nEe0 1nX0 11B0 1nX0 8C10 JCL0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 Dd0 1Kn0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|21e4","America/Cambridge_Bay|-00 MST MWT MPT MDT CST CDT EST|0 70 60 60 60 60 50 50|012314141414141414141414141414141414141414141414141414141414567541414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141|-21Jc0 RO90 8x20 ix0 14HB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11A0 1nX0 2K0 WQ0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|15e2","America/Campo_Grande|LMT -04 -03|3C.s 40 30|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2glwl.w HdLl.w 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 1C10 Lz0 1Ip0 HX0 1zd0 On0 1HB0 IL0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1zd0 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1HB0 FX0|77e4","America/Cancun|LMT CST EST EDT CDT|5L.4 60 50 40 50|0123232341414141414141414141414141414141412|-1UQG0 2q2o0 yLB0 1lb0 14p0 1lb0 14p0 Lz0 xB0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 Dd0|63e4","America/Caracas|LMT CMT -0430 -04|4r.I 4r.E 4u 40|012323|-3eLvw.g ROnX.U 28KM2.k 1IwOu kqo0|29e5","America/Cayenne|LMT -04 -03|3t.k 40 30|012|-2mrwu.E 2gWou.E|58e3","America/Chicago|LMT CST CDT EST CWT CPT|5O.A 60 50 50 50 50|012121212121212121212121212121212121213121212121214512121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tFG0 1nEe0 1nX0 11B0 1nX0 1wp0 TX0 WN0 1qL0 1cN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 11B0 1Hz0 14p0 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 RB0 8x30 iw0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|92e5","America/Chihuahua|LMT MST CST MDT CDT|74.k 70 60 60 50|0121312424231313131313131313131313131313131313131313131313132|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 2zQN0 1lb0 14p0 1lb0 14q0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0|81e4","America/Ciudad_Juarez|LMT MST CST MDT CDT|75.U 70 60 60 50|0121312424231313131313131313131313131313131313131313131313132131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131213131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131313131|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 2zQN0 1lb0 14p0 1lb0 14q0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1wn0 cm0 EP0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 .1 9xX.X EP0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Costa_Rica|LMT SJMT CST CDT|5A.d 5A.d 60 50|01232323232|-3eLun.L 1fyo0 2lu0n.L Db0 1Kp0 Db0 pRB0 15b0 1kp0 mL0|12e5","America/Phoenix|LMT MST MDT MWT|7s.i 70 60 60|012121313121|-3tFF0 1nEe0 1nX0 11B0 1nX0 SgN0 4Al1 Ap0 1db0 SWqX 1cL0|42e5","America/Cuiaba|LMT -04 -03|3I.k 40 30|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2glwf.E HdLf.E 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 4a10 HX0 1zd0 On0 1HB0 IL0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1zd0 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1HB0 FX0|54e4","America/Danmarkshavn|LMT -03 -02 GMT|1e.E 30 20 0|01212121212121212121212121212121213|-2a5WJ.k 2z5fJ.k 19U0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 DC0|8","America/Dawson_Creek|LMT PST PDT PWT PPT MST|80.U 80 70 70 70 70|01213412121212121212121212121212121212121212121212121212125|-3tofX.4 1nspX.4 1in0 UGp0 8x10 iy0 3NB0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 ML0|12e3","America/Dawson|LMT YST YDT YWT YPT YDDT PST PDT MST|9h.E 90 80 80 80 70 80 70 70|0121213415167676767676767676767676767676767676767676767676767676767676767676767676767676767678|-2MSeG.k GWpG.k 1in0 1o10 13V0 Ser0 8x00 iz0 LCL0 1fA0 jrA0 fNd0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1z90|13e2","America/Denver|LMT MST MDT MWT MPT|6X.U 70 60 60 60|012121212134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tFF0 1nEe0 1nX0 11B0 1nX0 11B0 1qL0 WN0 mn0 Ord0 8x20 ix0 LCN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|26e5","America/Detroit|LMT CST EST EWT EPT EDT|5w.b 60 50 40 40 40|0123425252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252|-2Cgir.N peqr.N 156L0 8x40 iv0 6fd0 11z0 JxX1 SMX 1cN0 1cL0 aW10 1cL0 s10 1Vz0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|37e5","America/Edmonton|LMT MST MDT MWT MPT|7x.Q 70 60 60 60|0121212121212134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2yd4q.8 shdq.8 1in0 17d0 hz0 2dB0 1fz0 1a10 11z0 1qN0 WL0 1qN0 11z0 IGN0 8x20 ix0 3NB0 11z0 XQp0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|10e5","America/Eirunepe|LMT -05 -04|4D.s 50 40|0121212121212121212121212121212121|-2glvk.w HdLk.w 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 dPB0 On0 yTd0 d5X0|31e3","America/El_Salvador|LMT CST CDT|5U.M 60 50|012121|-1XiG3.c 2Fvc3.c WL0 1qN0 WL0|11e5","America/Tijuana|LMT MST PST PDT PWT PPT|7M.4 70 80 70 70 70|012123245232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1UQF0 4Q00 8mM0 8lc0 SN0 1cL0 pHB0 83r0 zI0 5O10 1Rz0 cOO0 11A0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 BUp0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|20e5","America/Fort_Nelson|LMT PST PDT PWT PPT MST|8a.L 80 70 70 70 70|012134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121215|-3tofN.d 1nspN.d 1in0 UGp0 8x10 iy0 3NB0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0|39e2","America/Fort_Wayne|LMT CST CDT CWT CPT EST EDT|5I.C 60 50 50 50 50 40|0121212134121212121212121212151565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 QI10 Db0 RB0 8x30 iw0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 5Tz0 1o10 qLb0 1cL0 1cN0 1cL0 1qhd0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Fortaleza|LMT -03 -02|2y 30 20|0121212121212121212121212121212121212121|-2glxq HdLq 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 nsp0 WL0 1tB0 5z0 2mN0 On0|34e5","America/Glace_Bay|LMT AST ADT AWT APT|3X.M 40 30 30 30|012134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2IsI0.c CwO0.c 1in0 UGp0 8x50 iu0 iq10 11z0 Jg10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|19e3","America/Godthab|LMT -03 -02 -01|3q.U 30 20 10|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-2a5Ux.4 2z5dx.4 19U0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 2so0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|17e3","America/Goose_Bay|LMT NST NDT NST NDT NWT NPT AST ADT ADDT|41.E 3u.Q 2u.Q 3u 2u 2u 2u 40 30 20|0121343434343434356343434343434343434343434343434343434343437878787878787878787878787878787878787878787879787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787|-3tojW.k 1nspt.c 1in0 DXb0 2HbX.8 WL0 1qN0 WL0 1qN0 WL0 1tB0 TX0 1tB0 WL0 1qN0 WL0 1qN0 7UHu itu 1tB0 WL0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1tB0 WL0 1ld0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 S10 g0u 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14n1 1lb0 14p0 1nW0 11C0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zcX Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|76e2","America/Grand_Turk|LMT KMT EST EDT AST|4I.w 57.a 50 40 40|01232323232323232323232323232323232323232323232323232323232323232323232323243232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3eLvf.s RK0m.C 2HHBQ.O 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 7jA0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|37e2","America/Guatemala|LMT CST CDT|62.4 60 50|0121212121|-24KhV.U 2efXV.U An0 mtd0 Nz0 ifB0 17b0 zDB0 11z0|13e5","America/Guayaquil|LMT QMT -05 -04|5j.k 5e 50 40|01232|-3eLuE.E 1DNzS.E 2uILK rz0|27e5","America/Guyana|LMT -04 -0345 -03|3Q.D 40 3J 30|01231|-2mf87.l 8Hc7.l 2r7bJ Ey0f|80e4","America/Halifax|LMT AST ADT AWT APT|4e.o 40 30 30 30|0121212121212121212121212121212121212121212121212134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2IsHJ.A xzzJ.A 1db0 3I30 1in0 3HX0 IL0 1E10 ML0 1yN0 Pb0 1Bd0 Mn0 1Bd0 Rz0 1w10 Xb0 1w10 LX0 1w10 Xb0 1w10 Lz0 1C10 Jz0 1E10 OL0 1yN0 Un0 1qp0 Xb0 1qp0 11X0 1w10 Lz0 1HB0 LX0 1C10 FX0 1w10 Xb0 1qp0 Xb0 1BB0 LX0 1td0 Xb0 1qp0 Xb0 Rf0 8x50 iu0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 3Qp0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 3Qp0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 6i10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|39e4","America/Havana|LMT HMT CST CDT|5t.s 5t.A 50 40|0123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3eLuu.w 1qx00.8 72zu.o ML0 sld0 An0 1Nd0 Db0 1Nd0 An0 6Ep0 An0 1Nd0 An0 JDd0 Mn0 1Ap0 On0 1fd0 11X0 1qN0 WL0 1wp0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 14n0 1ld0 14L0 1kN0 15b0 1kp0 1cL0 1cN0 1fz0 1a10 1fz0 1fB0 11z0 14p0 1nX0 11B0 1nX0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 1a10 1in0 1a10 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 17c0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 11A0 6i00 Rc0 1wo0 U00 1tA0 Rc0 1wo0 U00 1wo0 U00 1zc0 U00 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0|21e5","America/Hermosillo|LMT MST CST MDT PST|7n.Q 70 60 60 80|0121312141313131|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 otX0 gmN0 P2N0 13Vd0 1lb0 14p0 1lb0 14p0 1lb0|64e4","America/Indiana/Knox|LMT CST CDT CWT CPT EST|5K.u 60 50 50 50 50|01212134121212121212121212121212121212151212121212121212121212121212121212121212121212121252121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 3NB0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 11z0 1o10 11z0 1o10 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 3Cn0 8wp0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 z8o0 1o00 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Indiana/Marengo|LMT CST CDT CWT CPT EST EDT|5J.n 60 50 50 50 50 40|01212134121212121212121215656565656525656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 dyN0 11z0 6fd0 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 jrz0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1VA0 LA0 1BX0 1e6p0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Indiana/Petersburg|LMT CST CDT CWT CPT EST EDT|5N.7 60 50 50 50 50 40|012121341212121212121212121215121212121212121212121252125656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 njX0 WN0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 3Fb0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 19co0 1o00 Rd0 1zb0 Oo0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Indiana/Tell_City|LMT CST CDT CWT CPT EST EDT|5L.3 60 50 50 50 50 40|012121341212121212121212121512165652121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 njX0 WN0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 8wn0 1cN0 1cL0 1cN0 1cK0 1cN0 1cL0 1qhd0 1o00 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Indiana/Vevay|LMT CST CDT CWT CPT EST EDT|5E.g 60 50 50 50 50 40|0121213415656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 kPB0 Awn0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1lnd0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Indiana/Vincennes|LMT CST CDT CWT CPT EST EDT|5O.7 60 50 50 50 50 40|012121341212121212121212121212121565652125656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 g0p0 11z0 1o10 11z0 1qL0 WN0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 caL0 1cL0 1cN0 1cL0 1qhd0 1o00 Rd0 1zb0 Oo0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Indiana/Winamac|LMT CST CDT CWT CPT EST EDT|5K.p 60 50 50 50 50 40|012121341212121212121212121212121212121565652165656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 jrz0 1cL0 1cN0 1cL0 1qhd0 1o00 Rd0 1za0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Inuvik|-00 PST PDT MDT MST|0 80 70 60 70|01212121212121213434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-FnA0 L3K0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cK0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|35e2","America/Iqaluit|-00 EWT EPT EST EDT CST CDT|0 40 40 50 40 60 50|0123434343434343434343434343434343434343434343434343434343456343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-16K00 7nX0 iv0 14HB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11C0 1nX0 11A0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|67e2","America/Jamaica|LMT KMT EST EDT|57.a 57.a 50 40|01232323232323232323232|-3eLuQ.O RK00 2uM1Q.O 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0|94e4","America/Juneau|LMT LMT PST PWT PPT PDT YDT YST AKST AKDT|-f2.j 8V.F 80 70 70 70 80 90 90 80|0123425252525252525252525252625252578989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898|-48Pzs.L 1jVwq.s 1EX12.j 8x10 iy0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cM0 1cM0 1cL0 1cN0 1fz0 1a10 1fz0 co0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|33e3","America/Kentucky/Louisville|LMT CST CDT CWT CPT EST EDT|5H.2 60 50 50 50 50 40|01212121213412121212121212121212121212565656565656525656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 3Fd0 Nb0 LPd0 11z0 RB0 8x30 iw0 1nX1 e0X 9vd0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 xz0 gso0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1VA0 LA0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Kentucky/Monticello|LMT CST CDT CWT CPT EST EDT|5D.o 60 50 50 50 50 40|01212134121212121212121212121212121212121212121212121212121212121212121212565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFG0 1nEe0 1nX0 11B0 1nX0 SgN0 8x30 iw0 SWp0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11A0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/La_Paz|LMT CMT BST -04|4w.A 4w.A 3w.A 40|0123|-3eLvr.o 1FIo0 13b0|19e5","America/Lima|LMT LMT -05 -04|58.c 58.A 50 40|01232323232323232|-3eLuP.M JcM0.o 1bDzP.o zX0 1aN0 1cL0 1cN0 1cL0 1PrB0 zX0 1O10 zX0 6Gp0 zX0 98p0 zX0|11e6","America/Los_Angeles|LMT PST PDT PWT PPT|7Q.W 80 70 70 70|0121213412121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tFE0 1nEe0 1nX0 11B0 1nX0 SgN0 8x10 iy0 5Wp1 1VaX 3dA0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1a00 1fA0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|15e6","America/Maceio|LMT -03 -02|2m.Q 30 20|012121212121212121212121212121212121212121|-2glxB.8 HdLB.8 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 dMN0 Lz0 8Q10 WL0 1tB0 5z0 2mN0 On0|93e4","America/Managua|LMT MMT CST EST CDT|5J.8 5J.c 60 50 50|01232424232324242|-3eLue.Q 1Mhc0.4 1yAMe.M 4mn0 9Up0 Dz0 1K10 Dz0 s3F0 1KH0 DB0 9In0 k8p0 19X0 1o30 11y0|22e5","America/Manaus|LMT -04 -03|40.4 40 30|01212121212121212121212121212121|-2glvX.U HdKX.U 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 dPB0 On0|19e5","America/Martinique|LMT FFMT AST ADT|44.k 44.k 40 30|01232|-3eLvT.E PTA0 2LPbT.E 19X0|39e4","America/Matamoros|LMT CST CDT|6u 60 50|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1UQG0 2FjC0 1nX0 i6p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|45e4","America/Mazatlan|LMT MST CST MDT PST|75.E 70 60 60 80|0121312141313131313131313131313131313131313131313131313131313131|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 otX0 gmN0 P2N0 13Vd0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0|44e4","America/Menominee|LMT CST CDT CWT CPT EST|5O.r 60 50 50 50 50|012121341212152121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3pdG9.x 1jce9.x 1nX0 11B0 1nX0 SgN0 8x30 iw0 1o10 11z0 LCN0 1fz0 6410 9Jb0 1cM0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|85e2","America/Merida|LMT CST EST CDT|5W.s 60 50 50|0121313131313131313131313131313131313131313131313131313131|-1UQG0 2q2o0 2hz0 wu30 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0|11e5","America/Metlakatla|LMT LMT PST PWT PPT PDT AKST AKDT|-fd.G 8K.i 80 70 70 70 90 80|0123425252525252525252525252525252526767672676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-48Pzs.L 1jVwf.5 1EX1d.G 8x10 iy0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1hU10 Rd0 1zb0 Op0 1zb0 Op0 1zb0 uM0 jB0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|14e2","America/Mexico_City|LMT MST CST MDT CDT CWT|6A.A 70 60 60 50 50|012131242425242424242424242424242424242424242424242424242424242424242|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 gEn0 TX0 3xd0 Jb0 6zB0 SL0 e5d0 17b0 1Pff0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0|20e6","America/Miquelon|LMT AST -03 -02|3I.E 40 30 20|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-2mKkf.k 2LTAf.k gQ10 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|61e2","America/Moncton|LMT EST AST ADT AWT APT|4j.8 50 40 30 30 30|0123232323232323232323245232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3txvE.Q J4ME.Q CwN0 1in0 zAo0 An0 1Nd0 An0 1Nd0 An0 1Nd0 An0 1Nd0 An0 1Nd0 An0 1K10 Lz0 1zB0 NX0 1u10 Wn0 S20 8x50 iu0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 3Cp0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14n1 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 ReX 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|64e3","America/Monterrey|LMT CST CDT|6F.g 60 50|0121212121212121212121212121212121212121212121212121212121|-1UQG0 2FjC0 1nX0 i6p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0|41e5","America/Montevideo|LMT MMT -04 -03 -0330 -0230 -02 -0130|3I.P 3I.P 40 30 3u 2u 20 1u|012343434343434343434343435353636353636375363636363636363636363636363636363636363636363|-2tRUf.9 sVc0 8jcf.9 1db0 1dcu 1cLu 1dcu 1cLu ircu 11zu 1o0u 11zu 1o0u 11zu 1o0u 11zu 1qMu WLu 1qMu WLu 1fAu 1cLu 1o0u 11zu NAu 3jXu zXu Dq0u 19Xu pcu jz0 cm10 19X0 6tB0 1fbu 3o0u jX0 4vB0 xz0 3Cp0 mmu 1a10 IMu Db0 4c10 uL0 1Nd0 An0 1SN0 uL0 mp0 28L0 iPB0 un0 1SN0 xz0 1zd0 Lz0 1zd0 Rb0 1zd0 On0 1wp0 Rb0 s8p0 1fB0 1ip0 11z0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 14n0 1ld0 14n0 1ld0 14n0 1o10 11z0 1o10 11z0 1o10 11z0|17e5","America/Toronto|LMT EST EDT EWT EPT|5h.w 50 40 40 40|012121212121212121212121212121212121212121212123412121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-32B6G.s UFdG.s 1in0 11Wu 1nzu 1fD0 WJ0 1wr0 Nb0 1Ap0 On0 1zd0 On0 1wp0 TX0 1tB0 TX0 1tB0 TX0 1tB0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 4kM0 8x40 iv0 1o10 11z0 1nX0 11z0 1o10 11z0 1o10 1qL0 11D0 1nX0 11B0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|65e5","America/New_York|LMT EST EDT EWT EPT|4U.2 50 40 40 40|012121212121212121212121212121212121212121212121213412121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tFH0 1nEe0 1nX0 11B0 1nX0 11B0 1qL0 1a10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 RB0 8x40 iv0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|21e6","America/Nome|LMT LMT NST NWT NPT BST BDT YST AKST AKDT|-cW.m b1.C b0 a0 a0 b0 a0 90 90 80|01234256565656565656565656565656565678989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898989898|-48Pzs.L 1jVyu.p 1EX1W.m 8wW0 iB0 Qlb0 52O0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cl0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|38e2","America/Noronha|LMT -02 -01|29.E 20 10|0121212121212121212121212121212121212121|-2glxO.k HdKO.k 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 nsp0 WL0 1tB0 2L0 2pB0 On0|30e2","America/North_Dakota/Beulah|LMT MST MDT MWT MPT CST CDT|6L.7 70 60 60 60 60 50|0121213412121212121212121212121212121212121212121212121212121212121212121212121212121212121212125656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFF0 1nEe0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Oo0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/North_Dakota/Center|LMT MST MDT MWT MPT CST CDT|6J.c 70 60 60 60 60 50|0121213412121212121212121212121212121212121212121212121212125656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFF0 1nEe0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14o0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/North_Dakota/New_Salem|LMT MST MDT MWT MPT CST CDT|6J.D 70 60 60 60 60 50|0121213412121212121212121212121212121212121212121212121212121212121212121212121212565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tFF0 1nEe0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14o0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","America/Ojinaga|LMT MST CST MDT CDT|6V.E 70 60 60 50|0121312424231313131313131313131313131313131313131313131313132424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242|-1UQF0 deL0 8lc0 17c0 10M0 1dd0 2zQN0 1lb0 14p0 1lb0 14q0 1lb0 14p0 1nX0 11B0 1nX0 1fB0 WL0 1fB0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 U10 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1wn0 Rc0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|23e3","America/Paramaribo|LMT PMT PMT -0330 -03|3E.E 3E.Q 3E.A 3u 30|01234|-2nDUj.k Wqo0.c qanX.I 1yVXN.o|24e4","America/Port-au-Prince|LMT PPMT EST EDT|4N.k 4N 50 40|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3eLva.E 15RLX.E 2FnMb 19X0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14q0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 i6n0 1nX0 11B0 1nX0 d430 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 3iN0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|23e5","America/Rio_Branco|LMT -05 -04|4v.c 50 40|01212121212121212121212121212121|-2glvs.M HdLs.M 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 NBd0 d5X0|31e4","America/Porto_Velho|LMT -04 -03|4f.A 40 30|012121212121212121212121212121|-2glvI.o HdKI.o 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0|37e4","America/Punta_Arenas|LMT SMT -05 -04 -03|4H.E 4G.J 50 40 30|01213132323232323232343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-3eLvg.k MJbX.5 fJAh.f 5knG.J 1Vzh.f jRAG.J 1pbh.f 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 nHX0 op0 blz0 ko0 Qeo0 WL0 1zd0 On0 1ip0 11z0 1o10 11z0 1qN0 WL0 1ld0 14n0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 46n0 Ap0|","America/Winnipeg|LMT CST CDT CWT CPT|6s.A 60 50 50 50|0121212134121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3kLtv.o 1a3bv.o WL0 3ND0 1in0 Jap0 Rb0 aCN0 8x30 iw0 1tB0 11z0 1ip0 11z0 1o10 11z0 1o10 11z0 1rd0 10L0 1op0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 1cL0 1cN0 11z0 6i10 WL0 6i10 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1a00 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1a00 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1o00 14o0 1lc0 14o0 1lc0 14o0 1o00 11A0 1o00 11A0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|66e4","America/Rankin_Inlet|-00 CST CDT EST|0 60 50 50|01212121212121212121212121212121212121212121212121212121212321212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-vDc0 Bjk0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|26e2","America/Recife|LMT -03 -02|2j.A 30 20|0121212121212121212121212121212121212121|-2glxE.o HdLE.o 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 nsp0 WL0 1tB0 2L0 2pB0 On0|33e5","America/Regina|LMT MST MDT MWT MPT CST|6W.A 70 60 60 60 60|012121212121212121212121341212121212121212121212121215|-2AD51.o uHe1.o 1in0 s2L0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 66N0 1cL0 1cN0 19X0 1fB0 1cL0 1fB0 1cL0 1cN0 1cL0 M30 8x20 ix0 1ip0 1cL0 1ip0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 3NB0 1cL0 1cN0|19e4","America/Resolute|-00 CST CDT EST|0 60 50 50|01212121212121212121212121212121212121212121212121212121212321212121212321212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-SnA0 103I0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|229","America/Santarem|LMT -04 -03|3C.M 40 30|0121212121212121212121212121212|-2glwl.c HdLl.c 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 qe10 xb0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 NBd0|21e4","America/Santiago|LMT SMT -05 -04 -03|4G.J 4G.J 50 40 30|0121313232323232323432343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-3eLvh.f MJc0 fJAh.f 5knG.J 1Vzh.f jRAG.J 1pbh.f 11d0 1oL0 11d0 1oL0 11d0 1oL0 11d0 1pb0 11d0 nHX0 op0 9Bz0 hX0 1q10 ko0 Qeo0 WL0 1zd0 On0 1ip0 11z0 1o10 11z0 1qN0 WL0 1ld0 14n0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 46n0 Ap0 1Nb0 Ap0 1Nb0 Ap0 1zb0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0|62e5","America/Santo_Domingo|LMT SDMT EST EDT -0430 AST|4D.A 4E 50 40 4u 40|012324242424242525|-3eLvk.o 1Jic0.o 1lJMk Mn0 6sp0 Lbu 1Cou yLu 1RAu wLu 1QMu xzu 1Q0u xXu 1PAu 13jB0 e00|29e5","America/Sao_Paulo|LMT -03 -02|36.s 30 20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2glwR.w HdKR.w 1cc0 1e10 1bX0 Ezd0 So0 1vA0 Mn0 1BB0 ML0 1BB0 zX0 pTd0 PX0 2ep0 nz0 1C10 zX0 1C10 LX0 1C10 Mn0 H210 Rb0 1tB0 IL0 1Fd0 FX0 1EN0 FX0 1HB0 Lz0 1EN0 Lz0 1C10 IL0 1HB0 Db0 1HB0 On0 1zd0 On0 1zd0 Lz0 1zd0 Rb0 1wN0 Wn0 1tB0 Rb0 1tB0 WL0 1tB0 Rb0 1zd0 On0 1HB0 FX0 1C10 Lz0 1Ip0 HX0 1zd0 On0 1HB0 IL0 1wp0 On0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 Rb0 1zd0 Lz0 1C10 Lz0 1C10 On0 1zd0 On0 1zd0 On0 1zd0 On0 1HB0 FX0|20e6","America/Scoresbysund|LMT -02 -01 +00|1r.Q 20 10 0|012132323232323232323232323232323232323232323232323232323232323232323232323232323232323232121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2a5Ww.8 2z5ew.8 1a00 1cK0 1cL0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 2pA0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|452","America/Sitka|LMT LMT PST PWT PPT PDT YST AKST AKDT|-eW.L 91.d 80 70 70 70 90 90 80|0123425252525252525252525252525252567878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787878787|-48Pzs.L 1jVwu 1EX0W.L 8x10 iy0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 co0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|90e2","America/St_Johns|LMT NST NDT NST NDT NWT NPT NDDT|3u.Q 3u.Q 2u.Q 3u 2u 2u 2u 1u|012121212121212121212121212121212121213434343434343435634343434343434343434343434343434343434343434343434343434343434343434343434343434343437343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-3tokt.8 1l020 14L0 1nB0 1in0 1gm0 Dz0 1JB0 1cL0 1cN0 1cL0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 1cL0 1cN0 1cL0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 19X0 1fB0 1cL0 1fB0 19X0 1fB0 19X0 10O0 eKX.8 19X0 1iq0 WL0 1qN0 WL0 1qN0 WL0 1tB0 TX0 1tB0 WL0 1qN0 WL0 1qN0 7UHu itu 1tB0 WL0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1tB0 WL0 1ld0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14n1 1lb0 14p0 1nW0 11C0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zcX Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|11e4","America/Swift_Current|LMT MST MDT MWT MPT CST|7b.k 70 60 60 60 60|012134121212121212121215|-2AD4M.E uHdM.E 1in0 UGp0 8x20 ix0 1o10 17b0 1ip0 11z0 1o10 11z0 1o10 11z0 isN0 1cL0 3Cp0 1cL0 1cN0 11z0 1qN0 WL0 pMp0|16e3","America/Tegucigalpa|LMT CST CDT|5M.Q 60 50|01212121|-1WGGb.8 2ETcb.8 WL0 1qN0 WL0 GRd0 AL0|11e5","America/Thule|LMT AST ADT|4z.8 40 30|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2a5To.Q 31NBo.Q 1cL0 1cN0 1cL0 1fB0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|656","America/Vancouver|LMT PST PDT PWT PPT|8c.s 80 70 70 70|01213412121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3tofL.w 1nspL.w 1in0 UGp0 8x10 iy0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|23e5","America/Whitehorse|LMT YST YDT YWT YPT YDDT PST PDT MST|90.c 90 80 80 80 70 80 70 70|0121213415167676767676767676767676767676767676767676767676767676767676767676767676767676767678|-2MSeX.M GWpX.M 1in0 1o10 13V0 Ser0 8x00 iz0 LCL0 1fA0 LA0 ytd0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1z90|23e3","America/Yakutat|LMT LMT YST YWT YPT YDT AKST AKDT|-eF.5 9i.T 90 80 80 80 90 80|0123425252525252525252525252525252526767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676767676|-48Pzs.L 1jVwL.G 1EX1F.5 8x00 iz0 Vo10 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 cn0 10q0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|642","Antarctica/Casey|-00 +08 +11|0 -80 -b0|012121212121212121|-2q00 1DjS0 T90 40P0 KL0 blz0 3m10 1o30 14k0 1kr0 12l0 1o01 14kX 1lf1 14kX 1lf1 13bX|10","Antarctica/Davis|-00 +07 +05|0 -70 -50|01012121|-vyo0 iXt0 alj0 1D7v0 VB0 3Wn0 KN0|70","Pacific/Port_Moresby|LMT PMMT +10|-9M.E -9M.w -a0|012|-3D8VM.E AvA0.8|25e4","Antarctica/Macquarie|-00 AEST AEDT|0 -a0 -b0|0121012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-2OPc0 Fb40 1a00 4SK0 1ayy0 Lvs0 1cM0 1o00 Rc0 1wo0 Rc0 1wo0 U00 1wo0 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1qM0 WM0 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1wo0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 11A0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 11A0 1o00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1cM0 1cM0 3Co0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|1","Antarctica/Mawson|-00 +06 +05|0 -60 -50|012|-CEo0 2fyk0|60","Pacific/Auckland|LMT NZMT NZST NZST NZDT|-bD.4 -bu -cu -c0 -d0|012131313131313131313131313134343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-46jLD.4 2nEO9.4 Lz0 1tB0 11zu 1o0u 11zu 1o0u 11zu 1o0u 14nu 1lcu 14nu 1lcu 1lbu 11Au 1nXu 11Au 1nXu 11Au 1nXu 11Au 1nXu 11Au 1qLu WMu 1qLu 11Au 1n1bu IM0 1C00 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1qM0 14o0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1io0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00|14e5","Antarctica/Palmer|-00 -03 -04 -02|0 30 40 20|0121212121213121212121212121212121212121212121212121212121212121212121212121212121|-cao0 nD0 1vd0 SL0 1vd0 17z0 1cN0 1fz0 1cN0 1cL0 1cN0 asn0 Db0 jsN0 14N0 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 46n0 Ap0|40","Antarctica/Rothera|-00 -03|0 30|01|gOo0|130","Asia/Riyadh|LMT +03|-36.Q -30|01|-TvD6.Q|57e5","Antarctica/Troll|-00 +00 +02|0 0 -20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|1puo0 hd0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|40","Antarctica/Vostok|-00 +07 +05|0 -70 -50|01012|-tjA0 1rWh0 1Nj0 1aTv0|25","Europe/Berlin|LMT CET CEST CEMT|-R.s -10 -20 -30|012121212121212321212321212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-36RcR.s UbWR.s 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 kL0 Nc0 m10 WM0 1ao0 1cp0 dX0 jz0 Dd0 1io0 17c0 1fA0 1a00 1ehA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|41e5","Asia/Almaty|LMT +05 +06 +07|-57.M -50 -60 -70|012323232323232323232321232323232323232323232323232|-1Pc57.M eUo7.M 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0|15e5","Asia/Amman|LMT EET EEST +03|-2n.I -20 -30 -30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212123|-1yW2n.I 1HiMn.I KL0 1oN0 11b0 1oN0 11b0 1pd0 1dz0 1cp0 11b0 1op0 11b0 fO10 1db0 1e10 1cL0 1cN0 1cL0 1cN0 1fz0 1pd0 10n0 1ld0 14n0 1hB0 15b0 1ip0 19X0 1cN0 1cL0 1cN0 17b0 1ld0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1So0 y00 1fc0 1dc0 1co0 1dc0 1cM0 1cM0 1cM0 1o00 11A0 1lc0 17c0 1cM0 1cM0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 4bX0 Dd0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 LA0 1C00|25e5","Asia/Anadyr|LMT +12 +13 +14 +11|-bN.U -c0 -d0 -e0 -b0|01232121212121212121214121212121212121212121212121212121212141|-1PcbN.U eUnN.U 23CL0 1db0 2q10 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 2sp0 WM0|13e3","Asia/Aqtau|LMT +04 +05 +06|-3l.4 -40 -50 -60|012323232323232323232123232312121212121212121212|-1Pc3l.4 eUnl.4 24PX0 2pX0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0|15e4","Asia/Aqtobe|LMT +04 +05 +06|-3M.E -40 -50 -60|0123232323232323232321232323232323232323232323232|-1Pc3M.E eUnM.E 23CL0 3Db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0|27e4","Asia/Ashgabat|LMT +04 +05 +06|-3R.w -40 -50 -60|0123232323232323232323212|-1Pc3R.w eUnR.w 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0|41e4","Asia/Atyrau|LMT +03 +05 +06 +04|-3r.I -30 -50 -60 -40|01232323232323232323242323232323232324242424242|-1Pc3r.I eUor.I 24PW0 2pX0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 2sp0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0|","Asia/Baghdad|LMT BMT +03 +04|-2V.E -2V.A -30 -40|0123232323232323232323232323232323232323232323232323232|-3eLCV.E 18ao0.4 2ACnV.A 11b0 1cp0 1dz0 1dd0 1db0 1cN0 1cp0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1de0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0 1dc0 1dc0 1cM0 1dc0 1cM0 1dc0 1cM0 1dc0|66e5","Asia/Qatar|LMT +04 +03|-3q.8 -40 -30|012|-21Jfq.8 27BXq.8|96e4","Asia/Baku|LMT +03 +04 +05|-3j.o -30 -40 -50|01232323232323232323232123232323232323232323232323232323232323232|-1Pc3j.o 1jUoj.o WCL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 1cM0 9Je0 1o00 11z0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|27e5","Asia/Bangkok|LMT BMT +07|-6G.4 -6G.4 -70|012|-3D8SG.4 1C000|15e6","Asia/Barnaul|LMT +06 +07 +08|-5z -60 -70 -80|0123232323232323232323212323232321212121212121212121212121212121212|-21S5z pCnz 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 p90 LE0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3rd0|","Asia/Beirut|LMT EET EEST|-2m -20 -30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3D8Om 1BWom 1on0 1410 1db0 19B0 1in0 1ip0 WL0 1lQp0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 11b0 q6N0 En0 1oN0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 11b0 1op0 11b0 dA10 17b0 1iN0 17b0 1iN0 17b0 1iN0 17b0 1vB0 SL0 1mp0 13z0 1iN0 17b0 1iN0 17b0 1jd0 12n0 1a10 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0|22e5","Asia/Bishkek|LMT +05 +06 +07|-4W.o -50 -60 -70|012323232323232323232321212121212121212121212121212|-1Pc4W.o eUnW.o 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2e00 1tX0 17b0 1ip0 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1cPu 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0|87e4","Asia/Brunei|LMT +0730 +08 +0820 +09|-7l.k -7u -80 -8k -90|0123232323232323242|-1KITl.k gDbP.k 6ynu AnE 1O0k AnE 1NAk AnE 1NAk AnE 1NAk AnE 1O0k AnE 1NAk AnE pAk 8Fz0|42e4","Asia/Kolkata|LMT HMT MMT IST +0630|-5R.s -5R.k -5l.a -5u -6u|01234343|-4Fg5R.s BKo0.8 1rDcw.a 1r2LP.a 1un0 HB0 7zX0|15e6","Asia/Chita|LMT +08 +09 +10|-7x.Q -80 -90 -a0|012323232323232323232321232323232323232323232323232323232323232312|-21Q7x.Q pAnx.Q 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3re0|33e4","Asia/Choibalsan|LMT +07 +08 +10 +09|-7C -70 -80 -a0 -90|0123434343434343434343434343434343434343434343424242|-2APHC 2UkoC cKn0 1da0 1dd0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 6hD0 11z0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 3Db0 h1f0 1cJ0 1cP0 1cJ0|38e3","Asia/Shanghai|LMT CST CDT|-85.H -80 -90|012121212121212121212121212121|-2M0U5.H Iuo5.H 18n0 OjB0 Rz0 11d0 1wL0 A10 8HX0 1G10 Tz0 1ip0 1jX0 1cN0 11b0 1oN0 aL0 1tU30 Rb0 1o10 11z0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0|23e6","Asia/Colombo|LMT MMT +0530 +06 +0630|-5j.o -5j.w -5u -60 -6u|012342432|-3D8Rj.o 13inX.Q 1rFbN.w 1zzu 7Apu 23dz0 11zu n3cu|22e5","Asia/Dhaka|LMT HMT +0630 +0530 +06 +07|-61.E -5R.k -6u -5u -60 -70|01232454|-3eLG1.E 26008.k 1unn.k HB0 m6n0 2kxbu 1i00|16e6","Asia/Damascus|LMT EET EEST +03|-2p.c -20 -30 -30|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212123|-21Jep.c Hep.c 17b0 1ip0 17b0 1ip0 17b0 1ip0 19X0 1xRB0 11X0 1oN0 10L0 1pB0 11b0 1oN0 10L0 1mp0 13X0 1oN0 11b0 1pd0 11b0 1oN0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 11b0 1oN0 11b0 1oN0 11b0 1pd0 11b0 1oN0 Nb0 1AN0 Nb0 bcp0 19X0 1gp0 19X0 3ld0 1xX0 Vd0 1Bz0 Sp0 1vX0 10p0 1dz0 1cN0 1cL0 1db0 1db0 1g10 1an0 1ap0 1db0 1fd0 1db0 1cN0 1db0 1dd0 1db0 1cp0 1dz0 1c10 1dX0 1cN0 1db0 1dd0 1db0 1cN0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1db0 1cN0 1db0 1cN0 19z0 1fB0 1qL0 11B0 1on0 Wp0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0|26e5","Asia/Dili|LMT +08 +09|-8m.k -80 -90|01212|-2le8m.k 1dnXm.k 1nfA0 Xld0|19e4","Asia/Dubai|LMT +04|-3F.c -40|01|-21JfF.c|39e5","Asia/Dushanbe|LMT +05 +06 +07|-4z.c -50 -60 -70|012323232323232323232321|-1Pc4z.c eUnz.c 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2hB0|76e4","Asia/Famagusta|LMT EET EEST +03|-2f.M -20 -30 -30|0121212121212121212121212121212121212121212121212121212121212121212121212121212121212312121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1Vc2f.M 2a3cf.M 1cL0 1qp0 Xz0 19B0 19X0 1fB0 1db0 1cp0 1cL0 1fB0 19X0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1o30 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 15U0 2Ks0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|","Asia/Gaza|LMT EET EEST IST IDT|-2h.Q -20 -30 -20 -30|012121212121212121212121212121212123434343434343434343434343434343121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2MBCh.Q 1Azeh.Q MM0 iM0 4JA0 10o0 1pA0 10M0 1pA0 16o0 1jA0 16o0 1jA0 pBa0 Vz0 1oN0 11b0 1oO0 10N0 1pz0 10N0 1pb0 10N0 1pb0 10N0 1pb0 10N0 1pz0 10N0 1pb0 10N0 1pb0 11d0 1oL0 dW0 hfB0 Db0 1fB0 Rb0 bXB0 gM0 8Q00 IM0 1wo0 TX0 1HB0 IL0 1s10 10n0 1o10 WL0 1zd0 On0 1ld0 11z0 1o10 14n0 1o10 14n0 1nd0 12n0 1nd0 Xz0 1q10 12n0 M10 C00 17c0 1io0 17c0 1io0 17c0 1o00 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 17c0 1io0 18N0 1bz0 19z0 1gp0 1610 1iL0 11z0 1o10 14o0 1lA1 SKX 1xd1 MKX 1AN0 1a00 1fA0 1cL0 1cN0 1nX0 1210 1nA0 1210 1qL0 WN0 1qL0 WN0 1qL0 11c0 1on0 11B0 1o00 11A0 1qo0 XA0 1qp0 1cN0 1cL0 17d0 1in0 14p0 1lb0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1lb0 14p0 1in0 17d0 1cL0 1cN0 19X0 e10 2L0 WN0 14n0 gN0 5z0 11B0 WL0 e10 bb0 11B0 TX0 e10 dX0 11B0 On0 gN0 gL0 11B0 Lz0 e10 pb0 WN0 IL0 e10 rX0 WN0 Db0 gN0 uL0 11B0 xz0 e10 An0 11B0 rX0 gN0 Db0 11B0 pb0 e10 Lz0 WN0 mn0 e10 On0 WN0 gL0 gN0 Rb0 11B0 bb0 e10 WL0 11B0 5z0 gN0 11z0 11B0 2L0 gN0 14n0 1fB0 1cL0 1a10 1fz0 14p0 1lb0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1nX0 14p0 1in0 17d0 1fz0 1a10 19X0 1fB0 17b0 e10 5z0 WN0 14n0 e10 8n0 WN0 WL0 gN0 bb0 11B0 Rb0 e10 gL0 11B0 Lz0 gN0 jz0 11B0 IL0 gN0 pb0 WN0 FX0 e10 uL0 WN0 An0 gN0 xz0 11B0 uL0 e10 Db0 11B0 rX0 e10 FX0 11B0 mn0 gN0 IL0 11B0 jz0 e10 Rb0 WN0 dX0 gN0 TX0 WN0 bb0 gN0 WL0 11B0 5z0 e10 14n0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0|18e5","Asia/Hebron|LMT EET EEST IST IDT|-2k.n -20 -30 -20 -30|01212121212121212121212121212121212343434343434343434343434343434312121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2MBCk.n 1Azek.n MM0 iM0 4JA0 10o0 1pA0 10M0 1pA0 16o0 1jA0 16o0 1jA0 pBa0 Vz0 1oN0 11b0 1oO0 10N0 1pz0 10N0 1pb0 10N0 1pb0 10N0 1pb0 10N0 1pz0 10N0 1pb0 10N0 1pb0 11d0 1oL0 dW0 hfB0 Db0 1fB0 Rb0 bXB0 gM0 8Q00 IM0 1wo0 TX0 1HB0 IL0 1s10 10n0 1o10 WL0 1zd0 On0 1ld0 11z0 1o10 14n0 1o10 14n0 1nd0 12n0 1nd0 Xz0 1q10 12n0 M10 C00 17c0 1io0 17c0 1io0 17c0 1o00 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 17c0 1io0 18N0 1bz0 19z0 1gp0 1610 1iL0 12L0 1mN0 14o0 1lc0 Tb0 1xd1 MKX bB0 cn0 1cN0 1a00 1fA0 1cL0 1cN0 1nX0 1210 1nA0 1210 1qL0 WN0 1qL0 WN0 1qL0 11c0 1on0 11B0 1o00 11A0 1qo0 XA0 1qp0 1cN0 1cL0 17d0 1in0 14p0 1lb0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1lb0 14p0 1in0 17d0 1cL0 1cN0 19X0 e10 2L0 WN0 14n0 gN0 5z0 11B0 WL0 e10 bb0 11B0 TX0 e10 dX0 11B0 On0 gN0 gL0 11B0 Lz0 e10 pb0 WN0 IL0 e10 rX0 WN0 Db0 gN0 uL0 11B0 xz0 e10 An0 11B0 rX0 gN0 Db0 11B0 pb0 e10 Lz0 WN0 mn0 e10 On0 WN0 gL0 gN0 Rb0 11B0 bb0 e10 WL0 11B0 5z0 gN0 11z0 11B0 2L0 gN0 14n0 1fB0 1cL0 1a10 1fz0 14p0 1lb0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1nX0 14p0 1in0 17d0 1fz0 1a10 19X0 1fB0 17b0 e10 5z0 WN0 14n0 e10 8n0 WN0 WL0 gN0 bb0 11B0 Rb0 e10 gL0 11B0 Lz0 gN0 jz0 11B0 IL0 gN0 pb0 WN0 FX0 e10 uL0 WN0 An0 gN0 xz0 11B0 uL0 e10 Db0 11B0 rX0 e10 FX0 11B0 mn0 gN0 IL0 11B0 jz0 e10 Rb0 WN0 dX0 gN0 TX0 WN0 bb0 gN0 WL0 11B0 5z0 e10 14n0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0|25e4","Asia/Ho_Chi_Minh|LMT PLMT +07 +08 +09|-76.u -76.u -70 -80 -90|0123423232|-2yC76.u bK00 1h7b6.u 5lz0 18o0 3Oq0 k5b0 aW00 BAM0|90e5","Asia/Hong_Kong|LMT HKT HKST HKWT JST|-7A.G -80 -90 -8u -90|0123412121212121212121212121212121212121212121212121212121212121212121|-2CFH0 1taO0 Hc0 xUu 9tBu 11z0 1tDu Rc0 1wo0 11A0 1cM0 11A0 1o00 11A0 1o00 11A0 1o00 14o0 1o00 11A0 1nX0 U10 1tz0 U10 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 U10 1tz0 U10 1wn0 Rd0 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 17d0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 s10 1Vz0 1cN0 1cL0 1cN0 1cL0 6fd0 14n0|73e5","Asia/Hovd|LMT +06 +07 +08|-66.A -60 -70 -80|012323232323232323232323232323232323232323232323232|-2APG6.A 2Uko6.A cKn0 1db0 1dd0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 6hD0 11z0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 kEp0 1cJ0 1cP0 1cJ0|81e3","Asia/Irkutsk|LMT IMT +07 +08 +09|-6V.5 -6V.5 -70 -80 -90|012343434343434343434343234343434343434343434343434343434343434343|-3D8SV.5 1Bxc0 pjXV.5 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|60e4","Europe/Istanbul|LMT IMT EET EEST +03 +04|-1T.Q -1U.U -20 -30 -30 -40|01232323232323232323232323232323232323232323232345423232323232323232323232323232323232323232323232323232323232323234|-3D8NT.Q 1ePXW.U dzzU.U 11b0 8tB0 1on0 1410 1db0 19B0 1in0 3Rd0 Un0 1oN0 11b0 zSN0 CL0 mp0 1Vz0 1gN0 8yn0 1yp0 ML0 1kp0 17b0 1ip0 17b0 1fB0 19X0 1ip0 19X0 1ip0 17b0 qdB0 38L0 1jd0 Tz0 l6O0 11A0 WN0 1qL0 TB0 1tX0 U10 1tz0 11B0 1in0 17d0 z90 cne0 pb0 2Cp0 1800 14o0 1dc0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1a00 1fA0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WO0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 Xc0 1qo0 WM0 1qM0 11A0 1o00 1200 1nA0 11A0 1tA0 U00 15w0|13e6","Asia/Jakarta|LMT BMT +0720 +0730 +09 +08 WIB|-77.c -77.c -7k -7u -90 -80 -70|012343536|-49jH7.c 2hiLL.c luM0 mPzO 8vWu 6kpu 4PXu xhcu|31e6","Asia/Jayapura|LMT +09 +0930 WIT|-9m.M -90 -9u -90|0123|-1uu9m.M sMMm.M L4nu|26e4","Asia/Jerusalem|LMT JMT IST IDT IDDT|-2k.S -2k.E -20 -30 -40|012323232323232432323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3D8Ok.S 1wvA0.e SyOk.E MM0 iM0 4JA0 10o0 1pA0 10M0 1pA0 16o0 1jA0 16o0 1jA0 3LA0 Eo0 oo0 1co0 1dA0 16o0 10M0 1jc0 1tA0 14o0 1cM0 1a00 11A0 1Nc0 Ao0 1Nc0 Ao0 1Ko0 LA0 1o00 WM0 EQK0 Db0 1fB0 Rb0 bXB0 gM0 8Q00 IM0 1wo0 TX0 1HB0 IL0 1s10 10n0 1o10 WL0 1zd0 On0 1ld0 11z0 1o10 14n0 1o10 14n0 1nd0 12n0 1nd0 Xz0 1q10 12n0 1hB0 1dX0 1ep0 1aL0 1eN0 17X0 1nf0 11z0 1tB0 19W0 1e10 17b0 1ep0 1gL0 18N0 1fz0 1eN0 17b0 1gq0 1gn0 19d0 1dz0 1c10 17X0 1hB0 1gn0 19d0 1dz0 1c10 17X0 1kp0 1dz0 1c10 1aL0 1eN0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1rz0 W10 1rz0 W10 1rz0 10N0 1oL0 10N0 1oL0 10N0 1oL0|81e4","Asia/Kabul|LMT +04 +0430|-4A.M -40 -4u|012|-3eLEA.M 2dTcA.M|46e5","Asia/Kamchatka|LMT +11 +12 +13|-ay.A -b0 -c0 -d0|012323232323232323232321232323232323232323232323232323232323212|-1SLKy.A ivXy.A 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 2sp0 WM0|18e4","Asia/Karachi|LMT +0530 +0630 +05 PKT PKST|-4s.c -5u -6u -50 -50 -60|012134545454|-2xoss.c 1qOKW.c 7zX0 eup0 LqMu 1fy00 1cL0 dK10 11b0 1610 1jX0|24e6","Asia/Urumqi|LMT +06|-5O.k -60|01|-1GgtO.k|32e5","Asia/Kathmandu|LMT +0530 +0545|-5F.g -5u -5J|012|-21JhF.g 2EGMb.g|12e5","Asia/Khandyga|LMT +08 +09 +10 +11|-92.d -80 -90 -a0 -b0|0123232323232323232323212323232323232323232323232343434343434343432|-21Q92.d pAp2.d 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 qK0 yN0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 17V0 7zD0|66e2","Asia/Krasnoyarsk|LMT +06 +07 +08|-6b.q -60 -70 -80|01232323232323232323232123232323232323232323232323232323232323232|-21Hib.q prAb.q 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|10e5","Asia/Kuala_Lumpur|LMT SMT +07 +0720 +0730 +09 +08|-6T.p -6T.p -70 -7k -7u -90 -80|01234546|-2M0ST.p aIM0 17anT.p l5XE 17bO 8Fyu 1so10|71e5","Asia/Macau|LMT CST +09 +10 CDT|-7y.a -80 -90 -a0 -90|012323214141414141414141414141414141414141414141414141414141414141414141|-2CFHy.a 1uqKy.a PX0 1kn0 15B0 11b0 4Qq0 1oM0 11c0 1ko0 1u00 11A0 1cM0 11c0 1o00 11A0 1o00 11A0 1oo0 1400 1o00 11A0 1o00 U00 1tA0 U00 1wo0 Rc0 1wru U10 1tz0 U10 1tz0 U10 1tz0 U10 1wn0 Rd0 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 17d0 1cK0 1cO0 1cK0 1cO0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 s10 1Vz0 1cN0 1cL0 1cN0 1cL0 6fd0 14n0|57e4","Asia/Magadan|LMT +10 +11 +12|-a3.c -a0 -b0 -c0|012323232323232323232321232323232323232323232323232323232323232312|-1Pca3.c eUo3.c 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3Cq0|95e3","Asia/Makassar|LMT MMT +08 +09 WITA|-7V.A -7V.A -80 -90 -80|01234|-21JjV.A vfc0 myLV.A 8ML0|15e5","Asia/Manila|LMT LMT PST PDT JST|fU -84 -80 -90 -90|01232423232|-54m84 2clc0 1vfc4 AL0 cK10 65X0 mXB0 vX0 VK10 1db0|24e6","Asia/Nicosia|LMT EET EEST|-2d.s -20 -30|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-1Vc2d.s 2a3cd.s 1cL0 1qp0 Xz0 19B0 19X0 1fB0 1db0 1cp0 1cL0 1fB0 19X0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1o30 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|32e4","Asia/Novokuznetsk|LMT +06 +07 +08|-5M.M -60 -70 -80|012323232323232323232321232323232323232323232323232323232323212|-1PctM.M eULM.M 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 2sp0 WM0|55e4","Asia/Novosibirsk|LMT +06 +07 +08|-5v.E -60 -70 -80|0123232323232323232323212323212121212121212121212121212121212121212|-21Qnv.E pAFv.E 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 ml0 Os0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 4eN0|15e5","Asia/Omsk|LMT +05 +06 +07|-4R.u -50 -60 -70|01232323232323232323232123232323232323232323232323232323232323232|-224sR.u pMLR.u 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|12e5","Asia/Oral|LMT +03 +05 +06 +04|-3p.o -30 -50 -60 -40|01232323232323232424242424242424242424242424242|-1Pc3p.o eUop.o 23CK0 3Db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 2pB0 1cM0 1fA0 1cM0 1cM0 IM0 1EM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0|27e4","Asia/Pontianak|LMT PMT +0730 +09 +08 WITA WIB|-7h.k -7h.k -7u -90 -80 -80 -70|012324256|-2ua7h.k XE00 munL.k 8Rau 6kpu 4PXu xhcu Wqnu|23e4","Asia/Pyongyang|LMT KST JST KST|-8n -8u -90 -90|012313|-2um8n 97XR 1lTzu 2Onc0 6BA0|29e5","Asia/Qostanay|LMT +04 +05 +06|-4e.s -40 -50 -60|012323232323232323232123232323232323232323232323|-1Pc4e.s eUoe.s 23CL0 3Db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0|","Asia/Qyzylorda|LMT +04 +05 +06|-4l.Q -40 -50 -60|01232323232323232323232323232323232323232323232|-1Pc4l.Q eUol.Q 23CL0 3Db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 3ao0 1EM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 zQl0|73e4","Asia/Rangoon|LMT RMT +0630 +09|-6o.L -6o.L -6u -90|01232|-3D8So.L 1BnA0 SmnS.L 7j9u|48e5","Asia/Sakhalin|LMT +09 +11 +12 +10|-9u.M -90 -b0 -c0 -a0|01232323232323232323232423232323232424242424242424242424242424242|-2AGVu.M 1BoMu.M 1qFa0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 2pB0 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3rd0|58e4","Asia/Samarkand|LMT +04 +05 +06|-4r.R -40 -50 -60|01232323232323232323232|-1Pc4r.R eUor.R 23CL0 3Db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0|36e4","Asia/Seoul|LMT KST JST KST KDT KDT|-8r.Q -8u -90 -90 -a0 -9u|012343434343151515151515134343|-2um8r.Q 97XV.Q 1m1zu 6CM0 Fz0 1kN0 14n0 1kN0 14L0 1zd0 On0 69B0 2I0u OL0 1FB0 Rb0 1qN0 TX0 1tB0 TX0 1tB0 TX0 1tB0 TX0 2ap0 12FBu 11A0 1o00 11A0|23e6","Asia/Srednekolymsk|LMT +10 +11 +12|-ae.Q -a0 -b0 -c0|01232323232323232323232123232323232323232323232323232323232323232|-1Pcae.Q eUoe.Q 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|35e2","Asia/Taipei|LMT CST JST CDT|-86 -80 -90 -90|012131313131313131313131313131313131313131|-30bk6 1FDc6 joM0 1yo0 Tz0 1ip0 1jX0 1cN0 11b0 1oN0 11b0 1oN0 11b0 1oN0 11b0 10N0 1BX0 10p0 1pz0 10p0 1pz0 10p0 1db0 1dd0 1db0 1cN0 1db0 1cN0 1db0 1cN0 1db0 1BB0 ML0 1Bd0 ML0 uq10 1db0 1cN0 1db0 97B0 AL0|74e5","Asia/Tashkent|LMT +05 +06 +07|-4B.b -50 -60 -70|012323232323232323232321|-1Pc4B.b eUnB.b 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0|23e5","Asia/Tbilisi|LMT TBMT +03 +04 +05|-2X.b -2X.b -30 -40 -50|01234343434343434343434323232343434343434343434323|-3D8OX.b 1LUM0 1jUnX.b WCL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 1cK0 1cL0 1cN0 1cL0 1cN0 2pz0 1cL0 1fB0 3Nz0 11B0 1nX0 11B0 1qL0 WN0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 An0 Os0 WM0|11e5","Asia/Tehran|LMT TMT +0330 +0430 +04 +05|-3p.I -3p.I -3u -4u -40 -50|012345423232323232323232323232323232323232323232323232323232323232323232|-2btDp.I Llc0 1FHaT.I 1pc0 120u Rc0 XA0 Wou JX0 1dB0 1en0 pNB0 UL0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 64p0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0 1cp0 1dz0 1cp0 1dz0 1cN0 1dz0 1cp0 1dz0|14e6","Asia/Thimphu|LMT +0530 +06|-5W.A -5u -60|012|-Su5W.A 1BGMs.A|79e3","Asia/Tokyo|LMT JST JDT|-9i.X -90 -a0|0121212121|-3jE90 2qSo0 Rc0 1lc0 14o0 1zc0 Oo0 1zc0 Oo0|38e6","Asia/Tomsk|LMT +06 +07 +08|-5D.P -60 -70 -80|0123232323232323232323212323232323232323232323212121212121212121212|-21NhD.P pxzD.P 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 co0 1bB0 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3Qp0|10e5","Asia/Ulaanbaatar|LMT +07 +08 +09|-77.w -70 -80 -90|012323232323232323232323232323232323232323232323232|-2APH7.w 2Uko7.w cKn0 1db0 1dd0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 6hD0 11z0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 kEp0 1cJ0 1cP0 1cJ0|12e5","Asia/Ust-Nera|LMT +08 +09 +12 +11 +10|-9w.S -80 -90 -c0 -b0 -a0|012343434343434343434345434343434343434343434343434343434343434345|-21Q9w.S pApw.S 23CL0 1d90 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 17V0 7zD0|65e2","Asia/Vladivostok|LMT +09 +10 +11|-8L.v -90 -a0 -b0|01232323232323232323232123232323232323232323232323232323232323232|-1SJIL.v itXL.v 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|60e4","Asia/Yakutsk|LMT +08 +09 +10|-8C.W -80 -90 -a0|01232323232323232323232123232323232323232323232323232323232323232|-21Q8C.W pAoC.W 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|28e4","Asia/Yekaterinburg|LMT PMT +04 +05 +06|-42.x -3J.5 -40 -50 -60|012343434343434343434343234343434343434343434343434343434343434343|-2ag42.x 7mQh.s qBvJ.5 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|14e5","Asia/Yerevan|LMT +03 +04 +05|-2W -30 -40 -50|0123232323232323232323212121212323232323232323232323232323232|-1Pc2W 1jUnW WCL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 2pB0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 4RX0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0|13e5","Atlantic/Azores|LMT HMT -02 -01 +00 WET|1G.E 1S.w 20 10 0 0|01232323232323232323232323232323232323232323234323432343234323232323232323232323232323232323232323232343434343434343434343434343434345434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-3tomh.k 18aoh.k aPX0 Sp0 LX0 1vc0 Tc0 1uM0 SM0 1vc0 Tc0 1vc0 SM0 1vc0 6600 1co0 3E00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 3I00 17c0 1cM0 1cM0 3Fc0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 1tA0 1cM0 1dc0 1400 gL0 IM0 s10 U00 dX0 Rc0 pd0 Rc0 gL0 Oo0 pd0 Rc0 gL0 Oo0 pd0 14o0 1cM0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 qIl0 1cM0 1fA0 1cM0 1cM0 1cN0 1cL0 1cN0 1cM0 1cM0 1cM0 1cM0 1cN0 1cL0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cL0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|25e4","Atlantic/Bermuda|LMT BMT BST AST ADT|4j.i 4j.i 3j.i 40 30|0121213434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-3eLvE.G 16mo0 1bb0 1i10 11X0 ru30 thbE.G 1PX0 11B0 1tz0 Rd0 1zb0 Op0 1zb0 3I10 Lz0 1EN0 FX0 1HB0 FX0 1Kp0 Db0 1Kp0 Db0 1Kp0 FX0 93d0 11z0 GAp0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|65e3","Atlantic/Canary|LMT -01 WET WEST|11.A 10 0 -10|01232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-1UtaW.o XPAW.o 1lAK0 1a10 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|54e4","Atlantic/Cape_Verde|LMT -02 -01|1y.4 20 10|01212|-2ldW0 1eEo0 7zX0 1djf0|50e4","Atlantic/Faroe|LMT WET WEST|r.4 0 -10|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2uSnw.U 2Wgow.U 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|49e3","Atlantic/Madeira|LMT FMT -01 +00 +01 WET WEST|17.A 17.A 10 0 -10 0 -10|01232323232323232323232323232323232323232323234323432343234323232323232323232323232323232323232323232565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565656565|-3tomQ.o 18anQ.o aPX0 Sp0 LX0 1vc0 Tc0 1uM0 SM0 1vc0 Tc0 1vc0 SM0 1vc0 6600 1co0 3E00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 3I00 17c0 1cM0 1cM0 3Fc0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 1tA0 1cM0 1dc0 1400 gL0 IM0 s10 U00 dX0 Rc0 pd0 Rc0 gL0 Oo0 pd0 Rc0 gL0 Oo0 pd0 14o0 1cM0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 qIl0 1cM0 1fA0 1cM0 1cM0 1cN0 1cL0 1cN0 1cM0 1cM0 1cM0 1cM0 1cN0 1cL0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|27e4","Atlantic/South_Georgia|LMT -02|2q.8 20|01|-3eLxx.Q|30","Atlantic/Stanley|LMT SMT -04 -03 -02|3P.o 3P.o 40 30 20|0123232323232323434323232323232323232323232323232323232323232323232323|-3eLw8.A S200 12bA8.A 19X0 1fB0 19X0 1ip0 19X0 1fB0 19X0 1fB0 19X0 1fB0 Cn0 1Cc10 WL0 1qL0 U10 1tz0 2mN0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1tz0 U10 1tz0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1tz0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qL0 WN0 1qN0 U10 1wn0 Rd0 1wn0 U10 1tz0 U10 1tz0 U10 1tz0 U10 1tz0 U10 1wn0 U10 1tz0 U10 1tz0 U10|21e2","Australia/Sydney|LMT AEST AEDT|-a4.Q -a0 -b0|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-32oW4.Q RlC4.Q xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00 17c00 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 14o0 1o00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1tA0 WM0 1tA0 U00 1tA0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 11A0 1o00 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|40e5","Australia/Adelaide|LMT ACST ACST ACDT|-9e.k -90 -9u -au|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323|-32oVe.k ak0e.k H1Bu xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00 17c00 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 Oo0 1zc0 WM0 1qM0 Rc0 1zc0 U00 1tA0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|11e5","Australia/Brisbane|LMT AEST AEDT|-ac.8 -a0 -b0|012121212121212121|-32Bmc.8 Ry2c.8 xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00 17c00 LA0 H1A0 Oo0 1zc0 Oo0 1zc0 Oo0|20e5","Australia/Broken_Hill|LMT AEST ACST ACST ACDT|-9p.M -a0 -90 -9u -au|0123434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-32oVp.M 3Lzp.M 6wp0 H1Bu xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00 17c00 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 14o0 1o00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1tA0 WM0 1tA0 U00 1tA0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|18e3","Australia/Hobart|LMT AEST AEDT|-9N.g -a0 -b0|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-3109N.g Pk1N.g 1a00 1qM0 Oo0 1zc0 Oo0 TAo0 yM0 1cM0 1cM0 1fA0 1a00 VfA0 1cM0 1o00 Rc0 1wo0 Rc0 1wo0 U00 1wo0 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 11A0 1qM0 WM0 1qM0 Oo0 1zc0 Oo0 1zc0 Oo0 1wo0 WM0 1tA0 WM0 1tA0 U00 1tA0 U00 1tA0 11A0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 11A0 1o00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|21e4","Australia/Darwin|LMT ACST ACST ACDT|-8H.k -90 -9u -au|01232323232|-32oUH.k ajXH.k H1Bu xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00|12e4","Australia/Eucla|LMT +0845 +0945|-8z.s -8J -9J|01212121212121212121|-30nIz.s PkpO.s xc0 10jc0 yM0 1cM0 1cM0 1gSo0 Oo0 l5A0 Oo0 iJA0 G00 zU00 IM0 1qM0 11A0 1o00 11A0|368","Australia/Lord_Howe|LMT AEST +1030 +1130 +11|-aA.k -a0 -au -bu -b0|01232323232424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424|-32oWA.k 3tzAA.k 1zdu Rb0 1zd0 On0 1zd0 On0 1zd0 On0 1zd0 TXu 1qMu WLu 1tAu WLu 1tAu TXu 1tAu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1zcu Onu 1zcu Onu 1zcu 11zu 1o0u 11zu 1o0u 11zu 1o0u 11zu 1qMu WLu 11Au 1nXu 1qMu 11zu 1o0u 11zu 1o0u 11zu 1qMu WLu 1qMu 11zu 1o0u WLu 1qMu 14nu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1fzu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu 1cLu 1fAu 1cLu 1cMu 1cLu 1cMu 1cLu 1cMu|347","Australia/Lindeman|LMT AEST AEDT|-9T.U -a0 -b0|0121212121212121212121|-32BlT.U Ry1T.U xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00 17c00 LA0 H1A0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0|10","Australia/Melbourne|LMT AEST AEDT|-9D.Q -a0 -b0|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212|-32oVD.Q RlBD.Q xc0 10jc0 yM0 1cM0 1cM0 1fA0 1a00 17c00 LA0 1C00 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 U00 1qM0 WM0 1qM0 11A0 1tA0 U00 1tA0 U00 1tA0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 11A0 1o00 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 14o0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|39e5","Australia/Perth|LMT AWST AWDT|-7H.o -80 -90|01212121212121212121|-30nHH.o PkpH.o xc0 10jc0 yM0 1cM0 1cM0 1gSo0 Oo0 l5A0 Oo0 iJA0 G00 zU00 IM0 1qM0 11A0 1o00 11A0|18e5","CET|CET CEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 16M0 1gMM0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|","Pacific/Easter|LMT EMT -07 -06 -05|7h.s 7h.s 70 60 50|0123232323232323232323232323234343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434|-3eLsG.w 1HRc0 1s4IG.w WL0 1zd0 On0 1ip0 11z0 1o10 11z0 1qN0 WL0 1ld0 14n0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 2pA0 11z0 1o10 11z0 1qN0 WL0 1qN0 WL0 1qN0 1cL0 1cN0 11z0 1o10 11z0 1qN0 WL0 1fB0 19X0 1qN0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1ip0 1fz0 1fB0 11z0 1qN0 WL0 1qN0 WL0 1qN0 WL0 1qN0 11z0 1o10 11z0 1o10 11z0 1qN0 WL0 1qN0 17b0 1ip0 11z0 1o10 19X0 1fB0 1nX0 G10 1EL0 Op0 1zb0 Rd0 1wn0 Rd0 46n0 Ap0 1Nb0 Ap0 1Nb0 Ap0 1zb0 11B0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0 1nX0 11B0 1qL0 WN0 1qL0 11B0 1nX0 11B0 1nX0 11B0|30e2","CST6CDT|CST CDT CWT CPT|60 50 50 50|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261s0 1nX0 11B0 1nX0 SgN0 8x30 iw0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","EET|EET EEST|-20 -30|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|hDB0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|","Europe/Dublin|LMT DMT IST GMT BST IST|p.l p.l -y.D 0 -10 -10|012343434343435353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353535353|-3BHby.D 1ra20 Rc0 1fzy.D 14M0 1fc0 1g00 1co0 1dc0 1co0 1oo0 1400 1dc0 19A0 1io0 1io0 WM0 1o00 14o0 1o00 17c0 1io0 17c0 1fA0 1a00 1lc0 17c0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1cM0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1io0 1qM0 Dc0 g600 14o0 1wo0 17c0 1io0 11A0 1o00 17c0 1fA0 1a00 1fA0 1cM0 1fA0 1a00 17c0 1fA0 1a00 1io0 17c0 1lc0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1a00 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1tA0 IM0 90o0 U00 1tA0 U00 1tA0 U00 1tA0 U00 1tA0 WM0 1qM0 WM0 1qM0 WM0 1tA0 U00 1tA0 U00 1tA0 11z0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 14o0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|12e5","EST|EST|50|0||","EST5EDT|EST EDT EWT EPT|50 40 40 40|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261t0 1nX0 11B0 1nX0 SgN0 8x40 iv0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","Etc/GMT-0|GMT|0|0||","Etc/GMT-1|+01|-10|0||","Etc/GMT-10|+10|-a0|0||","Etc/GMT-11|+11|-b0|0||","Etc/GMT-12|+12|-c0|0||","Etc/GMT-13|+13|-d0|0||","Etc/GMT-14|+14|-e0|0||","Etc/GMT-2|+02|-20|0||","Etc/GMT-3|+03|-30|0||","Etc/GMT-4|+04|-40|0||","Etc/GMT-5|+05|-50|0||","Etc/GMT-6|+06|-60|0||","Etc/GMT-7|+07|-70|0||","Etc/GMT-8|+08|-80|0||","Etc/GMT-9|+09|-90|0||","Etc/GMT+1|-01|10|0||","Etc/GMT+10|-10|a0|0||","Etc/GMT+11|-11|b0|0||","Etc/GMT+12|-12|c0|0||","Etc/GMT+2|-02|20|0||","Etc/GMT+3|-03|30|0||","Etc/GMT+4|-04|40|0||","Etc/GMT+5|-05|50|0||","Etc/GMT+6|-06|60|0||","Etc/GMT+7|-07|70|0||","Etc/GMT+8|-08|80|0||","Etc/GMT+9|-09|90|0||","Etc/UTC|UTC|0|0||","Europe/Brussels|LMT BMT WET CET CEST WEST|-h.u -h.u 0 -10 -20 -10|012343434325252525252525252525252525252525252525252525434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-3D8Mh.u u1Ah.u SO00 3zX0 11c0 1iO0 11A0 1o00 11A0 my0 Ic0 1qM0 Rc0 1EM0 UM0 1u00 10o0 1io0 1io0 17c0 1a00 1fA0 1cM0 1cM0 1io0 17c0 1fA0 1a00 1io0 1a30 1io0 17c0 1fA0 1a00 1io0 17c0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 y00 5Wn0 WM0 1fA0 1cM0 16M0 1iM0 16M0 1C00 Uo0 1eeo0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|21e5","Europe/Andorra|LMT WET CET CEST|-6.4 0 -10 -20|0123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-2M0M6.4 1Pnc6.4 1xIN0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|79e3","Europe/Astrakhan|LMT +03 +04 +05|-3c.c -30 -40 -50|012323232323232323212121212121212121212121212121212121212121212|-1Pcrc.c eUMc.c 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 2pB0 1cM0 1fA0 1cM0 3Co0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3rd0|10e5","Europe/Athens|LMT AMT EET EEST CEST CET|-1y.Q -1y.Q -20 -30 -20 -10|0123234545232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-30SNy.Q OMM1 CNbx.Q mn0 kU10 9b0 3Es0 Xa0 1fb0 1dd0 k3X0 Nz0 SCp0 1vc0 SO0 1cM0 1a00 1ao0 1fc0 1a10 1fG0 1cg0 1dX0 1bX0 1cQ0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|35e5","Europe/London|LMT GMT BST BDST|1.f 0 -10 -20|01212121212121212121212121212121212121212121212121232323232321212321212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-4VgnW.J 2KHdW.J Rc0 1fA0 14M0 1fc0 1g00 1co0 1dc0 1co0 1oo0 1400 1dc0 19A0 1io0 1io0 WM0 1o00 14o0 1o00 17c0 1io0 17c0 1fA0 1a00 1lc0 17c0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1cM0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1io0 1qM0 Dc0 2Rz0 Dc0 1zc0 Oo0 1zc0 Rc0 1wo0 17c0 1iM0 FA0 xB0 1fA0 1a00 14o0 bb0 LA0 xB0 Rc0 1wo0 11A0 1o00 17c0 1fA0 1a00 1fA0 1cM0 1fA0 1a00 17c0 1fA0 1a00 1io0 17c0 1lc0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1a00 1qM0 WM0 1qM0 11A0 1o00 WM0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1tA0 IM0 90o0 U00 1tA0 U00 1tA0 U00 1tA0 U00 1tA0 WM0 1qM0 WM0 1qM0 WM0 1tA0 U00 1tA0 U00 1tA0 11z0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1o00 14o0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|10e6","Europe/Belgrade|LMT CET CEST|-1m -10 -20|012121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3topm 2juLm 3IP0 WM0 1fA0 1cM0 1cM0 1rc0 Qo0 1vmo0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|12e5","Europe/Prague|LMT PMT CET CEST GMT|-V.I -V.I -10 -20 0|0123232323232323232423232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-4QbAV.I 1FDc0 XPaV.I 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 1cM0 1qM0 11c0 mp0 xA0 mn0 17c0 1io0 17c0 1fc0 1ao0 1bNc0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|13e5","Europe/Bucharest|LMT BMT EET EEST|-1I.o -1I.o -20 -30|01232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3awpI.o 1AU00 20LI.o RA0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1Axc0 On0 1fA0 1a10 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cK0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cL0 1cN0 1cL0 1fB0 1nX0 11E0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|19e5","Europe/Budapest|LMT CET CEST|-1g.k -10 -20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-3cK1g.k 124Lg.k 11d0 1iO0 11A0 1o00 11A0 1oo0 11c0 1lc0 17c0 O1V0 3Nf0 WM0 1fA0 1cM0 1cM0 1oJ0 1dd0 1020 1fX0 1cp0 1cM0 1cM0 1cM0 1fA0 1a00 bhy0 Rb0 1wr0 Rc0 1C00 LA0 1C00 LA0 SNW0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cO0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|17e5","Europe/Zurich|LMT BMT CET CEST|-y.8 -t.K -10 -20|0123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-4HyMy.8 1Dw04.m 1SfAt.K 11A0 1o00 11A0 1xG10 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|38e4","Europe/Chisinau|LMT CMT BMT EET EEST CEST CET MSK MSD|-1T.k -1T -1I.o -20 -30 -20 -10 -30 -40|0123434343434343434345656578787878787878787878434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343434343|-3D8NT.k 1wNA0.k wGMa.A 20LI.o RA0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 27A0 2en0 39g0 WM0 1fA0 1cM0 V90 1t7z0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 gL0 WO0 1cM0 1cM0 1cK0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1nX0 11D0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|67e4","Europe/Gibraltar|LMT GMT BST BDST CET CEST|l.o 0 -10 -20 -10 -20|0121212121212121212121212121212121212121212121212123232323232121232121212121212121212145454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-3BHbC.A 1ra1C.A Rc0 1fA0 14M0 1fc0 1g00 1co0 1dc0 1co0 1oo0 1400 1dc0 19A0 1io0 1io0 WM0 1o00 14o0 1o00 17c0 1io0 17c0 1fA0 1a00 1lc0 17c0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1cM0 1io0 17c0 1fA0 1a00 1io0 17c0 1io0 17c0 1fA0 1a00 1io0 1qM0 Dc0 2Rz0 Dc0 1zc0 Oo0 1zc0 Rc0 1wo0 17c0 1iM0 FA0 xB0 1fA0 1a00 14o0 bb0 LA0 xB0 Rc0 1wo0 11A0 1o00 17c0 1fA0 1a00 1fA0 1cM0 1fA0 1a00 17c0 1fA0 1a00 1io0 17c0 1lc0 17c0 1fA0 10Jz0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|30e3","Europe/Helsinki|LMT HMT EET EEST|-1D.N -1D.N -20 -30|01232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3H0ND.N 1Iu00 OULD.N 1dA0 1xGq0 1cM0 1cM0 1cM0 1cN0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|12e5","Europe/Kaliningrad|LMT CET CEST EET EEST MSK MSD +03|-1m -10 -20 -20 -30 -30 -40 -30|012121212121212343565656565656565654343434343434343434343434343434343434343434373|-36Rdm UbXm 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 390 7A0 1en0 12N0 1pbb0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|44e4","Europe/Kiev|LMT KMT EET MSK CEST CET MSD EEST|-22.4 -22.4 -20 -30 -20 -10 -40 -30|01234545363636363636363636367272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272727272|-3D8O2.4 1LUM0 eUo2.4 rnz0 2Hg0 WM0 1fA0 da0 1v4m0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 Db0 3220 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o10 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|34e5","Europe/Kirov|LMT +03 +04 +05 MSD MSK MSK|-3i.M -30 -40 -50 -40 -30 -40|0123232323232323232454524545454545454545454545454545454545454565|-22WM0 qH90 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 2pz0 1cN0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|48e4","Europe/Lisbon|LMT WET WEST WEMT CET CEST|A.J 0 -10 -20 -10 -20|01212121212121212121212121212121212121212121232123212321232121212121212121212121212121212121212121214121212121212121212121212121212124545454212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2le00 aPX0 Sp0 LX0 1vc0 Tc0 1uM0 SM0 1vc0 Tc0 1vc0 SM0 1vc0 6600 1co0 3E00 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 3I00 17c0 1cM0 1cM0 3Fc0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Dc0 1tA0 1cM0 1dc0 1400 gL0 IM0 s10 U00 dX0 Rc0 pd0 Rc0 gL0 Oo0 pd0 Rc0 gL0 Oo0 pd0 14o0 1cM0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 pvy0 1cM0 1cM0 1fA0 1cM0 1cM0 1cN0 1cL0 1cN0 1cM0 1cM0 1cM0 1cM0 1cN0 1cL0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|27e5","Europe/Madrid|LMT WET WEST WEMT CET CEST|e.I 0 -10 -20 -10 -20|0121212121212121212321454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454|-2M0M0 G5z0 19B0 1cL0 1dd0 b1z0 18p0 3HX0 17d0 1fz0 1a10 1io0 1a00 1in0 17d0 iIn0 Hd0 1cL0 bb0 1200 2s20 14n0 5aL0 Mp0 1vz0 17d0 1in0 17d0 1in0 17d0 1in0 17d0 6hX0 11B0 XHX0 1a10 1fz0 1a10 19X0 1cN0 1fz0 1a10 1fC0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|62e5","Europe/Malta|LMT CET CEST|-W.4 -10 -20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-35rcW.4 SXzW.4 Lz0 1cN0 1db0 1410 1on0 Wp0 1qL0 17d0 1cL0 M3B0 5M20 WM0 1fA0 1co0 17c0 1iM0 16m0 1de0 1lc0 14m0 1lc0 WO0 1qM0 GTW0 On0 1C10 LA0 1C00 LA0 1EM0 LA0 1C00 LA0 1zc0 Oo0 1C00 Oo0 1co0 1cM0 1lA0 Xc0 1qq0 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1o10 11z0 1iN0 19z0 1fB0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|42e4","Europe/Minsk|LMT MMT EET MSK CEST CET MSD EEST +03|-1O.g -1O -20 -30 -20 -10 -40 -30 -30|012345454363636363636363636372727272727272727272727272727272727272728|-3D8NO.g 1LUM0.g eUnO qNX0 3gQ0 WM0 1fA0 1cM0 Al0 1tsn0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 3Fc0 1cN0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0|19e5","Europe/Paris|LMT PMT WET WEST CEST CET WEMT|-9.l -9.l 0 -10 -20 -10 -20|01232323232323232323232323232323232323232323232323234545463654545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545|-3bQ09.l MDA0 cNb9.l HA0 19A0 1iM0 11c0 1oo0 Wo0 1rc0 QM0 1EM0 UM0 1u00 10o0 1io0 1wo0 Rc0 1a00 1fA0 1cM0 1cM0 1io0 17c0 1fA0 1a00 1io0 1a00 1io0 17c0 1fA0 1a00 1io0 17c0 1cM0 1cM0 1a00 1io0 1cM0 1cM0 1a00 1fA0 1io0 17c0 1cM0 1cM0 1a00 1fA0 1io0 1qM0 Df0 Ik0 5M30 WM0 1fA0 1cM0 Vx0 hB0 1aq0 16M0 1ekn0 1cL0 1fC0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|11e6","Europe/Moscow|LMT MMT MMT MST MDST MSD MSK +05 EET EEST MSK|-2u.h -2u.h -2v.j -3v.j -4v.j -40 -30 -50 -20 -30 -40|01232434565756865656565656565656565698656565656565656565656565656565656565656a6|-3D8Ou.h 1sQM0 2pyW.W 1bA0 11X0 GN0 1Hb0 c4v.j ik0 3DA0 dz0 15A0 c10 2q10 iM10 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0|16e6","Europe/Riga|LMT RMT LST EET MSK CEST CET MSD EEST|-1A.y -1A.y -2A.y -20 -30 -20 -10 -40 -30|0121213456565647474747474747474838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383838383|-3D8NA.y 1xde0 11A0 1iM0 ko0 gWm0 yDXA.y 2bX0 3fE0 WM0 1fA0 1cM0 1cM0 4m0 1sLy0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cN0 1o00 11A0 1o00 11A0 1qM0 3oo0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|64e4","Europe/Rome|LMT RMT CET CEST|-N.U -N.U -10 -20|012323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-4aU0N.U 15snN.U T000 Lz0 1cN0 1db0 1410 1on0 Wp0 1qL0 17d0 1cL0 M3B0 5M20 WM0 1fA0 1cM0 16M0 1iM0 16m0 1de0 1lc0 14m0 1lc0 WO0 1qM0 GTW0 On0 1C10 LA0 1C00 LA0 1EM0 LA0 1C00 LA0 1zc0 Oo0 1C00 Oo0 1C00 LA0 1zc0 Oo0 1C00 LA0 1C00 LA0 1zc0 Oo0 1C00 Oo0 1zc0 Oo0 1fC0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|39e5","Europe/Samara|LMT +03 +04 +05|-3k.k -30 -40 -50|0123232323232323232121232323232323232323232323232323232323212|-22WM0 qH90 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 2pB0 1cM0 1fA0 2y10 14m0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 2sp0 WM0|12e5","Europe/Saratov|LMT +03 +04 +05|-34.i -30 -40 -50|012323232323232321212121212121212121212121212121212121212121212|-22WM0 qH90 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 2pB0 1cM0 1cM0 1cM0 1fA0 1cM0 3Co0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 5810|","Europe/Simferopol|LMT SMT EET MSK CEST CET MSD EEST MSK|-2g.o -2g -20 -30 -20 -10 -40 -30 -40|0123454543636363636363636363272727636363727272727272727272727272727272727283|-3D8Og.o 1LUM0.o eUog rEn0 2qs0 WM0 1fA0 1cM0 3V0 1u0L0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1Q00 4eN0 1cM0 1cM0 1cM0 1cM0 dV0 WO0 1cM0 1cM0 1fy0 1o30 11B0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11z0 1nW0|33e4","Europe/Sofia|LMT IMT EET CET CEST EEST|-1x.g -1U.U -20 -10 -20 -30|0123434325252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252525252|-3D8Nx.g AiLA.k 1UFeU.U WM0 1fA0 1cM0 1cM0 1cN0 1mKH0 1dd0 1fb0 1ap0 1fb0 1a20 1fy0 1a30 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cK0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 1nX0 11E0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|12e5","Europe/Tallinn|LMT TMT CET CEST EET MSK MSD EEST|-1D -1D -10 -20 -20 -30 -40 -30|0123214532323565656565656565657474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474|-3D8ND 1wI00 teD 11A0 1Ta0 4rXl KSLD 2FX0 2Jg0 WM0 1fA0 1cM0 18J0 1sTX0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o10 11A0 1qM0 5QM0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|41e4","Europe/Tirane|LMT CET CEST|-1j.k -10 -20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-2glBj.k 14pcj.k 5LC0 WM0 4M0 1fCK0 10n0 1op0 11z0 1pd0 11z0 1qN0 WL0 1qp0 Xb0 1qp0 Xb0 1qp0 11z0 1lB0 11z0 1qN0 11z0 1iN0 16n0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|42e4","Europe/Ulyanovsk|LMT +03 +04 +05 +02|-3d.A -30 -40 -50 -20|01232323232323232321214121212121212121212121212121212121212121212|-22WM0 qH90 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 2pB0 1cM0 1fA0 2pB0 IM0 rX0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 3rd0|13e5","Europe/Vienna|LMT CET CEST|-15.l -10 -20|01212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121212121|-36Rd5.l UbX5.l 11d0 1iO0 11A0 1o00 11A0 3KM0 14o0 LA00 6i00 WM0 1fA0 1cM0 1cM0 1cM0 400 2qM0 1ao0 1co0 1cM0 1io0 17c0 1gHa0 19X0 1cP0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|18e5","Europe/Vilnius|LMT WMT KMT CET EET MSK CEST MSD EEST|-1F.g -1o -1z.A -10 -20 -30 -20 -40 -30|0123435636365757575757575757584848484848484848463648484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484848484|-3D8NF.g 1u5Ah.g 6ILM.o 1Ooz.A zz0 Mfd0 29W0 3is0 WM0 1fA0 1cM0 LV0 1tgL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11B0 1o00 11A0 1qM0 8io0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|54e4","Europe/Volgograd|LMT +03 +04 +05 MSD MSK MSK|-2V.E -30 -40 -50 -40 -30 -40|012323232323232324545452454545454545454545454545454545454545456525|-21IqV.E psLV.E 23CL0 1db0 1cN0 1db0 1cN0 1db0 1dd0 1cO0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1cM0 1cM0 1fA0 1cM0 2pz0 1cN0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 8Hz0 9Jd0 5gn0|10e5","Europe/Warsaw|LMT WMT CET CEST EET EEST|-1o -1o -10 -20 -20 -30|0123232345423232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232|-3D8No 1qDA0 1LXo 11d0 1iO0 11A0 1o00 11A0 1on0 11A0 6zy0 HWP0 5IM0 WM0 1fA0 1cM0 1dz0 1mL0 1en0 15B0 1aq0 1nA0 11A0 1io0 17c0 1fA0 1a00 iDX0 LA0 1cM0 1cM0 1C00 Oo0 1cM0 1cM0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1C00 LA0 uso0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cN0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|17e5","HST|HST|a0|0||","Indian/Chagos|LMT +05 +06|-4N.E -50 -60|012|-2xosN.E 3AGLN.E|30e2","Indian/Maldives|LMT MMT +05|-4S -4S -50|012|-3D8QS 3eLA0|35e4","Indian/Mauritius|LMT +04 +05|-3O -40 -50|012121|-2xorO 34unO 14L0 12kr0 11z0|15e4","Pacific/Kwajalein|LMT +11 +10 +09 -12 +12|-b9.k -b0 -a0 -90 c0 -c0|0123145|-2M0X9.k 1rDA9.k akp0 6Up0 12ry0 Wan0|14e3","MET|MET MEST|-10 -20|01010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-2aFe0 11d0 1iO0 11A0 1o00 11A0 Qrc0 6i00 WM0 1fA0 1cM0 1cM0 1cM0 16M0 1gMM0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|","MST|MST|70|0||","MST7MDT|MST MDT MWT MPT|70 60 60 60|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261r0 1nX0 11B0 1nX0 SgN0 8x20 ix0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","Pacific/Chatham|LMT +1215 +1245 +1345|-cd.M -cf -cJ -dJ|0123232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323|-46jMd.M 37RbW.M 1adef IM0 1C00 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Oo0 1zc0 Rc0 1zc0 Oo0 1qM0 14o0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1lc0 14o0 1lc0 14o0 1lc0 17c0 1io0 17c0 1io0 17c0 1io0 17c0 1io0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1io0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00|600","Pacific/Apia|LMT LMT -1130 -11 -10 +14 +13|-cx.4 bq.U bu b0 a0 -e0 -d0|012343456565656565656565656|-38Fox.4 J1A0 1yW03.4 2rRbu 1ff0 1a00 CI0 AQ0 1cM0 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1a00 1fA0 1cM0 1fA0 1a00 1fA0 1a00 1fA0|37e3","Pacific/Bougainville|LMT PMMT +10 +09 +11|-am.g -9M.w -a0 -90 -b0|012324|-3D8Wm.g AvAx.I 1TCLM.w 7CN0 2MQp0|18e4","Pacific/Efate|LMT +11 +12|-bd.g -b0 -c0|012121212121212121212121|-2l9nd.g 2uNXd.g Dc0 n610 1cL0 1cN0 1cL0 1fB0 19X0 1fB0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1fB0 Lz0 1Nd0 An0|66e3","Pacific/Enderbury|-00 -12 -11 +13|0 c0 b0 -d0|0123|-1iIo0 1GsA0 B7X0|1","Pacific/Fakaofo|LMT -11 +13|bo.U b0 -d0|012|-2M0Az.4 4ufXz.4|483","Pacific/Fiji|LMT +12 +13|-bT.I -c0 -d0|012121212121212121212121212121|-2bUzT.I 3m8NT.I LA0 1EM0 IM0 nJc0 LA0 1o00 Rc0 1wo0 Ao0 1Nc0 Ao0 1Q00 xz0 1SN0 uM0 1SM0 uM0 1VA0 s00 1VA0 s00 1VA0 s00 20o0 pc0 2hc0 bc0|88e4","Pacific/Tarawa|LMT +12|-bw.4 -c0|01|-2M0Xw.4|29e3","Pacific/Galapagos|LMT -05 -06|5W.o 50 60|01212|-1yVS1.A 2dTz1.A gNd0 rz0|25e3","Pacific/Gambier|LMT -09|8X.M 90|01|-2jof0.c|125","Pacific/Guadalcanal|LMT +11|-aD.M -b0|01|-2joyD.M|11e4","Pacific/Guam|LMT LMT GST +09 GDT ChST|el -9D -a0 -90 -b0 -a0|0123242424242424242425|-54m9D 2glc0 1DFbD 6pB0 AhB0 3QL0 g2p0 3p91 WOX rX0 1zd0 Rb0 1wp0 Rb0 5xd0 rX0 5sN0 zb1 1C0X On0 ULb0|17e4","Pacific/Honolulu|LMT HST HDT HWT HPT HST|av.q au 9u 9u 9u a0|01213415|-3061s.y 1uMdW.y 8x0 lef0 8wWu iAu 46p0|37e4","Pacific/Kiritimati|LMT -1040 -10 +14|at.k aE a0 -e0|0123|-2M0Bu.E 3bIMa.E B7Xk|51e2","Pacific/Kosrae|LMT LMT +11 +09 +10 +12|d8.4 -aP.U -b0 -90 -a0 -c0|0123243252|-54maP.U 2glc0 xsnP.U axC0 HBy0 akp0 axd0 WOK0 1bdz0|66e2","Pacific/Marquesas|LMT -0930|9i 9u|01|-2joeG|86e2","Pacific/Pago_Pago|LMT LMT SST|-cB.c bm.M b0|012|-38FoB.c J1A0|37e2","Pacific/Nauru|LMT +1130 +09 +12|-b7.E -bu -90 -c0|01213|-1Xdn7.E QCnB.E 7mqu 1lnbu|10e3","Pacific/Niue|LMT -1120 -11|bj.E bk b0|012|-FScE.k suo0.k|12e2","Pacific/Norfolk|LMT +1112 +1130 +1230 +11 +12|-bb.Q -bc -bu -cu -b0 -c0|0123245454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545454545|-2M0Xb.Q 21ILX.Q W01G Oo0 1COo0 9Jcu 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0|25e4","Pacific/Noumea|LMT +11 +12|-b5.M -b0 -c0|01212121|-2l9n5.M 2EqM5.M xX0 1PB0 yn0 HeP0 Ao0|98e3","Pacific/Palau|LMT LMT +09|f2.4 -8V.U -90|012|-54m8V.U 2glc0|21e3","Pacific/Pitcairn|LMT -0830 -08|8E.k 8u 80|012|-2M0Dj.E 3UVXN.E|56","Pacific/Rarotonga|LMT LMT -1030 -0930 -10|-dk.U aD.4 au 9u a0|01234343434343434343434343434|-2Otpk.U 28zc0 13tbO.U IL0 1zcu Onu 1zcu Onu 1zcu Rbu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Onu 1zcu Rbu 1zcu Onu 1zcu Onu 1zcu Onu|13e3","Pacific/Tahiti|LMT -10|9W.g a0|01|-2joe1.I|18e4","Pacific/Tongatapu|LMT +1220 +13 +14|-cj.c -ck -d0 -e0|01232323232|-XbMj.c BgLX.c 1yndk 15A0 1wo0 xz0 1Q10 xz0 zWN0 s00|75e3","PST8PDT|PST PDT PWT PPT|80 70 70 70|010102301010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|-261q0 1nX0 11B0 1nX0 SgN0 8x10 iy0 QwN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1cN0 1cL0 1cN0 1cL0 s10 1Vz0 LB0 1BX0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 1cN0 1fz0 1a10 1fz0 1cN0 1cL0 1cN0 1cL0 1cN0 1cL0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 14p0 1lb0 14p0 1lb0 14p0 1nX0 11B0 1nX0 11B0 1nX0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Rd0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0 Op0 1zb0|","WET|WET WEST|0 -10|010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010|hDB0 1a00 1fA0 1cM0 1cM0 1cM0 1fA0 1a00 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1cM0 1fA0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1qM0 WM0 1qM0 WM0 1qM0 11A0 1o00 11A0 1o00 11A0 1o00|"],links:["Africa/Abidjan|Africa/Accra","Africa/Abidjan|Africa/Bamako","Africa/Abidjan|Africa/Banjul","Africa/Abidjan|Africa/Conakry","Africa/Abidjan|Africa/Dakar","Africa/Abidjan|Africa/Freetown","Africa/Abidjan|Africa/Lome","Africa/Abidjan|Africa/Nouakchott","Africa/Abidjan|Africa/Ouagadougou","Africa/Abidjan|Africa/Timbuktu","Africa/Abidjan|Atlantic/Reykjavik","Africa/Abidjan|Atlantic/St_Helena","Africa/Abidjan|Iceland","Africa/Cairo|Egypt","Africa/Johannesburg|Africa/Maseru","Africa/Johannesburg|Africa/Mbabane","Africa/Lagos|Africa/Bangui","Africa/Lagos|Africa/Brazzaville","Africa/Lagos|Africa/Douala","Africa/Lagos|Africa/Kinshasa","Africa/Lagos|Africa/Libreville","Africa/Lagos|Africa/Luanda","Africa/Lagos|Africa/Malabo","Africa/Lagos|Africa/Niamey","Africa/Lagos|Africa/Porto-Novo","Africa/Maputo|Africa/Blantyre","Africa/Maputo|Africa/Bujumbura","Africa/Maputo|Africa/Gaborone","Africa/Maputo|Africa/Harare","Africa/Maputo|Africa/Kigali","Africa/Maputo|Africa/Lubumbashi","Africa/Maputo|Africa/Lusaka","Africa/Nairobi|Africa/Addis_Ababa","Africa/Nairobi|Africa/Asmara","Africa/Nairobi|Africa/Asmera","Africa/Nairobi|Africa/Dar_es_Salaam","Africa/Nairobi|Africa/Djibouti","Africa/Nairobi|Africa/Kampala","Africa/Nairobi|Africa/Mogadishu","Africa/Nairobi|Indian/Antananarivo","Africa/Nairobi|Indian/Comoro","Africa/Nairobi|Indian/Mayotte","Africa/Tripoli|Libya","America/Adak|America/Atka","America/Adak|US/Aleutian","America/Anchorage|US/Alaska","America/Argentina/Buenos_Aires|America/Buenos_Aires","America/Argentina/Catamarca|America/Argentina/ComodRivadavia","America/Argentina/Catamarca|America/Catamarca","America/Argentina/Cordoba|America/Cordoba","America/Argentina/Cordoba|America/Rosario","America/Argentina/Jujuy|America/Jujuy","America/Argentina/Mendoza|America/Mendoza","America/Chicago|US/Central","America/Denver|America/Shiprock","America/Denver|Navajo","America/Denver|US/Mountain","America/Detroit|US/Michigan","America/Edmonton|America/Yellowknife","America/Edmonton|Canada/Mountain","America/Fort_Wayne|America/Indiana/Indianapolis","America/Fort_Wayne|America/Indianapolis","America/Fort_Wayne|US/East-Indiana","America/Godthab|America/Nuuk","America/Halifax|Canada/Atlantic","America/Havana|Cuba","America/Indiana/Knox|America/Knox_IN","America/Indiana/Knox|US/Indiana-Starke","America/Iqaluit|America/Pangnirtung","America/Jamaica|Jamaica","America/Kentucky/Louisville|America/Louisville","America/Los_Angeles|US/Pacific","America/Manaus|Brazil/West","America/Mazatlan|Mexico/BajaSur","America/Mexico_City|Mexico/General","America/New_York|US/Eastern","America/Noronha|Brazil/DeNoronha","America/Panama|America/Atikokan","America/Panama|America/Cayman","America/Panama|America/Coral_Harbour","America/Phoenix|America/Creston","America/Phoenix|US/Arizona","America/Puerto_Rico|America/Anguilla","America/Puerto_Rico|America/Antigua","America/Puerto_Rico|America/Aruba","America/Puerto_Rico|America/Blanc-Sablon","America/Puerto_Rico|America/Curacao","America/Puerto_Rico|America/Dominica","America/Puerto_Rico|America/Grenada","America/Puerto_Rico|America/Guadeloupe","America/Puerto_Rico|America/Kralendijk","America/Puerto_Rico|America/Lower_Princes","America/Puerto_Rico|America/Marigot","America/Puerto_Rico|America/Montserrat","America/Puerto_Rico|America/Port_of_Spain","America/Puerto_Rico|America/St_Barthelemy","America/Puerto_Rico|America/St_Kitts","America/Puerto_Rico|America/St_Lucia","America/Puerto_Rico|America/St_Thomas","America/Puerto_Rico|America/St_Vincent","America/Puerto_Rico|America/Tortola","America/Puerto_Rico|America/Virgin","America/Regina|Canada/Saskatchewan","America/Rio_Branco|America/Porto_Acre","America/Rio_Branco|Brazil/Acre","America/Santiago|Chile/Continental","America/Sao_Paulo|Brazil/East","America/St_Johns|Canada/Newfoundland","America/Tijuana|America/Ensenada","America/Tijuana|America/Santa_Isabel","America/Tijuana|Mexico/BajaNorte","America/Toronto|America/Montreal","America/Toronto|America/Nassau","America/Toronto|America/Nipigon","America/Toronto|America/Thunder_Bay","America/Toronto|Canada/Eastern","America/Vancouver|Canada/Pacific","America/Whitehorse|Canada/Yukon","America/Winnipeg|America/Rainy_River","America/Winnipeg|Canada/Central","Asia/Ashgabat|Asia/Ashkhabad","Asia/Bangkok|Asia/Phnom_Penh","Asia/Bangkok|Asia/Vientiane","Asia/Bangkok|Indian/Christmas","Asia/Brunei|Asia/Kuching","Asia/Dhaka|Asia/Dacca","Asia/Dubai|Asia/Muscat","Asia/Dubai|Indian/Mahe","Asia/Dubai|Indian/Reunion","Asia/Ho_Chi_Minh|Asia/Saigon","Asia/Hong_Kong|Hongkong","Asia/Jerusalem|Asia/Tel_Aviv","Asia/Jerusalem|Israel","Asia/Kathmandu|Asia/Katmandu","Asia/Kolkata|Asia/Calcutta","Asia/Kuala_Lumpur|Asia/Singapore","Asia/Kuala_Lumpur|Singapore","Asia/Macau|Asia/Macao","Asia/Makassar|Asia/Ujung_Pandang","Asia/Nicosia|Europe/Nicosia","Asia/Qatar|Asia/Bahrain","Asia/Rangoon|Asia/Yangon","Asia/Rangoon|Indian/Cocos","Asia/Riyadh|Antarctica/Syowa","Asia/Riyadh|Asia/Aden","Asia/Riyadh|Asia/Kuwait","Asia/Seoul|ROK","Asia/Shanghai|Asia/Chongqing","Asia/Shanghai|Asia/Chungking","Asia/Shanghai|Asia/Harbin","Asia/Shanghai|PRC","Asia/Taipei|ROC","Asia/Tehran|Iran","Asia/Thimphu|Asia/Thimbu","Asia/Tokyo|Japan","Asia/Ulaanbaatar|Asia/Ulan_Bator","Asia/Urumqi|Asia/Kashgar","Atlantic/Faroe|Atlantic/Faeroe","Australia/Adelaide|Australia/South","Australia/Brisbane|Australia/Queensland","Australia/Broken_Hill|Australia/Yancowinna","Australia/Darwin|Australia/North","Australia/Hobart|Australia/Currie","Australia/Hobart|Australia/Tasmania","Australia/Lord_Howe|Australia/LHI","Australia/Melbourne|Australia/Victoria","Australia/Perth|Australia/West","Australia/Sydney|Australia/ACT","Australia/Sydney|Australia/Canberra","Australia/Sydney|Australia/NSW","Etc/GMT-0|Etc/GMT","Etc/GMT-0|Etc/GMT+0","Etc/GMT-0|Etc/GMT0","Etc/GMT-0|Etc/Greenwich","Etc/GMT-0|GMT","Etc/GMT-0|GMT+0","Etc/GMT-0|GMT-0","Etc/GMT-0|GMT0","Etc/GMT-0|Greenwich","Etc/UTC|Etc/UCT","Etc/UTC|Etc/Universal","Etc/UTC|Etc/Zulu","Etc/UTC|UCT","Etc/UTC|UTC","Etc/UTC|Universal","Etc/UTC|Zulu","Europe/Belgrade|Europe/Ljubljana","Europe/Belgrade|Europe/Podgorica","Europe/Belgrade|Europe/Sarajevo","Europe/Belgrade|Europe/Skopje","Europe/Belgrade|Europe/Zagreb","Europe/Berlin|Arctic/Longyearbyen","Europe/Berlin|Atlantic/Jan_Mayen","Europe/Berlin|Europe/Copenhagen","Europe/Berlin|Europe/Oslo","Europe/Berlin|Europe/Stockholm","Europe/Brussels|Europe/Amsterdam","Europe/Brussels|Europe/Luxembourg","Europe/Chisinau|Europe/Tiraspol","Europe/Dublin|Eire","Europe/Helsinki|Europe/Mariehamn","Europe/Istanbul|Asia/Istanbul","Europe/Istanbul|Turkey","Europe/Kiev|Europe/Kyiv","Europe/Kiev|Europe/Uzhgorod","Europe/Kiev|Europe/Zaporozhye","Europe/Lisbon|Portugal","Europe/London|Europe/Belfast","Europe/London|Europe/Guernsey","Europe/London|Europe/Isle_of_Man","Europe/London|Europe/Jersey","Europe/London|GB","Europe/London|GB-Eire","Europe/Moscow|W-SU","Europe/Paris|Europe/Monaco","Europe/Prague|Europe/Bratislava","Europe/Rome|Europe/San_Marino","Europe/Rome|Europe/Vatican","Europe/Warsaw|Poland","Europe/Zurich|Europe/Busingen","Europe/Zurich|Europe/Vaduz","Indian/Maldives|Indian/Kerguelen","Pacific/Auckland|Antarctica/McMurdo","Pacific/Auckland|Antarctica/South_Pole","Pacific/Auckland|NZ","Pacific/Chatham|NZ-CHAT","Pacific/Easter|Chile/EasterIsland","Pacific/Enderbury|Pacific/Kanton","Pacific/Guadalcanal|Pacific/Pohnpei","Pacific/Guadalcanal|Pacific/Ponape","Pacific/Guam|Pacific/Saipan","Pacific/Honolulu|Pacific/Johnston","Pacific/Honolulu|US/Hawaii","Pacific/Kwajalein|Kwajalein","Pacific/Pago_Pago|Pacific/Midway","Pacific/Pago_Pago|Pacific/Samoa","Pacific/Pago_Pago|US/Samoa","Pacific/Port_Moresby|Antarctica/DumontDUrville","Pacific/Port_Moresby|Pacific/Chuuk","Pacific/Port_Moresby|Pacific/Truk","Pacific/Port_Moresby|Pacific/Yap","Pacific/Tarawa|Pacific/Funafuti","Pacific/Tarawa|Pacific/Majuro","Pacific/Tarawa|Pacific/Wake","Pacific/Tarawa|Pacific/Wallis"],countries:["AD|Europe/Andorra","AE|Asia/Dubai","AF|Asia/Kabul","AG|America/Puerto_Rico America/Antigua","AI|America/Puerto_Rico America/Anguilla","AL|Europe/Tirane","AM|Asia/Yerevan","AO|Africa/Lagos Africa/Luanda","AQ|Antarctica/Casey Antarctica/Davis Antarctica/Mawson Antarctica/Palmer Antarctica/Rothera Antarctica/Troll Antarctica/Vostok Pacific/Auckland Pacific/Port_Moresby Asia/Riyadh Antarctica/McMurdo Antarctica/DumontDUrville Antarctica/Syowa","AR|America/Argentina/Buenos_Aires America/Argentina/Cordoba America/Argentina/Salta America/Argentina/Jujuy America/Argentina/Tucuman America/Argentina/Catamarca America/Argentina/La_Rioja America/Argentina/San_Juan America/Argentina/Mendoza America/Argentina/San_Luis America/Argentina/Rio_Gallegos America/Argentina/Ushuaia","AS|Pacific/Pago_Pago","AT|Europe/Vienna","AU|Australia/Lord_Howe Antarctica/Macquarie Australia/Hobart Australia/Melbourne Australia/Sydney Australia/Broken_Hill Australia/Brisbane Australia/Lindeman Australia/Adelaide Australia/Darwin Australia/Perth Australia/Eucla","AW|America/Puerto_Rico America/Aruba","AX|Europe/Helsinki Europe/Mariehamn","AZ|Asia/Baku","BA|Europe/Belgrade Europe/Sarajevo","BB|America/Barbados","BD|Asia/Dhaka","BE|Europe/Brussels","BF|Africa/Abidjan Africa/Ouagadougou","BG|Europe/Sofia","BH|Asia/Qatar Asia/Bahrain","BI|Africa/Maputo Africa/Bujumbura","BJ|Africa/Lagos Africa/Porto-Novo","BL|America/Puerto_Rico America/St_Barthelemy","BM|Atlantic/Bermuda","BN|Asia/Kuching Asia/Brunei","BO|America/La_Paz","BQ|America/Puerto_Rico America/Kralendijk","BR|America/Noronha America/Belem America/Fortaleza America/Recife America/Araguaina America/Maceio America/Bahia America/Sao_Paulo America/Campo_Grande America/Cuiaba America/Santarem America/Porto_Velho America/Boa_Vista America/Manaus America/Eirunepe America/Rio_Branco","BS|America/Toronto America/Nassau","BT|Asia/Thimphu","BW|Africa/Maputo Africa/Gaborone","BY|Europe/Minsk","BZ|America/Belize","CA|America/St_Johns America/Halifax America/Glace_Bay America/Moncton America/Goose_Bay America/Toronto America/Iqaluit America/Winnipeg America/Resolute America/Rankin_Inlet America/Regina America/Swift_Current America/Edmonton America/Cambridge_Bay America/Inuvik America/Dawson_Creek America/Fort_Nelson America/Whitehorse America/Dawson America/Vancouver America/Panama America/Puerto_Rico America/Phoenix America/Blanc-Sablon America/Atikokan America/Creston","CC|Asia/Yangon Indian/Cocos","CD|Africa/Maputo Africa/Lagos Africa/Kinshasa Africa/Lubumbashi","CF|Africa/Lagos Africa/Bangui","CG|Africa/Lagos Africa/Brazzaville","CH|Europe/Zurich","CI|Africa/Abidjan","CK|Pacific/Rarotonga","CL|America/Santiago America/Punta_Arenas Pacific/Easter","CM|Africa/Lagos Africa/Douala","CN|Asia/Shanghai Asia/Urumqi","CO|America/Bogota","CR|America/Costa_Rica","CU|America/Havana","CV|Atlantic/Cape_Verde","CW|America/Puerto_Rico America/Curacao","CX|Asia/Bangkok Indian/Christmas","CY|Asia/Nicosia Asia/Famagusta","CZ|Europe/Prague","DE|Europe/Zurich Europe/Berlin Europe/Busingen","DJ|Africa/Nairobi Africa/Djibouti","DK|Europe/Berlin Europe/Copenhagen","DM|America/Puerto_Rico America/Dominica","DO|America/Santo_Domingo","DZ|Africa/Algiers","EC|America/Guayaquil Pacific/Galapagos","EE|Europe/Tallinn","EG|Africa/Cairo","EH|Africa/El_Aaiun","ER|Africa/Nairobi Africa/Asmara","ES|Europe/Madrid Africa/Ceuta Atlantic/Canary","ET|Africa/Nairobi Africa/Addis_Ababa","FI|Europe/Helsinki","FJ|Pacific/Fiji","FK|Atlantic/Stanley","FM|Pacific/Kosrae Pacific/Port_Moresby Pacific/Guadalcanal Pacific/Chuuk Pacific/Pohnpei","FO|Atlantic/Faroe","FR|Europe/Paris","GA|Africa/Lagos Africa/Libreville","GB|Europe/London","GD|America/Puerto_Rico America/Grenada","GE|Asia/Tbilisi","GF|America/Cayenne","GG|Europe/London Europe/Guernsey","GH|Africa/Abidjan Africa/Accra","GI|Europe/Gibraltar","GL|America/Nuuk America/Danmarkshavn America/Scoresbysund America/Thule","GM|Africa/Abidjan Africa/Banjul","GN|Africa/Abidjan Africa/Conakry","GP|America/Puerto_Rico America/Guadeloupe","GQ|Africa/Lagos Africa/Malabo","GR|Europe/Athens","GS|Atlantic/South_Georgia","GT|America/Guatemala","GU|Pacific/Guam","GW|Africa/Bissau","GY|America/Guyana","HK|Asia/Hong_Kong","HN|America/Tegucigalpa","HR|Europe/Belgrade Europe/Zagreb","HT|America/Port-au-Prince","HU|Europe/Budapest","ID|Asia/Jakarta Asia/Pontianak Asia/Makassar Asia/Jayapura","IE|Europe/Dublin","IL|Asia/Jerusalem","IM|Europe/London Europe/Isle_of_Man","IN|Asia/Kolkata","IO|Indian/Chagos","IQ|Asia/Baghdad","IR|Asia/Tehran","IS|Africa/Abidjan Atlantic/Reykjavik","IT|Europe/Rome","JE|Europe/London Europe/Jersey","JM|America/Jamaica","JO|Asia/Amman","JP|Asia/Tokyo","KE|Africa/Nairobi","KG|Asia/Bishkek","KH|Asia/Bangkok Asia/Phnom_Penh","KI|Pacific/Tarawa Pacific/Kanton Pacific/Kiritimati","KM|Africa/Nairobi Indian/Comoro","KN|America/Puerto_Rico America/St_Kitts","KP|Asia/Pyongyang","KR|Asia/Seoul","KW|Asia/Riyadh Asia/Kuwait","KY|America/Panama America/Cayman","KZ|Asia/Almaty Asia/Qyzylorda Asia/Qostanay Asia/Aqtobe Asia/Aqtau Asia/Atyrau Asia/Oral","LA|Asia/Bangkok Asia/Vientiane","LB|Asia/Beirut","LC|America/Puerto_Rico America/St_Lucia","LI|Europe/Zurich Europe/Vaduz","LK|Asia/Colombo","LR|Africa/Monrovia","LS|Africa/Johannesburg Africa/Maseru","LT|Europe/Vilnius","LU|Europe/Brussels Europe/Luxembourg","LV|Europe/Riga","LY|Africa/Tripoli","MA|Africa/Casablanca","MC|Europe/Paris Europe/Monaco","MD|Europe/Chisinau","ME|Europe/Belgrade Europe/Podgorica","MF|America/Puerto_Rico America/Marigot","MG|Africa/Nairobi Indian/Antananarivo","MH|Pacific/Tarawa Pacific/Kwajalein Pacific/Majuro","MK|Europe/Belgrade Europe/Skopje","ML|Africa/Abidjan Africa/Bamako","MM|Asia/Yangon","MN|Asia/Ulaanbaatar Asia/Hovd Asia/Choibalsan","MO|Asia/Macau","MP|Pacific/Guam Pacific/Saipan","MQ|America/Martinique","MR|Africa/Abidjan Africa/Nouakchott","MS|America/Puerto_Rico America/Montserrat","MT|Europe/Malta","MU|Indian/Mauritius","MV|Indian/Maldives","MW|Africa/Maputo Africa/Blantyre","MX|America/Mexico_City America/Cancun America/Merida America/Monterrey America/Matamoros America/Chihuahua America/Ciudad_Juarez America/Ojinaga America/Mazatlan America/Bahia_Banderas America/Hermosillo America/Tijuana","MY|Asia/Kuching Asia/Singapore Asia/Kuala_Lumpur","MZ|Africa/Maputo","NA|Africa/Windhoek","NC|Pacific/Noumea","NE|Africa/Lagos Africa/Niamey","NF|Pacific/Norfolk","NG|Africa/Lagos","NI|America/Managua","NL|Europe/Brussels Europe/Amsterdam","NO|Europe/Berlin Europe/Oslo","NP|Asia/Kathmandu","NR|Pacific/Nauru","NU|Pacific/Niue","NZ|Pacific/Auckland Pacific/Chatham","OM|Asia/Dubai Asia/Muscat","PA|America/Panama","PE|America/Lima","PF|Pacific/Tahiti Pacific/Marquesas Pacific/Gambier","PG|Pacific/Port_Moresby Pacific/Bougainville","PH|Asia/Manila","PK|Asia/Karachi","PL|Europe/Warsaw","PM|America/Miquelon","PN|Pacific/Pitcairn","PR|America/Puerto_Rico","PS|Asia/Gaza Asia/Hebron","PT|Europe/Lisbon Atlantic/Madeira Atlantic/Azores","PW|Pacific/Palau","PY|America/Asuncion","QA|Asia/Qatar","RE|Asia/Dubai Indian/Reunion","RO|Europe/Bucharest","RS|Europe/Belgrade","RU|Europe/Kaliningrad Europe/Moscow Europe/Simferopol Europe/Kirov Europe/Volgograd Europe/Astrakhan Europe/Saratov Europe/Ulyanovsk Europe/Samara Asia/Yekaterinburg Asia/Omsk Asia/Novosibirsk Asia/Barnaul Asia/Tomsk Asia/Novokuznetsk Asia/Krasnoyarsk Asia/Irkutsk Asia/Chita Asia/Yakutsk Asia/Khandyga Asia/Vladivostok Asia/Ust-Nera Asia/Magadan Asia/Sakhalin Asia/Srednekolymsk Asia/Kamchatka Asia/Anadyr","RW|Africa/Maputo Africa/Kigali","SA|Asia/Riyadh","SB|Pacific/Guadalcanal","SC|Asia/Dubai Indian/Mahe","SD|Africa/Khartoum","SE|Europe/Berlin Europe/Stockholm","SG|Asia/Singapore","SH|Africa/Abidjan Atlantic/St_Helena","SI|Europe/Belgrade Europe/Ljubljana","SJ|Europe/Berlin Arctic/Longyearbyen","SK|Europe/Prague Europe/Bratislava","SL|Africa/Abidjan Africa/Freetown","SM|Europe/Rome Europe/San_Marino","SN|Africa/Abidjan Africa/Dakar","SO|Africa/Nairobi Africa/Mogadishu","SR|America/Paramaribo","SS|Africa/Juba","ST|Africa/Sao_Tome","SV|America/El_Salvador","SX|America/Puerto_Rico America/Lower_Princes","SY|Asia/Damascus","SZ|Africa/Johannesburg Africa/Mbabane","TC|America/Grand_Turk","TD|Africa/Ndjamena","TF|Asia/Dubai Indian/Maldives Indian/Kerguelen","TG|Africa/Abidjan Africa/Lome","TH|Asia/Bangkok","TJ|Asia/Dushanbe","TK|Pacific/Fakaofo","TL|Asia/Dili","TM|Asia/Ashgabat","TN|Africa/Tunis","TO|Pacific/Tongatapu","TR|Europe/Istanbul","TT|America/Puerto_Rico America/Port_of_Spain","TV|Pacific/Tarawa Pacific/Funafuti","TW|Asia/Taipei","TZ|Africa/Nairobi Africa/Dar_es_Salaam","UA|Europe/Simferopol Europe/Kyiv","UG|Africa/Nairobi Africa/Kampala","UM|Pacific/Pago_Pago Pacific/Tarawa Pacific/Midway Pacific/Wake","US|America/New_York America/Detroit America/Kentucky/Louisville America/Kentucky/Monticello America/Indiana/Indianapolis America/Indiana/Vincennes America/Indiana/Winamac America/Indiana/Marengo America/Indiana/Petersburg America/Indiana/Vevay America/Chicago America/Indiana/Tell_City America/Indiana/Knox America/Menominee America/North_Dakota/Center America/North_Dakota/New_Salem America/North_Dakota/Beulah America/Denver America/Boise America/Phoenix America/Los_Angeles America/Anchorage America/Juneau America/Sitka America/Metlakatla America/Yakutat America/Nome America/Adak Pacific/Honolulu","UY|America/Montevideo","UZ|Asia/Samarkand Asia/Tashkent","VA|Europe/Rome Europe/Vatican","VC|America/Puerto_Rico America/St_Vincent","VE|America/Caracas","VG|America/Puerto_Rico America/Tortola","VI|America/Puerto_Rico America/St_Thomas","VN|Asia/Bangkok Asia/Ho_Chi_Minh","VU|Pacific/Efate","WF|Pacific/Tarawa Pacific/Wallis","WS|Pacific/Apia","YE|Asia/Riyadh Asia/Aden","YT|Africa/Nairobi Indian/Mayotte","ZA|Africa/Johannesburg","ZM|Africa/Maputo Africa/Lusaka","ZW|Africa/Maputo Africa/Harare"]}),O}); \ No newline at end of file diff --git a/Products/ZenUI3/browser/resources/js/timezone/moment.min.js b/Products/ZenUI3/browser/resources/js/timezone/moment.min.js index 8e6866af04..8b80f200c0 100644 --- a/Products/ZenUI3/browser/resources/js/timezone/moment.min.js +++ b/Products/ZenUI3/browser/resources/js/timezone/moment.min.js @@ -1,7 +1,2 @@ -//! moment.js -//! version : 2.10.6 -//! authors : Tim Wood, Iskren Chernev, Moment.js contributors -//! license : MIT -//! momentjs.com -!function(a,b){"object"==typeof exports&&"undefined"!=typeof module?module.exports=b():"function"==typeof define&&define.amd?define(b):a.moment=b()}(this,function(){"use strict";function a(){return Hc.apply(null,arguments)}function b(a){Hc=a}function c(a){return"[object Array]"===Object.prototype.toString.call(a)}function d(a){return a instanceof Date||"[object Date]"===Object.prototype.toString.call(a)}function e(a,b){var c,d=[];for(c=0;c0)for(c in Jc)d=Jc[c],e=b[d],"undefined"!=typeof e&&(a[d]=e);return a}function n(b){m(this,b),this._d=new Date(null!=b._d?b._d.getTime():NaN),Kc===!1&&(Kc=!0,a.updateOffset(this),Kc=!1)}function o(a){return a instanceof n||null!=a&&null!=a._isAMomentObject}function p(a){return 0>a?Math.ceil(a):Math.floor(a)}function q(a){var b=+a,c=0;return 0!==b&&isFinite(b)&&(c=p(b)),c}function r(a,b,c){var d,e=Math.min(a.length,b.length),f=Math.abs(a.length-b.length),g=0;for(d=0;e>d;d++)(c&&a[d]!==b[d]||!c&&q(a[d])!==q(b[d]))&&g++;return g+f}function s(){}function t(a){return a?a.toLowerCase().replace("_","-"):a}function u(a){for(var b,c,d,e,f=0;f0;){if(d=v(e.slice(0,b).join("-")))return d;if(c&&c.length>=b&&r(e,c,!0)>=b-1)break;b--}f++}return null}function v(a){var b=null;if(!Lc[a]&&"undefined"!=typeof module&&module&&module.exports)try{b=Ic._abbr,require("./locale/"+a),w(b)}catch(c){}return Lc[a]}function w(a,b){var c;return a&&(c="undefined"==typeof b?y(a):x(a,b),c&&(Ic=c)),Ic._abbr}function x(a,b){return null!==b?(b.abbr=a,Lc[a]=Lc[a]||new s,Lc[a].set(b),w(a),Lc[a]):(delete Lc[a],null)}function y(a){var b;if(a&&a._locale&&a._locale._abbr&&(a=a._locale._abbr),!a)return Ic;if(!c(a)){if(b=v(a))return b;a=[a]}return u(a)}function z(a,b){var c=a.toLowerCase();Mc[c]=Mc[c+"s"]=Mc[b]=a}function A(a){return"string"==typeof a?Mc[a]||Mc[a.toLowerCase()]:void 0}function B(a){var b,c,d={};for(c in a)f(a,c)&&(b=A(c),b&&(d[b]=a[c]));return d}function C(b,c){return function(d){return null!=d?(E(this,b,d),a.updateOffset(this,c),this):D(this,b)}}function D(a,b){return a._d["get"+(a._isUTC?"UTC":"")+b]()}function E(a,b,c){return a._d["set"+(a._isUTC?"UTC":"")+b](c)}function F(a,b){var c;if("object"==typeof a)for(c in a)this.set(c,a[c]);else if(a=A(a),"function"==typeof this[a])return this[a](b);return this}function G(a,b,c){var d=""+Math.abs(a),e=b-d.length,f=a>=0;return(f?c?"+":"":"-")+Math.pow(10,Math.max(0,e)).toString().substr(1)+d}function H(a,b,c,d){var e=d;"string"==typeof d&&(e=function(){return this[d]()}),a&&(Qc[a]=e),b&&(Qc[b[0]]=function(){return G(e.apply(this,arguments),b[1],b[2])}),c&&(Qc[c]=function(){return this.localeData().ordinal(e.apply(this,arguments),a)})}function I(a){return a.match(/\[[\s\S]/)?a.replace(/^\[|\]$/g,""):a.replace(/\\/g,"")}function J(a){var b,c,d=a.match(Nc);for(b=0,c=d.length;c>b;b++)Qc[d[b]]?d[b]=Qc[d[b]]:d[b]=I(d[b]);return function(e){var f="";for(b=0;c>b;b++)f+=d[b]instanceof Function?d[b].call(e,a):d[b];return f}}function K(a,b){return a.isValid()?(b=L(b,a.localeData()),Pc[b]=Pc[b]||J(b),Pc[b](a)):a.localeData().invalidDate()}function L(a,b){function c(a){return b.longDateFormat(a)||a}var d=5;for(Oc.lastIndex=0;d>=0&&Oc.test(a);)a=a.replace(Oc,c),Oc.lastIndex=0,d-=1;return a}function M(a){return"function"==typeof a&&"[object Function]"===Object.prototype.toString.call(a)}function N(a,b,c){dd[a]=M(b)?b:function(a){return a&&c?c:b}}function O(a,b){return f(dd,a)?dd[a](b._strict,b._locale):new RegExp(P(a))}function P(a){return a.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(a,b,c,d,e){return b||c||d||e}).replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}function Q(a,b){var c,d=b;for("string"==typeof a&&(a=[a]),"number"==typeof b&&(d=function(a,c){c[b]=q(a)}),c=0;cd;d++){if(e=h([2e3,d]),c&&!this._longMonthsParse[d]&&(this._longMonthsParse[d]=new RegExp("^"+this.months(e,"").replace(".","")+"$","i"),this._shortMonthsParse[d]=new RegExp("^"+this.monthsShort(e,"").replace(".","")+"$","i")),c||this._monthsParse[d]||(f="^"+this.months(e,"")+"|^"+this.monthsShort(e,""),this._monthsParse[d]=new RegExp(f.replace(".",""),"i")),c&&"MMMM"===b&&this._longMonthsParse[d].test(a))return d;if(c&&"MMM"===b&&this._shortMonthsParse[d].test(a))return d;if(!c&&this._monthsParse[d].test(a))return d}}function X(a,b){var c;return"string"==typeof b&&(b=a.localeData().monthsParse(b),"number"!=typeof b)?a:(c=Math.min(a.date(),T(a.year(),b)),a._d["set"+(a._isUTC?"UTC":"")+"Month"](b,c),a)}function Y(b){return null!=b?(X(this,b),a.updateOffset(this,!0),this):D(this,"Month")}function Z(){return T(this.year(),this.month())}function $(a){var b,c=a._a;return c&&-2===j(a).overflow&&(b=c[gd]<0||c[gd]>11?gd:c[hd]<1||c[hd]>T(c[fd],c[gd])?hd:c[id]<0||c[id]>24||24===c[id]&&(0!==c[jd]||0!==c[kd]||0!==c[ld])?id:c[jd]<0||c[jd]>59?jd:c[kd]<0||c[kd]>59?kd:c[ld]<0||c[ld]>999?ld:-1,j(a)._overflowDayOfYear&&(fd>b||b>hd)&&(b=hd),j(a).overflow=b),a}function _(b){a.suppressDeprecationWarnings===!1&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+b)}function aa(a,b){var c=!0;return g(function(){return c&&(_(a+"\n"+(new Error).stack),c=!1),b.apply(this,arguments)},b)}function ba(a,b){od[a]||(_(b),od[a]=!0)}function ca(a){var b,c,d=a._i,e=pd.exec(d);if(e){for(j(a).iso=!0,b=0,c=qd.length;c>b;b++)if(qd[b][1].exec(d)){a._f=qd[b][0];break}for(b=0,c=rd.length;c>b;b++)if(rd[b][1].exec(d)){a._f+=(e[6]||" ")+rd[b][0];break}d.match(ad)&&(a._f+="Z"),va(a)}else a._isValid=!1}function da(b){var c=sd.exec(b._i);return null!==c?void(b._d=new Date(+c[1])):(ca(b),void(b._isValid===!1&&(delete b._isValid,a.createFromInputFallback(b))))}function ea(a,b,c,d,e,f,g){var h=new Date(a,b,c,d,e,f,g);return 1970>a&&h.setFullYear(a),h}function fa(a){var b=new Date(Date.UTC.apply(null,arguments));return 1970>a&&b.setUTCFullYear(a),b}function ga(a){return ha(a)?366:365}function ha(a){return a%4===0&&a%100!==0||a%400===0}function ia(){return ha(this.year())}function ja(a,b,c){var d,e=c-b,f=c-a.day();return f>e&&(f-=7),e-7>f&&(f+=7),d=Da(a).add(f,"d"),{week:Math.ceil(d.dayOfYear()/7),year:d.year()}}function ka(a){return ja(a,this._week.dow,this._week.doy).week}function la(){return this._week.dow}function ma(){return this._week.doy}function na(a){var b=this.localeData().week(this);return null==a?b:this.add(7*(a-b),"d")}function oa(a){var b=ja(this,1,4).week;return null==a?b:this.add(7*(a-b),"d")}function pa(a,b,c,d,e){var f,g=6+e-d,h=fa(a,0,1+g),i=h.getUTCDay();return e>i&&(i+=7),c=null!=c?1*c:e,f=1+g+7*(b-1)-i+c,{year:f>0?a:a-1,dayOfYear:f>0?f:ga(a-1)+f}}function qa(a){var b=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==a?b:this.add(a-b,"d")}function ra(a,b,c){return null!=a?a:null!=b?b:c}function sa(a){var b=new Date;return a._useUTC?[b.getUTCFullYear(),b.getUTCMonth(),b.getUTCDate()]:[b.getFullYear(),b.getMonth(),b.getDate()]}function ta(a){var b,c,d,e,f=[];if(!a._d){for(d=sa(a),a._w&&null==a._a[hd]&&null==a._a[gd]&&ua(a),a._dayOfYear&&(e=ra(a._a[fd],d[fd]),a._dayOfYear>ga(e)&&(j(a)._overflowDayOfYear=!0),c=fa(e,0,a._dayOfYear),a._a[gd]=c.getUTCMonth(),a._a[hd]=c.getUTCDate()),b=0;3>b&&null==a._a[b];++b)a._a[b]=f[b]=d[b];for(;7>b;b++)a._a[b]=f[b]=null==a._a[b]?2===b?1:0:a._a[b];24===a._a[id]&&0===a._a[jd]&&0===a._a[kd]&&0===a._a[ld]&&(a._nextDay=!0,a._a[id]=0),a._d=(a._useUTC?fa:ea).apply(null,f),null!=a._tzm&&a._d.setUTCMinutes(a._d.getUTCMinutes()-a._tzm),a._nextDay&&(a._a[id]=24)}}function ua(a){var b,c,d,e,f,g,h;b=a._w,null!=b.GG||null!=b.W||null!=b.E?(f=1,g=4,c=ra(b.GG,a._a[fd],ja(Da(),1,4).year),d=ra(b.W,1),e=ra(b.E,1)):(f=a._locale._week.dow,g=a._locale._week.doy,c=ra(b.gg,a._a[fd],ja(Da(),f,g).year),d=ra(b.w,1),null!=b.d?(e=b.d,f>e&&++d):e=null!=b.e?b.e+f:f),h=pa(c,d,e,g,f),a._a[fd]=h.year,a._dayOfYear=h.dayOfYear}function va(b){if(b._f===a.ISO_8601)return void ca(b);b._a=[],j(b).empty=!0;var c,d,e,f,g,h=""+b._i,i=h.length,k=0;for(e=L(b._f,b._locale).match(Nc)||[],c=0;c0&&j(b).unusedInput.push(g),h=h.slice(h.indexOf(d)+d.length),k+=d.length),Qc[f]?(d?j(b).empty=!1:j(b).unusedTokens.push(f),S(f,d,b)):b._strict&&!d&&j(b).unusedTokens.push(f);j(b).charsLeftOver=i-k,h.length>0&&j(b).unusedInput.push(h),j(b).bigHour===!0&&b._a[id]<=12&&b._a[id]>0&&(j(b).bigHour=void 0),b._a[id]=wa(b._locale,b._a[id],b._meridiem),ta(b),$(b)}function wa(a,b,c){var d;return null==c?b:null!=a.meridiemHour?a.meridiemHour(b,c):null!=a.isPM?(d=a.isPM(c),d&&12>b&&(b+=12),d||12!==b||(b=0),b):b}function xa(a){var b,c,d,e,f;if(0===a._f.length)return j(a).invalidFormat=!0,void(a._d=new Date(NaN));for(e=0;ef)&&(d=f,c=b));g(a,c||b)}function ya(a){if(!a._d){var b=B(a._i);a._a=[b.year,b.month,b.day||b.date,b.hour,b.minute,b.second,b.millisecond],ta(a)}}function za(a){var b=new n($(Aa(a)));return b._nextDay&&(b.add(1,"d"),b._nextDay=void 0),b}function Aa(a){var b=a._i,e=a._f;return a._locale=a._locale||y(a._l),null===b||void 0===e&&""===b?l({nullInput:!0}):("string"==typeof b&&(a._i=b=a._locale.preparse(b)),o(b)?new n($(b)):(c(e)?xa(a):e?va(a):d(b)?a._d=b:Ba(a),a))}function Ba(b){var f=b._i;void 0===f?b._d=new Date:d(f)?b._d=new Date(+f):"string"==typeof f?da(b):c(f)?(b._a=e(f.slice(0),function(a){return parseInt(a,10)}),ta(b)):"object"==typeof f?ya(b):"number"==typeof f?b._d=new Date(f):a.createFromInputFallback(b)}function Ca(a,b,c,d,e){var f={};return"boolean"==typeof c&&(d=c,c=void 0),f._isAMomentObject=!0,f._useUTC=f._isUTC=e,f._l=c,f._i=a,f._f=b,f._strict=d,za(f)}function Da(a,b,c,d){return Ca(a,b,c,d,!1)}function Ea(a,b){var d,e;if(1===b.length&&c(b[0])&&(b=b[0]),!b.length)return Da();for(d=b[0],e=1;ea&&(a=-a,c="-"),c+G(~~(a/60),2)+b+G(~~a%60,2)})}function Ka(a){var b=(a||"").match(ad)||[],c=b[b.length-1]||[],d=(c+"").match(xd)||["-",0,0],e=+(60*d[1])+q(d[2]);return"+"===d[0]?e:-e}function La(b,c){var e,f;return c._isUTC?(e=c.clone(),f=(o(b)||d(b)?+b:+Da(b))-+e,e._d.setTime(+e._d+f),a.updateOffset(e,!1),e):Da(b).local()}function Ma(a){return 15*-Math.round(a._d.getTimezoneOffset()/15)}function Na(b,c){var d,e=this._offset||0;return null!=b?("string"==typeof b&&(b=Ka(b)),Math.abs(b)<16&&(b=60*b),!this._isUTC&&c&&(d=Ma(this)),this._offset=b,this._isUTC=!0,null!=d&&this.add(d,"m"),e!==b&&(!c||this._changeInProgress?bb(this,Ya(b-e,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,a.updateOffset(this,!0),this._changeInProgress=null)),this):this._isUTC?e:Ma(this)}function Oa(a,b){return null!=a?("string"!=typeof a&&(a=-a),this.utcOffset(a,b),this):-this.utcOffset()}function Pa(a){return this.utcOffset(0,a)}function Qa(a){return this._isUTC&&(this.utcOffset(0,a),this._isUTC=!1,a&&this.subtract(Ma(this),"m")),this}function Ra(){return this._tzm?this.utcOffset(this._tzm):"string"==typeof this._i&&this.utcOffset(Ka(this._i)),this}function Sa(a){return a=a?Da(a).utcOffset():0,(this.utcOffset()-a)%60===0}function Ta(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()}function Ua(){if("undefined"!=typeof this._isDSTShifted)return this._isDSTShifted;var a={};if(m(a,this),a=Aa(a),a._a){var b=a._isUTC?h(a._a):Da(a._a);this._isDSTShifted=this.isValid()&&r(a._a,b.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}function Va(){return!this._isUTC}function Wa(){return this._isUTC}function Xa(){return this._isUTC&&0===this._offset}function Ya(a,b){var c,d,e,g=a,h=null;return Ia(a)?g={ms:a._milliseconds,d:a._days,M:a._months}:"number"==typeof a?(g={},b?g[b]=a:g.milliseconds=a):(h=yd.exec(a))?(c="-"===h[1]?-1:1,g={y:0,d:q(h[hd])*c,h:q(h[id])*c,m:q(h[jd])*c,s:q(h[kd])*c,ms:q(h[ld])*c}):(h=zd.exec(a))?(c="-"===h[1]?-1:1,g={y:Za(h[2],c),M:Za(h[3],c),d:Za(h[4],c),h:Za(h[5],c),m:Za(h[6],c),s:Za(h[7],c),w:Za(h[8],c)}):null==g?g={}:"object"==typeof g&&("from"in g||"to"in g)&&(e=_a(Da(g.from),Da(g.to)),g={},g.ms=e.milliseconds,g.M=e.months),d=new Ha(g),Ia(a)&&f(a,"_locale")&&(d._locale=a._locale),d}function Za(a,b){var c=a&&parseFloat(a.replace(",","."));return(isNaN(c)?0:c)*b}function $a(a,b){var c={milliseconds:0,months:0};return c.months=b.month()-a.month()+12*(b.year()-a.year()),a.clone().add(c.months,"M").isAfter(b)&&--c.months,c.milliseconds=+b-+a.clone().add(c.months,"M"),c}function _a(a,b){var c;return b=La(b,a),a.isBefore(b)?c=$a(a,b):(c=$a(b,a),c.milliseconds=-c.milliseconds,c.months=-c.months),c}function ab(a,b){return function(c,d){var e,f;return null===d||isNaN(+d)||(ba(b,"moment()."+b+"(period, number) is deprecated. Please use moment()."+b+"(number, period)."),f=c,c=d,d=f),c="string"==typeof c?+c:c,e=Ya(c,d),bb(this,e,a),this}}function bb(b,c,d,e){var f=c._milliseconds,g=c._days,h=c._months;e=null==e?!0:e,f&&b._d.setTime(+b._d+f*d),g&&E(b,"Date",D(b,"Date")+g*d),h&&X(b,D(b,"Month")+h*d),e&&a.updateOffset(b,g||h)}function cb(a,b){var c=a||Da(),d=La(c,this).startOf("day"),e=this.diff(d,"days",!0),f=-6>e?"sameElse":-1>e?"lastWeek":0>e?"lastDay":1>e?"sameDay":2>e?"nextDay":7>e?"nextWeek":"sameElse";return this.format(b&&b[f]||this.localeData().calendar(f,this,Da(c)))}function db(){return new n(this)}function eb(a,b){var c;return b=A("undefined"!=typeof b?b:"millisecond"),"millisecond"===b?(a=o(a)?a:Da(a),+this>+a):(c=o(a)?+a:+Da(a),c<+this.clone().startOf(b))}function fb(a,b){var c;return b=A("undefined"!=typeof b?b:"millisecond"),"millisecond"===b?(a=o(a)?a:Da(a),+a>+this):(c=o(a)?+a:+Da(a),+this.clone().endOf(b)b-f?(c=a.clone().add(e-1,"months"),d=(b-f)/(f-c)):(c=a.clone().add(e+1,"months"),d=(b-f)/(c-f)),-(e+d)}function kb(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")}function lb(){var a=this.clone().utc();return 0b;b++)if(this._weekdaysParse[b]||(c=Da([2e3,1]).day(b),d="^"+this.weekdays(c,"")+"|^"+this.weekdaysShort(c,"")+"|^"+this.weekdaysMin(c,""),this._weekdaysParse[b]=new RegExp(d.replace(".",""),"i")),this._weekdaysParse[b].test(a))return b}function Pb(a){var b=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=a?(a=Kb(a,this.localeData()),this.add(a-b,"d")):b}function Qb(a){var b=(this.day()+7-this.localeData()._week.dow)%7;return null==a?b:this.add(a-b,"d")}function Rb(a){return null==a?this.day()||7:this.day(this.day()%7?a:a-7)}function Sb(a,b){H(a,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),b)})}function Tb(a,b){return b._meridiemParse}function Ub(a){return"p"===(a+"").toLowerCase().charAt(0)}function Vb(a,b,c){return a>11?c?"pm":"PM":c?"am":"AM"}function Wb(a,b){b[ld]=q(1e3*("0."+a))}function Xb(){return this._isUTC?"UTC":""}function Yb(){return this._isUTC?"Coordinated Universal Time":""}function Zb(a){return Da(1e3*a)}function $b(){return Da.apply(null,arguments).parseZone()}function _b(a,b,c){var d=this._calendar[a];return"function"==typeof d?d.call(b,c):d}function ac(a){var b=this._longDateFormat[a],c=this._longDateFormat[a.toUpperCase()];return b||!c?b:(this._longDateFormat[a]=c.replace(/MMMM|MM|DD|dddd/g,function(a){return a.slice(1)}),this._longDateFormat[a])}function bc(){return this._invalidDate}function cc(a){return this._ordinal.replace("%d",a)}function dc(a){return a}function ec(a,b,c,d){var e=this._relativeTime[c];return"function"==typeof e?e(a,b,c,d):e.replace(/%d/i,a)}function fc(a,b){var c=this._relativeTime[a>0?"future":"past"];return"function"==typeof c?c(b):c.replace(/%s/i,b)}function gc(a){var b,c;for(c in a)b=a[c],"function"==typeof b?this[c]=b:this["_"+c]=b;this._ordinalParseLenient=new RegExp(this._ordinalParse.source+"|"+/\d{1,2}/.source)}function hc(a,b,c,d){var e=y(),f=h().set(d,b);return e[c](f,a)}function ic(a,b,c,d,e){if("number"==typeof a&&(b=a,a=void 0),a=a||"",null!=b)return hc(a,b,c,e);var f,g=[];for(f=0;d>f;f++)g[f]=hc(a,f,c,e);return g}function jc(a,b){return ic(a,b,"months",12,"month")}function kc(a,b){return ic(a,b,"monthsShort",12,"month")}function lc(a,b){return ic(a,b,"weekdays",7,"day")}function mc(a,b){return ic(a,b,"weekdaysShort",7,"day")}function nc(a,b){return ic(a,b,"weekdaysMin",7,"day")}function oc(){var a=this._data;return this._milliseconds=Wd(this._milliseconds),this._days=Wd(this._days),this._months=Wd(this._months),a.milliseconds=Wd(a.milliseconds),a.seconds=Wd(a.seconds),a.minutes=Wd(a.minutes),a.hours=Wd(a.hours),a.months=Wd(a.months),a.years=Wd(a.years),this}function pc(a,b,c,d){var e=Ya(b,c);return a._milliseconds+=d*e._milliseconds,a._days+=d*e._days,a._months+=d*e._months,a._bubble()}function qc(a,b){return pc(this,a,b,1)}function rc(a,b){return pc(this,a,b,-1)}function sc(a){return 0>a?Math.floor(a):Math.ceil(a)}function tc(){var a,b,c,d,e,f=this._milliseconds,g=this._days,h=this._months,i=this._data;return f>=0&&g>=0&&h>=0||0>=f&&0>=g&&0>=h||(f+=864e5*sc(vc(h)+g),g=0,h=0),i.milliseconds=f%1e3,a=p(f/1e3),i.seconds=a%60,b=p(a/60),i.minutes=b%60,c=p(b/60),i.hours=c%24,g+=p(c/24),e=p(uc(g)),h+=e,g-=sc(vc(e)),d=p(h/12),h%=12,i.days=g,i.months=h,i.years=d,this}function uc(a){return 4800*a/146097}function vc(a){return 146097*a/4800}function wc(a){var b,c,d=this._milliseconds;if(a=A(a),"month"===a||"year"===a)return b=this._days+d/864e5,c=this._months+uc(b),"month"===a?c:c/12;switch(b=this._days+Math.round(vc(this._months)),a){case"week":return b/7+d/6048e5;case"day":return b+d/864e5;case"hour":return 24*b+d/36e5;case"minute":return 1440*b+d/6e4;case"second":return 86400*b+d/1e3;case"millisecond":return Math.floor(864e5*b)+d;default:throw new Error("Unknown unit "+a)}}function xc(){return this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*q(this._months/12)}function yc(a){return function(){return this.as(a)}}function zc(a){return a=A(a),this[a+"s"]()}function Ac(a){return function(){return this._data[a]}}function Bc(){return p(this.days()/7)}function Cc(a,b,c,d,e){return e.relativeTime(b||1,!!c,a,d)}function Dc(a,b,c){var d=Ya(a).abs(),e=ke(d.as("s")),f=ke(d.as("m")),g=ke(d.as("h")),h=ke(d.as("d")),i=ke(d.as("M")),j=ke(d.as("y")),k=e0,k[4]=c,Cc.apply(null,k)}function Ec(a,b){return void 0===le[a]?!1:void 0===b?le[a]:(le[a]=b,!0)}function Fc(a){var b=this.localeData(),c=Dc(this,!a,b);return a&&(c=b.pastFuture(+this,c)),b.postformat(c)}function Gc(){var a,b,c,d=me(this._milliseconds)/1e3,e=me(this._days),f=me(this._months);a=p(d/60),b=p(a/60),d%=60,a%=60,c=p(f/12),f%=12;var g=c,h=f,i=e,j=b,k=a,l=d,m=this.asSeconds();return m?(0>m?"-":"")+"P"+(g?g+"Y":"")+(h?h+"M":"")+(i?i+"D":"")+(j||k||l?"T":"")+(j?j+"H":"")+(k?k+"M":"")+(l?l+"S":""):"P0D"}var Hc,Ic,Jc=a.momentProperties=[],Kc=!1,Lc={},Mc={},Nc=/(\[[^\[]*\])|(\\)?(Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Q|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Oc=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,Pc={},Qc={},Rc=/\d/,Sc=/\d\d/,Tc=/\d{3}/,Uc=/\d{4}/,Vc=/[+-]?\d{6}/,Wc=/\d\d?/,Xc=/\d{1,3}/,Yc=/\d{1,4}/,Zc=/[+-]?\d{1,6}/,$c=/\d+/,_c=/[+-]?\d+/,ad=/Z|[+-]\d\d:?\d\d/gi,bd=/[+-]?\d+(\.\d{1,3})?/,cd=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,dd={},ed={},fd=0,gd=1,hd=2,id=3,jd=4,kd=5,ld=6;H("M",["MM",2],"Mo",function(){return this.month()+1}),H("MMM",0,0,function(a){return this.localeData().monthsShort(this,a)}),H("MMMM",0,0,function(a){return this.localeData().months(this,a)}),z("month","M"),N("M",Wc),N("MM",Wc,Sc),N("MMM",cd),N("MMMM",cd),Q(["M","MM"],function(a,b){b[gd]=q(a)-1}),Q(["MMM","MMMM"],function(a,b,c,d){var e=c._locale.monthsParse(a,d,c._strict);null!=e?b[gd]=e:j(c).invalidMonth=a});var md="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),nd="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),od={};a.suppressDeprecationWarnings=!1;var pd=/^\s*(?:[+-]\d{6}|\d{4})-(?:(\d\d-\d\d)|(W\d\d$)|(W\d\d-\d)|(\d\d\d))((T| )(\d\d(:\d\d(:\d\d(\.\d+)?)?)?)?([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,qd=[["YYYYYY-MM-DD",/[+-]\d{6}-\d{2}-\d{2}/],["YYYY-MM-DD",/\d{4}-\d{2}-\d{2}/],["GGGG-[W]WW-E",/\d{4}-W\d{2}-\d/],["GGGG-[W]WW",/\d{4}-W\d{2}/],["YYYY-DDD",/\d{4}-\d{3}/]],rd=[["HH:mm:ss.SSSS",/(T| )\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss",/(T| )\d\d:\d\d:\d\d/],["HH:mm",/(T| )\d\d:\d\d/],["HH",/(T| )\d\d/]],sd=/^\/?Date\((\-?\d+)/i;a.createFromInputFallback=aa("moment construction falls back to js Date. This is discouraged and will be removed in upcoming major release. Please refer to https://github.com/moment/moment/issues/1407 for more info.",function(a){a._d=new Date(a._i+(a._useUTC?" UTC":""))}),H(0,["YY",2],0,function(){return this.year()%100}),H(0,["YYYY",4],0,"year"),H(0,["YYYYY",5],0,"year"),H(0,["YYYYYY",6,!0],0,"year"),z("year","y"),N("Y",_c),N("YY",Wc,Sc),N("YYYY",Yc,Uc),N("YYYYY",Zc,Vc),N("YYYYYY",Zc,Vc),Q(["YYYYY","YYYYYY"],fd),Q("YYYY",function(b,c){c[fd]=2===b.length?a.parseTwoDigitYear(b):q(b)}),Q("YY",function(b,c){c[fd]=a.parseTwoDigitYear(b)}),a.parseTwoDigitYear=function(a){return q(a)+(q(a)>68?1900:2e3)};var td=C("FullYear",!1);H("w",["ww",2],"wo","week"),H("W",["WW",2],"Wo","isoWeek"),z("week","w"),z("isoWeek","W"),N("w",Wc),N("ww",Wc,Sc),N("W",Wc),N("WW",Wc,Sc),R(["w","ww","W","WW"],function(a,b,c,d){b[d.substr(0,1)]=q(a)});var ud={dow:0,doy:6};H("DDD",["DDDD",3],"DDDo","dayOfYear"),z("dayOfYear","DDD"),N("DDD",Xc),N("DDDD",Tc),Q(["DDD","DDDD"],function(a,b,c){c._dayOfYear=q(a)}),a.ISO_8601=function(){};var vd=aa("moment().min is deprecated, use moment.min instead. https://github.com/moment/moment/issues/1548",function(){var a=Da.apply(null,arguments);return this>a?this:a}),wd=aa("moment().max is deprecated, use moment.max instead. https://github.com/moment/moment/issues/1548",function(){var a=Da.apply(null,arguments);return a>this?this:a});Ja("Z",":"),Ja("ZZ",""),N("Z",ad),N("ZZ",ad),Q(["Z","ZZ"],function(a,b,c){c._useUTC=!0,c._tzm=Ka(a)});var xd=/([\+\-]|\d\d)/gi;a.updateOffset=function(){};var yd=/(\-)?(?:(\d*)\.)?(\d+)\:(\d+)(?:\:(\d+)\.?(\d{3})?)?/,zd=/^(-)?P(?:(?:([0-9,.]*)Y)?(?:([0-9,.]*)M)?(?:([0-9,.]*)D)?(?:T(?:([0-9,.]*)H)?(?:([0-9,.]*)M)?(?:([0-9,.]*)S)?)?|([0-9,.]*)W)$/;Ya.fn=Ha.prototype;var Ad=ab(1,"add"),Bd=ab(-1,"subtract");a.defaultFormat="YYYY-MM-DDTHH:mm:ssZ";var Cd=aa("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(a){return void 0===a?this.localeData():this.locale(a)});H(0,["gg",2],0,function(){return this.weekYear()%100}),H(0,["GG",2],0,function(){return this.isoWeekYear()%100}),Db("gggg","weekYear"),Db("ggggg","weekYear"),Db("GGGG","isoWeekYear"),Db("GGGGG","isoWeekYear"),z("weekYear","gg"),z("isoWeekYear","GG"),N("G",_c),N("g",_c),N("GG",Wc,Sc),N("gg",Wc,Sc),N("GGGG",Yc,Uc),N("gggg",Yc,Uc),N("GGGGG",Zc,Vc),N("ggggg",Zc,Vc),R(["gggg","ggggg","GGGG","GGGGG"],function(a,b,c,d){b[d.substr(0,2)]=q(a)}),R(["gg","GG"],function(b,c,d,e){c[e]=a.parseTwoDigitYear(b)}),H("Q",0,0,"quarter"),z("quarter","Q"),N("Q",Rc),Q("Q",function(a,b){b[gd]=3*(q(a)-1)}),H("D",["DD",2],"Do","date"),z("date","D"),N("D",Wc),N("DD",Wc,Sc),N("Do",function(a,b){return a?b._ordinalParse:b._ordinalParseLenient}),Q(["D","DD"],hd),Q("Do",function(a,b){b[hd]=q(a.match(Wc)[0],10)});var Dd=C("Date",!0);H("d",0,"do","day"),H("dd",0,0,function(a){return this.localeData().weekdaysMin(this,a)}),H("ddd",0,0,function(a){return this.localeData().weekdaysShort(this,a)}),H("dddd",0,0,function(a){return this.localeData().weekdays(this,a)}),H("e",0,0,"weekday"),H("E",0,0,"isoWeekday"),z("day","d"),z("weekday","e"),z("isoWeekday","E"),N("d",Wc),N("e",Wc),N("E",Wc),N("dd",cd),N("ddd",cd),N("dddd",cd),R(["dd","ddd","dddd"],function(a,b,c){var d=c._locale.weekdaysParse(a);null!=d?b.d=d:j(c).invalidWeekday=a}),R(["d","e","E"],function(a,b,c,d){b[d]=q(a)});var Ed="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Fd="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Gd="Su_Mo_Tu_We_Th_Fr_Sa".split("_");H("H",["HH",2],0,"hour"),H("h",["hh",2],0,function(){return this.hours()%12||12}),Sb("a",!0),Sb("A",!1),z("hour","h"),N("a",Tb),N("A",Tb),N("H",Wc),N("h",Wc),N("HH",Wc,Sc),N("hh",Wc,Sc),Q(["H","HH"],id),Q(["a","A"],function(a,b,c){c._isPm=c._locale.isPM(a),c._meridiem=a}),Q(["h","hh"],function(a,b,c){b[id]=q(a),j(c).bigHour=!0});var Hd=/[ap]\.?m?\.?/i,Id=C("Hours",!0);H("m",["mm",2],0,"minute"),z("minute","m"),N("m",Wc),N("mm",Wc,Sc),Q(["m","mm"],jd);var Jd=C("Minutes",!1);H("s",["ss",2],0,"second"),z("second","s"),N("s",Wc),N("ss",Wc,Sc),Q(["s","ss"],kd);var Kd=C("Seconds",!1);H("S",0,0,function(){return~~(this.millisecond()/100)}),H(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),H(0,["SSS",3],0,"millisecond"),H(0,["SSSS",4],0,function(){return 10*this.millisecond()}),H(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),H(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),H(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),H(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),H(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),z("millisecond","ms"),N("S",Xc,Rc),N("SS",Xc,Sc),N("SSS",Xc,Tc);var Ld;for(Ld="SSSS";Ld.length<=9;Ld+="S")N(Ld,$c);for(Ld="S";Ld.length<=9;Ld+="S")Q(Ld,Wb);var Md=C("Milliseconds",!1);H("z",0,0,"zoneAbbr"),H("zz",0,0,"zoneName");var Nd=n.prototype;Nd.add=Ad,Nd.calendar=cb,Nd.clone=db,Nd.diff=ib,Nd.endOf=ub,Nd.format=mb,Nd.from=nb,Nd.fromNow=ob,Nd.to=pb,Nd.toNow=qb,Nd.get=F,Nd.invalidAt=Cb,Nd.isAfter=eb,Nd.isBefore=fb,Nd.isBetween=gb,Nd.isSame=hb,Nd.isValid=Ab,Nd.lang=Cd,Nd.locale=rb,Nd.localeData=sb,Nd.max=wd,Nd.min=vd,Nd.parsingFlags=Bb,Nd.set=F,Nd.startOf=tb,Nd.subtract=Bd,Nd.toArray=yb,Nd.toObject=zb,Nd.toDate=xb,Nd.toISOString=lb,Nd.toJSON=lb,Nd.toString=kb,Nd.unix=wb,Nd.valueOf=vb,Nd.year=td,Nd.isLeapYear=ia,Nd.weekYear=Fb,Nd.isoWeekYear=Gb,Nd.quarter=Nd.quarters=Jb,Nd.month=Y,Nd.daysInMonth=Z,Nd.week=Nd.weeks=na,Nd.isoWeek=Nd.isoWeeks=oa,Nd.weeksInYear=Ib,Nd.isoWeeksInYear=Hb,Nd.date=Dd,Nd.day=Nd.days=Pb,Nd.weekday=Qb,Nd.isoWeekday=Rb,Nd.dayOfYear=qa,Nd.hour=Nd.hours=Id,Nd.minute=Nd.minutes=Jd,Nd.second=Nd.seconds=Kd, -Nd.millisecond=Nd.milliseconds=Md,Nd.utcOffset=Na,Nd.utc=Pa,Nd.local=Qa,Nd.parseZone=Ra,Nd.hasAlignedHourOffset=Sa,Nd.isDST=Ta,Nd.isDSTShifted=Ua,Nd.isLocal=Va,Nd.isUtcOffset=Wa,Nd.isUtc=Xa,Nd.isUTC=Xa,Nd.zoneAbbr=Xb,Nd.zoneName=Yb,Nd.dates=aa("dates accessor is deprecated. Use date instead.",Dd),Nd.months=aa("months accessor is deprecated. Use month instead",Y),Nd.years=aa("years accessor is deprecated. Use year instead",td),Nd.zone=aa("moment().zone is deprecated, use moment().utcOffset instead. https://github.com/moment/moment/issues/1779",Oa);var Od=Nd,Pd={sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},Qd={LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},Rd="Invalid date",Sd="%d",Td=/\d{1,2}/,Ud={future:"in %s",past:"%s ago",s:"a few seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},Vd=s.prototype;Vd._calendar=Pd,Vd.calendar=_b,Vd._longDateFormat=Qd,Vd.longDateFormat=ac,Vd._invalidDate=Rd,Vd.invalidDate=bc,Vd._ordinal=Sd,Vd.ordinal=cc,Vd._ordinalParse=Td,Vd.preparse=dc,Vd.postformat=dc,Vd._relativeTime=Ud,Vd.relativeTime=ec,Vd.pastFuture=fc,Vd.set=gc,Vd.months=U,Vd._months=md,Vd.monthsShort=V,Vd._monthsShort=nd,Vd.monthsParse=W,Vd.week=ka,Vd._week=ud,Vd.firstDayOfYear=ma,Vd.firstDayOfWeek=la,Vd.weekdays=Lb,Vd._weekdays=Ed,Vd.weekdaysMin=Nb,Vd._weekdaysMin=Gd,Vd.weekdaysShort=Mb,Vd._weekdaysShort=Fd,Vd.weekdaysParse=Ob,Vd.isPM=Ub,Vd._meridiemParse=Hd,Vd.meridiem=Vb,w("en",{ordinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(a){var b=a%10,c=1===q(a%100/10)?"th":1===b?"st":2===b?"nd":3===b?"rd":"th";return a+c}}),a.lang=aa("moment.lang is deprecated. Use moment.locale instead.",w),a.langData=aa("moment.langData is deprecated. Use moment.localeData instead.",y);var Wd=Math.abs,Xd=yc("ms"),Yd=yc("s"),Zd=yc("m"),$d=yc("h"),_d=yc("d"),ae=yc("w"),be=yc("M"),ce=yc("y"),de=Ac("milliseconds"),ee=Ac("seconds"),fe=Ac("minutes"),ge=Ac("hours"),he=Ac("days"),ie=Ac("months"),je=Ac("years"),ke=Math.round,le={s:45,m:45,h:22,d:26,M:11},me=Math.abs,ne=Ha.prototype;ne.abs=oc,ne.add=qc,ne.subtract=rc,ne.as=wc,ne.asMilliseconds=Xd,ne.asSeconds=Yd,ne.asMinutes=Zd,ne.asHours=$d,ne.asDays=_d,ne.asWeeks=ae,ne.asMonths=be,ne.asYears=ce,ne.valueOf=xc,ne._bubble=tc,ne.get=zc,ne.milliseconds=de,ne.seconds=ee,ne.minutes=fe,ne.hours=ge,ne.days=he,ne.weeks=Bc,ne.months=ie,ne.years=je,ne.humanize=Fc,ne.toISOString=Gc,ne.toString=Gc,ne.toJSON=Gc,ne.locale=rb,ne.localeData=sb,ne.toIsoString=aa("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",Gc),ne.lang=Cd,H("X",0,0,"unix"),H("x",0,0,"valueOf"),N("x",_c),N("X",bd),Q("X",function(a,b,c){c._d=new Date(1e3*parseFloat(a,10))}),Q("x",function(a,b,c){c._d=new Date(q(a))}),a.version="2.10.6",b(Da),a.fn=Od,a.min=Fa,a.max=Ga,a.utc=h,a.unix=Zb,a.months=jc,a.isDate=d,a.locale=w,a.invalid=l,a.duration=Ya,a.isMoment=o,a.weekdays=lc,a.parseZone=$b,a.localeData=y,a.isDuration=Ia,a.monthsShort=kc,a.weekdaysMin=nc,a.defineLocale=x,a.weekdaysShort=mc,a.normalizeUnits=A,a.relativeTimeThreshold=Ec;var oe=a;return oe}); \ No newline at end of file +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):e.moment=t()}(this,function(){"use strict";var H;function _(){return H.apply(null,arguments)}function y(e){return e instanceof Array||"[object Array]"===Object.prototype.toString.call(e)}function F(e){return null!=e&&"[object Object]"===Object.prototype.toString.call(e)}function c(e,t){return Object.prototype.hasOwnProperty.call(e,t)}function L(e){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(e).length;for(var t in e)if(c(e,t))return;return 1}function g(e){return void 0===e}function w(e){return"number"==typeof e||"[object Number]"===Object.prototype.toString.call(e)}function V(e){return e instanceof Date||"[object Date]"===Object.prototype.toString.call(e)}function G(e,t){for(var n=[],s=e.length,i=0;i>>0,s=0;sWe(e)?(r=e+1,t-We(e)):(r=e,t);return{year:r,dayOfYear:n}}function Be(e,t,n){var s,i,r=qe(e.year(),t,n),r=Math.floor((e.dayOfYear()-r-1)/7)+1;return r<1?s=r+N(i=e.year()-1,t,n):r>N(e.year(),t,n)?(s=r-N(e.year(),t,n),i=e.year()+1):(i=e.year(),s=r),{week:s,year:i}}function N(e,t,n){var s=qe(e,t,n),t=qe(e+1,t,n);return(We(e)-s+t)/7}s("w",["ww",2],"wo","week"),s("W",["WW",2],"Wo","isoWeek"),h("w",n,u),h("ww",n,t),h("W",n,u),h("WW",n,t),Oe(["w","ww","W","WW"],function(e,t,n,s){t[s.substr(0,1)]=M(e)});function Je(e,t){return e.slice(t,7).concat(e.slice(0,t))}s("d",0,"do","day"),s("dd",0,0,function(e){return this.localeData().weekdaysMin(this,e)}),s("ddd",0,0,function(e){return this.localeData().weekdaysShort(this,e)}),s("dddd",0,0,function(e){return this.localeData().weekdays(this,e)}),s("e",0,0,"weekday"),s("E",0,0,"isoWeekday"),h("d",n),h("e",n),h("E",n),h("dd",function(e,t){return t.weekdaysMinRegex(e)}),h("ddd",function(e,t){return t.weekdaysShortRegex(e)}),h("dddd",function(e,t){return t.weekdaysRegex(e)}),Oe(["dd","ddd","dddd"],function(e,t,n,s){s=n._locale.weekdaysParse(e,s,n._strict);null!=s?t.d=s:p(n).invalidWeekday=e}),Oe(["d","e","E"],function(e,t,n,s){t[s]=M(e)});var Qe="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Xe="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Ke="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),et=i,tt=i,nt=i;function st(){function e(e,t){return t.length-e.length}for(var t,n,s,i=[],r=[],a=[],o=[],u=0;u<7;u++)s=l([2e3,1]).day(u),t=f(this.weekdaysMin(s,"")),n=f(this.weekdaysShort(s,"")),s=f(this.weekdays(s,"")),i.push(t),r.push(n),a.push(s),o.push(t),o.push(n),o.push(s);i.sort(e),r.sort(e),a.sort(e),o.sort(e),this._weekdaysRegex=new RegExp("^("+o.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+r.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+i.join("|")+")","i")}function it(){return this.hours()%12||12}function rt(e,t){s(e,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),t)})}function at(e,t){return t._meridiemParse}s("H",["HH",2],0,"hour"),s("h",["hh",2],0,it),s("k",["kk",2],0,function(){return this.hours()||24}),s("hmm",0,0,function(){return""+it.apply(this)+r(this.minutes(),2)}),s("hmmss",0,0,function(){return""+it.apply(this)+r(this.minutes(),2)+r(this.seconds(),2)}),s("Hmm",0,0,function(){return""+this.hours()+r(this.minutes(),2)}),s("Hmmss",0,0,function(){return""+this.hours()+r(this.minutes(),2)+r(this.seconds(),2)}),rt("a",!0),rt("A",!1),h("a",at),h("A",at),h("H",n,d),h("h",n,u),h("k",n,u),h("HH",n,t),h("hh",n,t),h("kk",n,t),h("hmm",me),h("hmmss",_e),h("Hmm",me),h("Hmmss",_e),v(["H","HH"],O),v(["k","kk"],function(e,t,n){e=M(e);t[O]=24===e?0:e}),v(["a","A"],function(e,t,n){n._isPm=n._locale.isPM(e),n._meridiem=e}),v(["h","hh"],function(e,t,n){t[O]=M(e),p(n).bigHour=!0}),v("hmm",function(e,t,n){var s=e.length-2;t[O]=M(e.substr(0,s)),t[b]=M(e.substr(s)),p(n).bigHour=!0}),v("hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[O]=M(e.substr(0,s)),t[b]=M(e.substr(s,2)),t[T]=M(e.substr(i)),p(n).bigHour=!0}),v("Hmm",function(e,t,n){var s=e.length-2;t[O]=M(e.substr(0,s)),t[b]=M(e.substr(s))}),v("Hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[O]=M(e.substr(0,s)),t[b]=M(e.substr(s,2)),t[T]=M(e.substr(i))});i=Re("Hours",!0);var ot,ut={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",w:"a week",ww:"%d weeks",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:Fe,monthsShort:Le,week:{dow:0,doy:6},weekdays:Qe,weekdaysMin:Ke,weekdaysShort:Xe,meridiemParse:/[ap]\.?m?\.?/i},W={},lt={};function dt(e){return e&&e.toLowerCase().replace("_","-")}function ht(e){for(var t,n,s,i,r=0;r=t&&function(e,t){for(var n=Math.min(e.length,t.length),s=0;s=t-1)break;t--}r++}return ot}function ct(t){var e,n;if(void 0===W[t]&&"undefined"!=typeof module&&module&&module.exports&&(n=t)&&n.match("^[^/\\\\]*$"))try{e=ot._abbr,require("./locale/"+t),ft(e)}catch(e){W[t]=null}return W[t]}function ft(e,t){return e&&((t=g(t)?P(e):mt(e,t))?ot=t:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+e+" not found. Did you forget to load it?")),ot._abbr}function mt(e,t){if(null===t)return delete W[e],null;var n,s=ut;if(t.abbr=e,null!=W[e])Q("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),s=W[e]._config;else if(null!=t.parentLocale)if(null!=W[t.parentLocale])s=W[t.parentLocale]._config;else{if(null==(n=ct(t.parentLocale)))return lt[t.parentLocale]||(lt[t.parentLocale]=[]),lt[t.parentLocale].push({name:e,config:t}),null;s=n._config}return W[e]=new K(X(s,t)),lt[e]&<[e].forEach(function(e){mt(e.name,e.config)}),ft(e),W[e]}function P(e){var t;if(!(e=e&&e._locale&&e._locale._abbr?e._locale._abbr:e))return ot;if(!y(e)){if(t=ct(e))return t;e=[e]}return ht(e)}function _t(e){var t=e._a;return t&&-2===p(e).overflow&&(t=t[Y]<0||11He(t[D],t[Y])?S:t[O]<0||24N(r,u,l)?p(s)._overflowWeeks=!0:null!=d?p(s)._overflowWeekday=!0:(h=$e(r,a,o,u,l),s._a[D]=h.year,s._dayOfYear=h.dayOfYear)),null!=e._dayOfYear&&(i=bt(e._a[D],n[D]),(e._dayOfYear>We(i)||0===e._dayOfYear)&&(p(e)._overflowDayOfYear=!0),d=ze(i,0,e._dayOfYear),e._a[Y]=d.getUTCMonth(),e._a[S]=d.getUTCDate()),t=0;t<3&&null==e._a[t];++t)e._a[t]=c[t]=n[t];for(;t<7;t++)e._a[t]=c[t]=null==e._a[t]?2===t?1:0:e._a[t];24===e._a[O]&&0===e._a[b]&&0===e._a[T]&&0===e._a[Te]&&(e._nextDay=!0,e._a[O]=0),e._d=(e._useUTC?ze:Ze).apply(null,c),r=e._useUTC?e._d.getUTCDay():e._d.getDay(),null!=e._tzm&&e._d.setUTCMinutes(e._d.getUTCMinutes()-e._tzm),e._nextDay&&(e._a[O]=24),e._w&&void 0!==e._w.d&&e._w.d!==r&&(p(e).weekdayMismatch=!0)}}function xt(e){if(e._f===_.ISO_8601)Yt(e);else if(e._f===_.RFC_2822)Ot(e);else{e._a=[],p(e).empty=!0;for(var t,n,s,i,r,a=""+e._i,o=a.length,u=0,l=ae(e._f,e._locale).match(te)||[],d=l.length,h=0;he.valueOf():e.valueOf()"}),u.toJSON=function(){return this.isValid()?this.toISOString():null},u.toString=function(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},u.unix=function(){return Math.floor(this.valueOf()/1e3)},u.valueOf=function(){return this._d.valueOf()-6e4*(this._offset||0)},u.creationData=function(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}},u.eraName=function(){for(var e,t=this.localeData().eras(),n=0,s=t.length;nthis.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},u.isLocal=function(){return!!this.isValid()&&!this._isUTC},u.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},u.isUtc=At,u.isUTC=At,u.zoneAbbr=function(){return this._isUTC?"UTC":""},u.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},u.dates=e("dates accessor is deprecated. Use date instead.",ge),u.months=e("months accessor is deprecated. Use month instead",Ie),u.years=e("years accessor is deprecated. Use year instead",Pe),u.zone=e("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",function(e,t){return null!=e?(this.utcOffset(e="string"!=typeof e?-e:e,t),this):-this.utcOffset()}),u.isDSTShifted=e("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",function(){var e,t;return g(this._isDSTShifted)&&(q(e={},this),(e=Nt(e))._a?(t=(e._isUTC?l:R)(e._a),this._isDSTShifted=this.isValid()&&0 Date: Mon, 3 Jun 2024 11:55:46 -0500 Subject: [PATCH 147/147] Pass an integer to ping's -w option. ZEN-34908 --- Products/ZenStatus/ping/CmdPingTask.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Products/ZenStatus/ping/CmdPingTask.py b/Products/ZenStatus/ping/CmdPingTask.py index fad7f149e2..0c2e65e0d1 100644 --- a/Products/ZenStatus/ping/CmdPingTask.py +++ b/Products/ZenStatus/ping/CmdPingTask.py @@ -52,7 +52,7 @@ def _detectPing(): log.info("ping6 not found in path") _PING_ARG_TEMPLATE = ( - "%(ping)s -n -s %(datalength)d -c 1 -t %(ttl)d -w %(timeout)f %(ip)s" + "%(ping)s -n -s %(datalength)d -c 1 -t %(ttl)d -w %(timeout)d %(ip)s" ) import platform @@ -61,7 +61,7 @@ def _detectPing(): log.info("Mac OS X detected; adjusting ping args.") _PING_ARG_TEMPLATE = ( "%(ping)s -n -s %(datalength)d -c 1 " - "-m %(ttl)d -t %(timeout)f %(ip)s" + "-m %(ttl)d -t %(timeout)d %(ip)s" ) elif system != "Linux": log.info( @@ -150,7 +150,7 @@ def _pingIp(self): ip=self.config.ip, version=self.config.ipVersion, ttl=64, - timeout=float(self._preferences.pingTimeOut), + timeout=int(self._preferences.pingTimeOut), datalength=self._daemon.options.dataLength if self._daemon.options.dataLength > 16 else 16,