Merge branch 'master' into call_shifter

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='shifthelper',
-    version='1.6.2',
+    version='1.7.0',
     description='a tool for helping people with a FACT night shift',
     url='https://github.com/fact-project/shifthelper',
     author='Dominik Neise, Maximilian Noethe, Sebastian Mueller',
@@ -16,25 +16,23 @@
     package_data={
         'shifthelper.db_cloner': ['logging.conf'],
     },
-    python_requires='>=3.6.0,<3.7',
+    python_requires='>=3.6.0',
     setup_requires=['pytest-runner'],
     tests_require=['pytest>=3.0.0', 'freezegun'],
     install_requires=[
-        'pandas==0.22.0',
+        'pandas~=0.22.0',
         'numpy==1.14.1',
-        'twilio==5.7.0',
         'requests',
         'python-dateutil',
         'sqlalchemy',
         'PyMySQL',
         'pytz',
         'numexpr',
-        'smart_fact_crawler @ https://github.com/fact-project/smart_fact_crawler/archive/v0.6.4.tar.gz',
-        'custos==0.0.7',
+        'smart_fact_crawler @ https://github.com/fact-project/smart_fact_crawler/archive/v0.7.0.tar.gz',
+        'custos[all]==0.1.1',
         'retrying',
         'wrapt',
         'python-json-logger',
-        'telepot',
         'cachetools',
     ],
     entry_points={'console_scripts': [

shifthelper/__main__.py

@@ -136,14 +136,6 @@ def main():
                     ],
                     category=CATEGORY_SHIFTER
                 ),
-                FactIntervalCheck(
-                    name='RelativeCameraTemperatureCheck',
-                    checklist=[
-                        conditions.is_shift_at_the_moment,
-                        conditions.is_rel_camera_temperature_high,
-                    ],
-                    category=CATEGORY_SHIFTER
-                ),
                 FactIntervalCheck(
                     name='BiasNotOperatingDuringDataRun',
                     checklist=[

shifthelper/conditions.py

@@ -7,8 +7,7 @@
 conditions are used by the Check-classes inside checks.py
 '''
 import re as regex
-from datetime import datetime, timedelta
-from pandas import to_datetime
+from datetime import datetime, timedelta, timezone
 import pandas as pd
 import numpy as np
 import logging
@@ -28,6 +27,12 @@
 
 
 log = logging.getLogger(__name__)
+UTC = timezone.utc
+
+
+def is_older(timestamp, delta):
+    ''' Test if a timestamp is older than a certain amount of time'''
+    return (datetime.now(tz=UTC) - timestamp) > delta
 
 
 @log_call_and_result
@@ -176,10 +181,10 @@ def is_feedback_not_calibrated():
 def is_high_windspeed():
     '''Wind speed > 50 km/h'''
     magic_weather = sfc.weather(fallback=True)
-    now = datetime.utcnow()
     max_age = timedelta(minutes=10)
+    timestamp = magic_weather.timestamp
 
-    if magic_weather.timestamp is not None and (now - magic_weather.timestamp) <= max_age:
+    if magic_weather.timestamp is not None and not is_older(timestamp, max_age):
         wind_speed = magic_weather.wind_speed.value
         return np.isnan(wind_speed) or wind_speed >= 50
     else:
@@ -192,15 +197,15 @@ def is_high_windspeed():
 def is_weather_outdatet():
     ''' MAGIC and TNG weather not updated in the last 10 minutes '''
     magic_weather = sfc.weather(fallback=True)
-    now = datetime.utcnow()
     max_age = timedelta(minutes=10)
+    timestamp = magic_weather.timestamp
 
-    if magic_weather.timestamp is not None and (now - magic_weather.timestamp) <= max_age:
+    if magic_weather.timestamp is not None and not is_older(timestamp, max_age):
         return False
     else:
         log.warning('MAGIC Weather outdated, falling back to TNG')
         tng_weather = sfc.tng_weather()
-        return (now - tng_weather.timestamp) >= max_age
+        return is_older(tng_weather.timestamp, max_age)
 
 
 @log_call_and_result
@@ -209,17 +214,17 @@ def is_smartfact_outdatet():
     timestamp = sfc.main_page().timestamp_1
     if timestamp is None:
         return True
-    return timestamp <= (datetime.utcnow() - timedelta(minutes=10))
+    return is_older(timestamp, timedelta(minutes=10))
 
 
 @log_call_and_result
 def is_high_windgusts():
     '''Wind gusts > 50 km/h or MAGIC weather not available and TNG wind > 50 km/h'''
     magic_weather = sfc.weather(fallback=True)
-    now = datetime.utcnow()
     max_age = timedelta(minutes=10)
+    timestamp = magic_weather.timestamp
 
-    if magic_weather.timestamp is not None and (now - magic_weather.timestamp) <= max_age:
+    if magic_weather.timestamp is not None and not is_older(timestamp, max_age):
         wind_gusts = magic_weather.wind_gusts.value
         return np.isnan(wind_gusts) or wind_gusts >= 50
     else:
@@ -312,7 +317,7 @@ def is_nobody_ready_for_shutdown():
     '''Nobody is ready for shutdown'''
     ready_for_shutdown = {}
     for username, since in fetch_users_awake().items():
-        since = to_datetime(since)
+        since = pd.to_datetime(since).tz_localize(UTC)
         if since > get_next_shutdown() - timedelta(minutes=30):
             ready_for_shutdown[username] = since
     return not ready_for_shutdown
@@ -327,23 +332,21 @@ def update_heartbeat():
         log.debug("HeartbeatMonitor offline?")
         return True
     else:
-        heartbeat_monitor_age = (
-            datetime.utcnow() -
-            pd.to_datetime(heartbeats['heartbeatMonitor'])
-        )
-        if heartbeat_monitor_age > timedelta(minutes=10):
+        timestamp = pd.to_datetime(heartbeats['heartbeatMonitor']).tz_convert(UTC)
+        if is_older(timestamp, timedelta(minutes=10)):
             log.debug('heartbeat_monitor_age > timedelta(minutes=10)')
             return True
     return False
 
+
 @log_call_and_result
 def is_dummy_alert_by_shifter():
     '''Dummy Alert'''
     log = logging.getLogger(__name__)
     for username, since in fetch_dummy_alerts().items():
-        since = to_datetime(since)
+        since = pd.to_datetime(since)
 
-        if since > datetime.utcnow() - timedelta(minutes=3):
+        if not is_older(since, timedelta(minutes=3)):
             log.debug('%s issued a dummy alert at: %s', username, since)
             try:
                 current_shifter = get_current_shifter().username
@@ -362,7 +365,7 @@ def is_dummy_alert_by_shifter():
 @log_call_and_result
 def is_20minutes_or_less_before_shutdown():
     '''20min before shutdown'''
-    return datetime.utcnow() > get_next_shutdown() - timedelta(minutes=20)
+    return (get_next_shutdown() - datetime.now(tz=UTC)) < timedelta(minutes=20)
 
 
 @log_call_and_result
@@ -379,7 +382,7 @@ def is_nobody_on_shift():
 @log_call_and_result
 def is_last_shutdown_already_10min_past():
     '''Last Shutdown is already 10min past'''
-    return get_last_shutdown() + timedelta(minutes=10) < datetime.utcnow()
+    return is_older(get_last_shutdown(), timedelta(minutes=10))
 
 
 @log_call_and_result
@@ -399,11 +402,11 @@ def is_trigger_rate_low_for_ten_minutes():
     current_trigger_rate = sfc.trigger_rate().trigger_rate.value
     self.history = self.history.append(
         [{
-            'timestamp': datetime.utcnow(),
+            'timestamp': datetime.now(tz=UTC),
             'rate': current_trigger_rate,
         }]
     )
-    now = datetime.utcnow()
+    now = datetime.now(tz=UTC)
     self.history = self.history[
         (now - self.history.timestamp) < timedelta(minutes=10)
     ]

shifthelper/tools/__init__.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import requests
 from retrying import retry, RetryError
-from datetime import datetime
+from datetime import datetime, timezone
 
 __all__ = ['create_db_connection', 'config']
 
@@ -70,13 +70,13 @@ def get_last_parking_checklist_entry():
                 'select * from park_checklist_filled',
                 conn
             )
-        return table.sort_values('created').iloc[-1].created
+        return table.sort_values('created').iloc[-1].created.tz_localize(timezone.utc)
     except IndexError:
         # In case we can not find out when the checklist was filled
         # we pretend it was only filled waaaay in the future.
         # In all checks, which check if it was *already* filled
         # this future timestamp will result as False
-        return datetime.max
+        return datetime.max.replace(tzinfo=timezone.utc)
 
 
 def fetch_users_awake():

shifthelper/tools/is_shift.py

@@ -1,13 +1,21 @@
 import pandas as pd
 from .. import tools
-from datetime import datetime
+from datetime import datetime, timezone
 from retrying import retry
 from cachetools import TTLCache, cached
 from cachetools.keys import hashkey
 
 from ..debug_log_wrapper import log_call_and_result
 
 
+UTC = timezone.utc
+
+
+def now_seconds(tz=UTC):
+    '''Current time rounded to full seconds, by default with tz UTC'''
+    return datetime.now(tz=tz).replace(microsecond=0)
+
+
 @cached(
     cache=TTLCache(1, ttl=5 * 60),
     key=lambda db: hashkey(None)
@@ -27,8 +35,8 @@ def get_last_startup_or_shutdown(
     db=None
 ):
     if current_time_rounded_to_seconds is None:
-        current_time_rounded_to_seconds = datetime.utcnow(
-            ).replace(microsecond=0)
+        current_time_rounded_to_seconds = now_seconds()
+
     if db is None:
         db = tools.create_db_connection()
 
@@ -44,9 +52,10 @@ def get_last_startup_or_shutdown(
     """.format(
         keys=(
             types.loc["Startup"].fMeasurementTypeKey,
-            types.loc["Shutdown"].fMeasurementTypeKey),
+            types.loc["Shutdown"].fMeasurementTypeKey
+        ),
         now=current_time_rounded_to_seconds
-        )
+    )
 
     with db.connect() as conn:
         return pd.merge(
@@ -64,9 +73,10 @@ def get_last_startup_or_shutdown(
 def is_shift_at_the_moment(time=None, db=None):
     '''There is a shift at the moment'''
     if time is None:
-        now = datetime.utcnow().replace(microsecond=0)
+        now = now_seconds()
     else:
         now = time.replace(microsecond=0)
+
     last_entry = get_last_startup_or_shutdown(
         current_time_rounded_to_seconds=now,
         db=db
@@ -77,8 +87,8 @@ def is_shift_at_the_moment(time=None, db=None):
 
 def get_next_shutdown(current_time_rounded_to_seconds=None, db=None):
     if current_time_rounded_to_seconds is None:
-        current_time_rounded_to_seconds = datetime.utcnow(
-            ).replace(microsecond=0)
+        current_time_rounded_to_seconds = now_seconds()
+
     if db is None:
         db = tools.create_db_connection()
 
@@ -94,26 +104,25 @@ def get_next_shutdown(current_time_rounded_to_seconds=None, db=None):
     """.format(
         key=(types.loc["Shutdown"].fMeasurementTypeKey),
         now=current_time_rounded_to_seconds
-        )
+    )
 
     try:
         with db.connect() as conn:
             return pd.merge(
                 pd.read_sql_query(query, conn),
                 types.reset_index(),
                 on="fMeasurementTypeKey"
-            ).iloc[0].fStart
+            ).iloc[0].fStart.tz_localize(UTC)
     except IndexError:
         # in case we cannot find the next shutdown,
         # we simply say the next shutdown is waaaay far in the future.
-        return datetime.max
+        return datetime.max.replace(tzinfo=timezone.utc)
 
 
 def get_last_shutdown(current_time_rounded_to_seconds=None, db=None):
     try:
         if current_time_rounded_to_seconds is None:
-            current_time_rounded_to_seconds = datetime.utcnow(
-                ).replace(microsecond=0)
+            current_time_rounded_to_seconds = now_seconds()
         if db is None:
             db = tools.create_db_connection()
 
@@ -129,15 +138,15 @@ def get_last_shutdown(current_time_rounded_to_seconds=None, db=None):
         """.format(
             key=(types.loc["Shutdown"].fMeasurementTypeKey),
             now=current_time_rounded_to_seconds
-            )
+        )
 
         with db.connect() as conn:
             return pd.merge(
                 pd.read_sql_query(query, conn),
                 types.reset_index(),
                 on="fMeasurementTypeKey"
-            ).iloc[0].fStart
+            ).iloc[0].fStart.tz_localize(UTC)
     except IndexError:
         # in case we cannot find the last shutdown,
         # we simply say the last shutdown was waaay in the past
-        return datetime.min
+        return datetime.min.replace(tzinfo=timezone.utc)