Merge pull request #2747 from data-for-change/dev

merge dev into master

anyway/flask_app.py

@@ -1285,10 +1285,10 @@ def infographics_data_by_location():
         output = get_infographics_mock_data()
     elif mock_data == "false":
         request_params = get_request_params_from_request_values(request.values)
-        output = get_infographics_data_for_location(request_params)
-        if not output:
+        if request_params is None:
             log_bad_request(request)
             return abort(http_client.NOT_FOUND)
+        output = get_infographics_data_for_location(request_params)
     else:
         log_bad_request(request)
         return abort(http_client.BAD_REQUEST)

anyway/request_params.py

@@ -78,67 +78,70 @@ def __eq__(self, other):
 
 # todo: merge with get_request_params()
 def get_request_params_from_request_values(vals: dict) -> Optional[RequestParams]:
-    news_flash_obj = extract_news_flash_obj(vals)
-    news_flash_description = (
-        news_flash_obj.description
-        if news_flash_obj is not None and news_flash_obj.description is not None
-        else None
-    )
-    news_flash_title = (
-        news_flash_obj.title
-        if news_flash_obj is not None and news_flash_obj.title is not None
-        else None
-    )
-    location = get_location_from_news_flash_or_request_values(news_flash_obj, vals)
-    if location is None:
-        return None
-    years_ago = vals.get("years_ago", BE_CONST.DEFAULT_NUMBER_OF_YEARS_AGO)
-    lang = vals.get("lang", "he")
-    location_text = location["text"]
-    gps = location["gps"]
-    location_info = location["data"]
-
-    if location_info is None:
-        return None
-    resolution = location_info.pop("resolution")
-    if resolution is None or resolution not in BE_CONST.SUPPORTED_RESOLUTIONS:
-        logging.error(f"Resolution empty or not supported: {resolution}.")
-        return None
-
-    if all(value is None for value in location_info.values()):
-        return None
-
     try:
-        years_ago = int(years_ago)
-    except (ValueError, TypeError):
-        return None
-    if years_ago < 0 or years_ago > 100:
+        news_flash_obj = extract_news_flash_obj(vals)
+        news_flash_description = (
+            news_flash_obj.description
+            if news_flash_obj is not None and news_flash_obj.description is not None
+            else None
+        )
+        news_flash_title = (
+            news_flash_obj.title
+            if news_flash_obj is not None and news_flash_obj.title is not None
+            else None
+        )
+        location = get_location_from_news_flash_or_request_values(news_flash_obj, vals)
+        if location is None:
+            return None
+        years_ago = vals.get("years_ago", BE_CONST.DEFAULT_NUMBER_OF_YEARS_AGO)
+        lang = vals.get("lang", "he")
+        location_text = location["text"]
+        gps = location.get("gps", {})
+        location_info = location["data"]
+
+        if location_info is None:
+            return None
+        resolution = location_info.pop("resolution")
+        if resolution is None or resolution not in BE_CONST.SUPPORTED_RESOLUTIONS:
+            logging.error(f"Resolution empty or not supported: {resolution}.")
+            return None
+
+        if all(value is None for value in location_info.values()):
+            return None
+
+        try:
+            years_ago = int(years_ago)
+        except (ValueError, TypeError):
+            return None
+        if years_ago < 0 or years_ago > 100:
+            return None
+        last_accident_date = get_latest_accident_date(table_obj=AccidentMarkerView, filters=None)
+        # converting to datetime object to get the date
+        end_time = last_accident_date.to_pydatetime().date()
+        start_time = datetime.date(end_time.year + 1 - years_ago, 1, 1)
+
+        widget_specific = {}
+        if resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD:
+            widget_specific.update({"road_segment_name": location_info.get("road_segment_name")})
+
+        request_params = RequestParams(
+            years_ago=years_ago,
+            location_text=location_text,
+            location_info=location_info,
+            # TODO: getting a warning on resolution=resolution: "Expected type 'dict', got 'int' instead"
+            resolution=resolution,
+            gps=gps,
+            start_time=start_time,
+            end_time=end_time,
+            lang=lang,
+            news_flash_description=news_flash_description,
+            news_flash_title=news_flash_title,
+            widget_specific=widget_specific
+        )
+        return request_params
+    except ValueError:
+        logging.exception(f"Exception while preparing request params. vals:{vals}.")
         return None
-    last_accident_date = get_latest_accident_date(table_obj=AccidentMarkerView, filters=None)
-    # converting to datetime object to get the date
-    end_time = last_accident_date.to_pydatetime().date()
-    start_time = datetime.date(end_time.year + 1 - years_ago, 1, 1)
-
-    widget_specific = {}
-    if resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD:
-        widget_specific.update({"road_segment_name": location_info.get("road_segment_name")})
-
-    request_params = RequestParams(
-        years_ago=years_ago,
-        location_text=location_text,
-        location_info=location_info,
-        # TODO: getting a warning on resolution=resolution: "Expected type 'dict', got 'int' instead"
-        resolution=resolution,
-        gps=gps,
-        start_time=start_time,
-        end_time=end_time,
-        lang=lang,
-        news_flash_description=news_flash_description,
-        news_flash_title=news_flash_title,
-        widget_specific=widget_specific
-    )
-    return request_params
-
 
 def get_location_from_news_flash_or_request_values(
     news_flash_obj: Optional[NewsFlash], vals: dict
@@ -220,9 +223,7 @@ def extract_road_segment_location(road_segment_id):
     data["road_segment_name"] = road_segment_name
     data["road_segment_id"] = int(road_segment_id)
     text = get_road_segment_location_text(road1, road_segment_name)
-    # fake gps - todo: fix
-    gps = {"lat": 32.825610, "lon": 35.165395}
-    return {"name": "location", "data": data, "gps": gps, "text": text}
+    return {"name": "location", "data": data, "text": text}
 
 
 # todo: fill both codes and names into location
@@ -233,9 +234,7 @@ def extract_street_location(input_vals: dict):
     for k in ["yishuv_name", "yishuv_symbol", "street1", "street1_hebrew"]:
         data[k] = vals[k]
     text = get_street_location_text(vals["yishuv_name"], vals["street1_hebrew"])
-    # fake gps - todo: fix
-    gps = {"lat": 32.825610, "lon": 35.165395}
-    return {"name": "location", "data": data, "gps": gps, "text": text}
+    return {"name": "location", "data": data, "text": text}
 
 
 def extract_street_location_suggestion_version(input_vals: dict):
@@ -271,12 +270,9 @@ def extract_non_urban_intersection_location(input_vals: dict):
     data = {"resolution": BE_CONST.ResolutionCategories.SUBURBAN_JUNCTION}
     for k in ["non_urban_intersection", "non_urban_intersection_hebrew", "road1", "road2"]:
         data[k] = vals[k]
-    # fake gps - todo: fix
-    gps = {"lat": 32.825610, "lon": 35.165395}
     return {
         "name": "location",
         "data": data,
-        "gps": gps,
         "text": vals["non_urban_intersection_hebrew"],
     }
 

anyway/widgets/road_segment_widgets/head_on_collisions_comparison_widget.py

@@ -117,7 +117,7 @@ def is_included(self) -> bool:
             else:
                 raise ValueError
         all_total = all_h2h + all_others  # pylint: disable=E0606
-        return segment_h2h > 1 and (segment_h2h / segment_total) > all_h2h / all_total
+        return segment_h2h > 1 and (segment_h2h / segment_total) > 2 * all_h2h / all_total
 
 
 # adding calls to _() for pybabel extraction

anyway/widgets/road_segment_widgets/street_view_widget.py

@@ -16,10 +16,14 @@ def __init__(self, request_params: RequestParams):
 
     def generate_items(self) -> None:
         self.items = {
-            "longitude": self.request_params.gps["lon"],
-            "latitude": self.request_params.gps["lat"],
+            "longitude": self.request_params.gps.get("lon"),
+            "latitude": self.request_params.gps.get("lat"),
         }
 
+    def is_included(self):
+        return self.request_params.gps and self.request_params.gps.get("lon") and self.request_params.gps.get("lat")
+
+
     @staticmethod
     def localize_items(request_params: RequestParams, items: Dict) -> Dict:
         items["data"]["text"] = {"title": _("Street view widget")}

docs/Architecture/CBS.md

@@ -6,7 +6,7 @@
 - **Relevant output**: unzipped CBS files saved in S3
 - **Relevant Storage directory**: S3 -> Bucket: dfc-anyway-cbs
 - **command**: `python3 main.py scripts importemail`
-- **Scheduling**: Nowadays runs in Jenkins once a week
+- **Scheduling**: Nowadays runs in Airflow once a week
 
 ### CBS: pulls data from s3, processes cbs data and pushes it to CBS tables
 Environment Variables: AWS_ACCESS_KEY, AWS_SECRET_KEY

tests/test_request_params.py

@@ -30,7 +30,6 @@ class TestRequestParams(unittest.TestCase):
                       'resolution': BE_CONST.ResolutionCategories.SUBURBAN_JUNCTION,
                       'road1': 669,
                       'road2': 71},
-             'gps': {'lat': 32.82561, 'lon': 35.165395},
              'name': 'location',
              'text': 'צומת השיטה'}
     nf = NewsFlash()
@@ -46,7 +45,7 @@ class TestRequestParams(unittest.TestCase):
         lang='he',
         news_flash_description=nf.description,
         news_flash_title=nf.title,
-        gps={"lat": 32.825610, "lon": 35.165395}
+        gps={}
     )
 
     @patch("anyway.request_params.fill_missing_non_urban_intersection_values")