2604 widget location accuracy

anyway/flask_app.py

@@ -1167,6 +1167,7 @@ def datetime_to_str(val: datetime.datetime) -> str:
         "street2_hebrew": fields.String(),
         "non_urban_intersection_hebrew": fields.String(),
         "road_segment_name": fields.String(),
+        "road_segment_id": fields.Integer(),
         "newsflash_location_qualification": fields.Integer(),
         "location_qualifying_user": fields.Integer(),
     },

anyway/infographics_utils.py

@@ -21,7 +21,7 @@
 )
 from anyway.backend_constants import BE_CONST, AccidentType
 from anyway.models import NewsFlash, AccidentMarkerView
-from anyway.parsers import resolution_dict
+from anyway.parsers.resolution_fields import ResolutionFields as RF
 from anyway.infographics_dictionaries import head_on_collisions_comparison_dict
 from anyway.parsers import infographics_data_cache_updater
 from anyway.widgets.widget import Widget, widgets_dict
@@ -244,7 +244,7 @@ def is_news_flash_resolution_supported(news_flash_obj: NewsFlash) -> bool:
         return False
     location = location_data[DATA]
     for cat in BE_CONST.SUPPORTED_RESOLUTIONS:
-        if cat.value in resolution_dict and set(resolution_dict[cat.value]) <= location.keys():
+        if set(RF.get_required_fields(cat.value)) <= location.keys():
             return True
     return False
 

anyway/models.py

@@ -885,48 +885,6 @@ class NewsFlash(Base):
     )
     location_qualifying_user = Column(BigInteger(), nullable=True)
 
-    def set_critical(
-        self,
-        years_before=5,
-        suburban_road_severe_value=10,
-        suburban_road_killed_value=3,
-        urban_severe_value=2,
-    ):
-        from anyway.widgets.all_locations_widgets.injured_count_by_severity_widget import (
-            InjuredCountBySeverityWidget,
-        )
-        from anyway.request_params import get_latest_accident_date, LocationInfo
-
-        if (self.road1 is None or self.road_segment_name is None) and (self.yishuv_name is None or self.street1_hebrew is None) :
-            return None
-        last_accident_date = get_latest_accident_date(table_obj=AccidentMarkerView, filters=None)
-        resolution = BE_CONST.ResolutionCategories(self.resolution)
-        end_time = last_accident_date.to_pydatetime().date()
-        start_time = datetime.date(end_time.year + 1 - years_before, 1, 1)
-        location_info = LocationInfo()
-        if resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD:
-            location_info["road1"]
-            location_info["road_segment_name"]
-        elif resolution == BE_CONST.ResolutionCategories.STREET:
-            location_info["yishuv_name"] = self.yishuv_name
-            location_info["street1_hebrew"] = self.street1_hebrew
-
-        critical_values = InjuredCountBySeverityWidget.get_injured_count_by_severity(resolution, location_info, start_time, end_time)
-        if critical_values == {}:
-            return None
-        critical = None
-        if resolution == BE_CONST.ResolutionCategories.SUBURBAN_ROAD:
-            critical = (
-                (critical_values["severe_injured_count"] / suburban_road_severe_value)
-                + (critical_values["killed_count"] / suburban_road_killed_value)
-            ) >= 1
-        elif resolution == BE_CONST.ResolutionCategories.STREET:
-            critical = (
-                (critical_values["severe_injured_count"] / urban_severe_value)
-                + (critical_values["killed_count"] / urban_severe_value)
-            ) >= 1
-        self.critical = critical
-
     def serialize(self):
         return {
             "id": self.id,
@@ -951,7 +909,7 @@ def serialize(self):
             "street1_hebrew": self.street1_hebrew,
             "street2_hebrew": self.street2_hebrew,
             "non_urban_intersection_hebrew": self.non_urban_intersection_hebrew,
-            "road_segment_name": self.road_segment_name,
+            "road_segment_id": self.road_segment_id,
             "newsflash_location_qualification": NewsflashLocationQualification(
                 self.newsflash_location_qualification
             ).get_label(),

anyway/parsers/__init__.py

@@ -1,26 +1,4 @@
-resolution_dict = {
-    "מחוז": ["region_hebrew"],
-    "נפה": ["district_hebrew"],
-    "עיר": ["yishuv_name"],
-    "רחוב": ["yishuv_name", "street1_hebrew"],
-    "צומת עירוני": ["yishuv_name", "street1_hebrew", "street2_hebrew"],
-    "כביש בינעירוני": ["road1", "road_segment_name"],
-    "צומת בינעירוני": ["non_urban_intersection", "non_urban_intersection_hebrew", "road1", "road2"],
-    "אחר": [
-        "region_hebrew",
-        "district_hebrew",
-        "yishuv_name",
-        "street1_hebrew",
-        "street2_hebrew",
-        "non_urban_intersection_hebrew",
-        "road1",
-        "road2",
-        "road_segment_name",
-        "road_segment_id",
-    ],
-}
-
-fields_to_resolution = {
-    ("yishuv_name", "street1_hebrew"): "רחוב",
-    ("road_segment_name", "road1"): "כביש בינעירוני",
-}
+"""
+resolution_dict is depracated. Its functionality was moved to class
+ResolutionFields, in module resolution_fields in same directory.
+"""

anyway/parsers/cbs/executor.py

@@ -1210,4 +1210,4 @@ def main(batch_size, source, load_start_year=None):
     except Exception as ex:
         print("Traceback: {0}".format(traceback.format_exc()))
         raise CBSParsingFailed(message=str(ex))
-        # Todo - send an email that an exception occured
+        # Todo - send an email that an exception occurred

anyway/parsers/infographics_data_cache_updater.py

@@ -1,12 +1,8 @@
 # -*- coding: utf-8 -*-
 
 from datetime import datetime
-from sqlalchemy import not_
 from anyway.models import (
     Base,
-    InfographicsDataCache,
-    InfographicsDataCacheTemp,
-    NewsFlash,
     RoadSegments,
     InfographicsRoadSegmentsDataCache,
     InfographicsRoadSegmentsDataCacheTemp,
@@ -34,80 +30,13 @@
 META = "meta"
 DATA = "data"
 ITEMS = "items"
-REGULAR_CACHE_TABLES = {CACHE: InfographicsDataCache, TEMP: InfographicsDataCacheTemp}
 STREET_CACHE_TABLES = {CACHE: InfographicsStreetDataCache, TEMP: InfographicsStreetDataCacheTemp}
 ROAD_SEGMENT_CACHE_TABLES = {
     CACHE: InfographicsRoadSegmentsDataCache,
     TEMP: InfographicsRoadSegmentsDataCacheTemp,
 }
 
 
-def is_in_cache(nf):
-    return (
-        len(CONST.INFOGRAPHICS_CACHE_YEARS_AGO)
-        == db.session.query(InfographicsDataCache)
-        .filter(InfographicsDataCache.news_flash_id == nf.get_id())
-        .count()
-    )
-
-
-# noinspection PyUnresolvedReferences
-def add_news_flash_to_cache(news_flash: NewsFlash):
-    try:
-        if not (
-            news_flash.accident
-            and anyway.infographics_utils.is_news_flash_resolution_supported(news_flash)
-            and news_flash.newsflash_location_qualification is not None
-        ):
-            logging.debug(
-                f"add_news_flash_to_cache: news flash does not qualify:{news_flash.serialize()}"
-            )
-            return True
-        db.get_engine().execute(
-            InfographicsDataCache.__table__.insert(),  # pylint: disable=no-member
-            [
-                {
-                    "news_flash_id": news_flash.get_id(),
-                    "years_ago": y,
-                    "data": anyway.infographics_utils.create_infographics_data(
-                        news_flash.get_id(), y, "he"
-                    ),
-                }
-                for y in CONST.INFOGRAPHICS_CACHE_YEARS_AGO
-            ],
-        )
-        logging.info(f"{news_flash.get_id()} added to cache")
-        return True
-    except Exception as e:
-        logging.exception(
-            f"Exception while inserting to cache. flash_id:{news_flash}), cause:{e.__cause__}"
-        )
-        return False
-
-
-def get_infographics_data_from_cache(news_flash_id, years_ago) -> Dict:
-    db_item = (
-        db.session.query(InfographicsDataCache)
-        .filter(InfographicsDataCache.news_flash_id == news_flash_id)
-        .filter(InfographicsDataCache.years_ago == years_ago)
-        .first()
-    )
-    logging.debug(f"retrieved from cache {type(db_item)}:{db_item}"[:70])
-    db.session.commit()
-    try:
-        if db_item:
-            return json.loads(db_item.get_data())
-        else:
-            return {}
-    except Exception as e:
-        logging.error(
-            f"Exception while extracting data from returned cache item flash_id:{news_flash_id},years:{years_ago})"
-            f"returned value {type(db_item)}"
-            f":cause:{e.__cause__}, class:{e.__class__}"
-        )
-        return {}
-
-
 def get_infographics_data_from_cache_by_road_segment(road_segment_id, years_ago) -> Dict:
     db_item = (
         db.session.query(InfographicsRoadSegmentsDataCache)
@@ -236,33 +165,6 @@ def copy_temp_into_cache(table: Dict[str, Base]):
     db.session.commit()
 
 
-# noinspection PyUnresolvedReferences
-def build_cache_into_temp():
-    start = datetime.now()
-    db.session.query(InfographicsDataCacheTemp).delete()
-    db.session.commit()
-    supported_resolutions = set([x.value for x in BE_CONST.SUPPORTED_RESOLUTIONS])
-    for y in CONST.INFOGRAPHICS_CACHE_YEARS_AGO:
-        logging.debug(f"processing years_ago:{y}")
-        db.get_engine().execute(
-            InfographicsDataCacheTemp.__table__.insert(),  # pylint: disable=no-member
-            [
-                {
-                    "news_flash_id": new_flash.get_id(),
-                    "years_ago": y,
-                    "data": anyway.infographics_utils.create_infographics_data(
-                        new_flash.get_id(), y, "he"
-                    ),
-                }
-                for new_flash in db.session.query(NewsFlash)
-                .filter(NewsFlash.accident)
-                .filter(NewsFlash.resolution.in_(supported_resolutions))
-                .all()
-            ],
-        )
-    logging.info(f"cache rebuild took:{str(datetime.now() - start)}")
-
-
 def get_streets() -> Iterable[Streets]:
     t = Streets
     street_iter = iter(db.session.query(t.yishuv_symbol, t.street).all())
@@ -340,21 +242,6 @@ def build_road_segments_cache_into_temp():
     logging.info(f"cache rebuild took:{str(datetime.now() - start)}")
 
 
-def get_cache_info():
-    cache_items = db.session.query(InfographicsDataCache).count()
-    tmp_items = db.session.query(InfographicsDataCacheTemp).count()
-    num_acc_flash_items = db.session.query(NewsFlash).filter(NewsFlash.accident).count()
-    num_acc_suburban_flash_items = (
-        db.session.query(NewsFlash)
-        .filter(NewsFlash.accident)
-        .filter(NewsFlash.resolution.in_(["כביש בינעירוני"]))
-        .filter(not_(NewsFlash.road_segment_name == None))
-        .count()
-    )
-    db.session.commit()
-    return f"num items in cache: {cache_items}, temp table: {tmp_items}, accident flashes:{num_acc_flash_items}, flashes processed:{num_acc_suburban_flash_items}"
-
-
 def main_for_road_segments():
     logging.info("Refreshing road segments infographics cache...")
     build_road_segments_cache_into_temp()
@@ -365,15 +252,3 @@ def main_for_road_segments():
 def main_for_street():
     build_street_cache_into_temp()
     copy_temp_into_cache(STREET_CACHE_TABLES)
-
-
-def main(update, info):
-    if update:
-        logging.info("Refreshing infographics cache...")
-        build_cache_into_temp()
-        copy_temp_into_cache(REGULAR_CACHE_TABLES)
-        logging.info("Refreshing infographics cache Done")
-    if info:
-        logging.info(get_cache_info())
-    else:
-        logging.debug(f"{info}")

anyway/parsers/location_extraction.py

@@ -7,7 +7,7 @@
 from geographiclib.geodesic import Geodesic
 from anyway.backend_constants import BE_CONST
 from anyway.models import NewsFlash
-from anyway.parsers import resolution_dict
+from anyway.parsers.resolution_fields import ResolutionFields as RF
 from anyway import secrets
 from anyway.models import AccidentMarkerView, RoadSegments
 from sqlalchemy import not_
@@ -191,7 +191,7 @@ def get_db_matching_location(db, latitude, longitude, resolution, road_no=None):
     """
     # READ MARKERS FROM DB
     geod = Geodesic.WGS84
-    relevant_fields = resolution_dict[resolution]
+    relevant_fields = RF.get_possible_fields(resolution)
     markers = db.get_markers_for_location_extraction()
     markers["geohash"] = markers.apply(
         lambda x: geohash.encode(x["latitude"], x["longitude"], precision=4), axis=1
@@ -538,12 +538,9 @@ def extract_geo_features(db, newsflash: NewsFlash, update_cbs_location_only: boo
     if location_from_db is not None:
         for k, v in location_from_db.items():
             setattr(newsflash, k, v)
-        all_resolutions = []
-        for _, v in resolution_dict.items():
-            all_resolutions += v
-        for resolution in all_resolutions:
-            if resolution not in location_from_db:
-                setattr(newsflash, resolution, None)
+        for field in RF.get_all_location_fields():
+            if field not in location_from_db:
+                setattr(newsflash, field, None)
     if (
         newsflash.road_segment_id is None
         and newsflash.road_segment_name is not None

anyway/parsers/news_flash_db_adapter.py

@@ -5,7 +5,6 @@
 import numpy as np
 from sqlalchemy import desc
 from flask_sqlalchemy import SQLAlchemy
-from anyway.parsers import infographics_data_cache_updater
 from anyway.parsers import timezones
 from anyway.models import NewsFlash
 from anyway.slack_accident_notifications import publish_notification
@@ -69,17 +68,15 @@ def insert_new_newsflash(self, newsflash: NewsFlash) -> None:
         self.__fill_na(newsflash)
         self.db.session.add(newsflash)
         self.db.session.commit()
-        infographics_data_cache_updater.add_news_flash_to_cache(newsflash)
         if os.environ.get("FLASK_ENV") == "production" and newsflash.accident:
             try:
                 DBAdapter.publish_notifications(newsflash)
             except Exception as e:
                 logging.error("publish notifications failed")
                 logging.error(e)
 
-
-    def get_newsflash_by_id(self, id):
-        return self.db.session.query(NewsFlash).filter(NewsFlash.id == id)
+    def get_newsflash_by_id(self, nid):
+        return self.db.session.query(NewsFlash).filter(NewsFlash.id == nid)
 
     def select_newsflash_where_source(self, source):
         return self.db.session.query(NewsFlash).filter(NewsFlash.source == source)