OSM-POI: Include brand property [DRAFT]

.gitignore

@@ -16,5 +16,5 @@ env
 !.env.local
 __pycache__
 
-
+debug
 kuwala/scripts/windows/

kuwala/common/python_utils/src/FileSelector.py

@@ -3,7 +3,6 @@
 from time import sleep
 import urllib.error
 
-from fuzzywuzzy import fuzz
 from hdx.data.dataset import Dataset
 from hdx.data.organization import Organization
 from hdx.hdx_configuration import Configuration
@@ -12,6 +11,7 @@
 from pyquery import PyQuery
 import questionary
 import requests.exceptions
+from thefuzz import fuzz
 
 CONTINENTS = [
     {"code": "af", "name": "Africa", "geofabrik": "africa"},
@@ -27,6 +27,7 @@
 
 def select_local_country(directory):
     continents = os.listdir(directory)
+    continents.remove("brand_names.csv")
     continent_names = list(
         map(
             lambda c: pcc.convert_continent_code_to_continent_name(c.upper()),

kuwala/common/python_utils/src/spark_udfs.py

@@ -1,6 +1,5 @@
 import json
 
-from fuzzywuzzy import fuzz
 import h3
 from pyspark.sql.functions import udf
 from pyspark.sql.types import (
@@ -13,6 +12,7 @@
     StructType,
 )
 from shapely.geometry import shape
+from thefuzz import fuzz
 
 DEFAULT_RESOLUTION = 11
 

kuwala/core/cli/requirements.txt

@@ -21,7 +21,7 @@ docopt==0.6.2
 email-validator==1.1.3
 et-xmlfile==1.1.0
 exchangerates==0.3.4
-fuzzywuzzy==0.18.0
+thefuzz==0.19.0
 greenlet==1.1.1
 hdx-python-api==5.2.4
 hdx-python-country==2.9.5

kuwala/core/database/importer/sql/create_tables.sql

@@ -148,7 +148,7 @@ CREATE TABLE IF NOT EXISTS google_osm_poi_matching (
     CONSTRAINT fk_google_osm_poi_matching_osm_id FOREIGN KEY(osm_type, osm_id) REFERENCES osm_poi(osm_type, osm_id)
 );
 
--- Creation of google_osm_poi_matching table
+-- Creation of google_custom_poi_matching table
 
 CREATE TABLE IF NOT EXISTS google_custom_poi_matching (
     custom_id text NOT NULL PRIMARY KEY,

kuwala/core/database/importer/src/population_density_importer.py

@@ -11,7 +11,7 @@ def import_population_density(
     database_properties,
     continent,
     country,
-    population_density_date,
+    population_density_date="",
 ):
     start_time = time.time()
 

kuwala/pipelines/osm-poi/requirements.txt

@@ -21,7 +21,7 @@ docopt==0.6.2
 email-validator==1.1.3
 et-xmlfile==1.1.0
 exchangerates==0.3.4
-fuzzywuzzy==0.18.0
+thefuzz==0.19.0
 greenlet==1.1.1
 h3==3.7.3
 hdx-python-api==5.2.4

kuwala/pipelines/osm-poi/src/Downloader.py

@@ -1,12 +1,18 @@
+import json
+import logging as log
 import os
+import shutil
+import urllib.request as req
+import zipfile
 
+import pandas as pd
 from python_utils.src.FileDownloader import download_file
 from python_utils.src.FileSelector import select_osm_file
 
 
 class Downloader:
     @staticmethod
-    def start(args):
+    def download_pbf(args):
         file = None
 
         if args.url is None:
@@ -27,3 +33,55 @@ def start(args):
         file_path += "/pbf/geo_fabrik.osm.pbf"
 
         download_file(url=args.url or file["url"], path=file_path)
+
+    @staticmethod
+    def download_names():
+        temp_files_dir = "../../../tmp/kuwala/osm_files/"
+        # here, instead of cloning the repository that recommended using extra library,
+        # we download the whole repo in zip, then extract it.
+        if not os.path.exists(temp_files_dir + "name-suggestion-index-main"):
+            log.info("Downloading brand and operator name reference...")
+            download_link = "https://github.com/osmlab/name-suggestion-index/archive/refs/heads/main.zip"
+            req.urlretrieve(download_link, temp_files_dir + "main.zip")
+            with zipfile.ZipFile(temp_files_dir + "main.zip", "r") as zip_ref:
+                zip_ref.extractall(temp_files_dir)
+            os.remove(temp_files_dir + "main.zip")
+
+        file_paths = [
+            temp_files_dir + "name-suggestion-index-main/data/brands",
+            temp_files_dir + "name-suggestion-index-main/data/operators",
+        ]
+        data = {"id": [], "display_name": [], "wiki_data": []}
+        log.info("Composing brand and operator name list...")
+        for file_path in file_paths:
+            for folder in os.listdir(file_path):
+                if os.path.isdir(os.path.join(file_path, folder)):
+                    for file in os.listdir(os.path.join(file_path, folder)):
+                        with open(os.path.join(file_path, folder, file)) as f:
+                            file_content = json.load(f)
+                        for item in file_content["items"]:
+                            wiki_data = id = display_name = None
+                            if "id" in item.keys():
+                                id = dict(item)["id"]
+                            if "displayName" in item.keys():
+                                display_name = dict(item)["displayName"]
+                            if "tags" in item.keys():
+                                if "brand:wikidata" in list(item["tags"].keys()):
+                                    wiki_data = dict(item["tags"].items())[
+                                        "brand:wikidata"
+                                    ]
+                                elif "operator:wikidata" in list(item["tags"].keys()):
+                                    wiki_data = dict(item["tags"].items())[
+                                        "operator:wikidata"
+                                    ]
+                            if str(id) != "nan":
+                                data["id"].append(id)
+                                data["display_name"].append(display_name)
+                                data["wiki_data"].append(wiki_data)
+
+        shutil.rmtree(temp_files_dir + "name-suggestion-index-main")
+        df = pd.DataFrame(data)
+        df.drop_duplicates(subset=["display_name", "wiki_data"])
+        df = df.sort_values("id")
+        df.to_csv(temp_files_dir + "brand_names.csv", index=False)
+        log.info("Done!")

kuwala/pipelines/osm-poi/src/main.py

@@ -34,6 +34,7 @@
         action = "download" if option == choices[0] else "process"
 
     if action == "download":
-        Downloader.start(args)
+        Downloader.download_pbf(args)
+        Downloader.download_names()
     else:
         Processor.start(args)