Combined collect_bbbike_subregion_download_catalogue() and collect_bb…

…bike_download_catalogue()

pydriosm/download_BBBike.py

@@ -40,6 +40,7 @@ def collect_bbbike_subregion_catalogue(confirmation_required=True, verbose=False
 
         except Exception as e:
             print("Failed to get the required information ... {}.".format(e))
+
     else:
         print("The information collection process was not activated. The existing local copy will be loaded instead.")
 
@@ -66,7 +67,8 @@ def fetch_bbbike_subregion_catalogue(catalogue_name, update=False, verbose=False
 
     path_to_pickle = cd_dat(catalogue_name + ".pickle")
     if not os.path.isfile(path_to_pickle) or update:
-        collect_bbbike_subregion_catalogue(confirmation_required=False)
+        collect_bbbike_subregion_catalogue(confirmation_required=False, verbose=verbose)
+
     try:
         bbbike_subregion_catalogue = load_pickle(path_to_pickle, verbose=verbose)
         return bbbike_subregion_catalogue
@@ -91,39 +93,48 @@ def regulate_bbbike_input_subregion_name(subregion_name):
 
 
 #
-def collect_bbbike_subregion_download_catalogue(subregion_name, confirmation_required=True, verbose=False):
+def collect_bbbike_download_catalogue(confirmation_required=True, verbose=False):
     """
-    :param subregion_name: [str]
     :param confirmation_required: [bool] (default: True)
     :param verbose: [bool] (default: False)
 
     Example:
-        subregion_name        = 'leeds'
         confirmation_required = True
         verbose               = True
-        collect_bbbike_subregion_download_catalogue(subregion_name, confirmation_required, verbose)
+        collect_bbbike_download_catalogue(confirmation_required, verbose)
     """
 
-    def parse_dlc(dlc):
-        dlc_href = dlc.get('href')  # URL
-        filename, download_url = dlc_href.strip('./'), urllib.parse.urljoin(url, dlc_href)
-        if not dlc.has_attr('title'):
-            file_format, file_size, last_update = 'Poly', None, None
-        else:
-            if len(dlc.contents) < 3:
-                file_format, file_size = 'Txt', None
-            else:
-                file_format, file_size, _ = dlc.contents  # File type and size
-                file_format, file_size = file_format.strip(), file_size.text
-            last_update = pd.to_datetime(dlc.get('title'))  # Date and time
-        parsed_dat = [filename, download_url, file_format, file_size, last_update]
-        return parsed_dat
-
-    subregion_name_ = regulate_bbbike_input_subregion_name(subregion_name)
     #
-    if confirmed("To collect BBBike download catalogue for \"{}\"? ".format(subregion_name_),
-                 confirmation_required=confirmation_required):
+    def collect_bbbike_subregion_download_catalogue(subregion_name):
+        """
+        :param subregion_name: [str]
+
+        Example:
+            subregion_name        = 'leeds'
+            confirmation_required = True
+            verbose               = True
+            collect_bbbike_subregion_download_catalogue(subregion_name, confirmation_required, verbose)
+        """
+
+        def parse_dlc(dlc):
+            dlc_href = dlc.get('href')  # URL
+            filename, download_url = dlc_href.strip('./'), urllib.parse.urljoin(url, dlc_href)
+            if not dlc.has_attr('title'):
+                file_format, file_size, last_update = 'Poly', None, None
+            else:
+                if len(dlc.contents) < 3:
+                    file_format, file_size = 'Txt', None
+                else:
+                    file_format, file_size, _ = dlc.contents  # File type and size
+                    file_format, file_size = file_format.strip(), file_size.text
+                last_update = pd.to_datetime(dlc.get('title'))  # Date and time
+            parsed_dat = [filename, download_url, file_format, file_size, last_update]
+            return parsed_dat
+
+        subregion_name_ = regulate_bbbike_input_subregion_name(subregion_name)
+        #
         try:
+            print("  \"{}\" ... ".format(subregion_name_), end="") if verbose else ""
             url = 'https://download.bbbike.org/osm/bbbike/{}/'.format(subregion_name_)
 
             source = urllib.request.urlopen(url)
@@ -135,80 +146,40 @@ def parse_dlc(dlc):
             subregion_downloads_catalogue = pd.DataFrame(parse_dlc(x) for x in download_links_class)
             subregion_downloads_catalogue.columns = ['Filename', 'URL', 'DataType', 'Size', 'LastUpdate']
 
-            path_to_file = cd_dat_bbbike(subregion_name_, subregion_name_ + "-download-catalogue.pickle")
-            save_pickle(subregion_downloads_catalogue, path_to_file, verbose=verbose)
-
-        except Exception as e:
-            print("Failed to collect download catalogue for \"{}\". {}".format(subregion_name_, e))
-    else:
-        print("The information collection process was not activated. The existing local copy will be loaded instead.")
-
-
-#
-def fetch_bbbike_subregion_download_catalogue(subregion_name, update=False, confirmation_required=True, verbose=False):
-    """
-    :param subregion_name: [str]
-    :param update: [bool] (default: False)
-    :param confirmation_required: [bool] (default: True)
-    :param verbose: [bool] (default: False)
-    :return: [pd.DataFrame]
-
-    Example:
-        subregion_name        = 'leeds'
-        update                = False
-        confirmation_required = True
-        verbose               = True
-        fetch_bbbike_subregion_download_catalogue(subregion_name, update, confirmation_required, verbose)
-    """
-    subregion_name_ = regulate_bbbike_input_subregion_name(subregion_name)
-    path_to_file = cd_dat_bbbike(subregion_name_, subregion_name_ + "-download-catalogue.pickle")
-    if not os.path.isfile(path_to_file) or update:
-        collect_bbbike_subregion_download_catalogue(subregion_name, confirmation_required=confirmation_required,
-                                                    verbose=verbose)
-    try:
-        subregion_download_catalogue = load_pickle(path_to_file, verbose=verbose)
-        return subregion_download_catalogue
-    except Exception as e:
-        print(e)
+            # path_to_file = cd_dat_bbbike(subregion_name_, subregion_name_ + "-download-catalogue.pickle")
+            # save_pickle(subregion_downloads_catalogue, path_to_file, verbose=verbose)
+            print("Done. ") if verbose else ""
 
+        except Exception as e_:
+            subregion_downloads_catalogue = None
+            print("Failed. {}".format(subregion_name_, e_)) if verbose else ""
 
-#
-def collect_bbbike_download_catalogue(confirmation_required=True, verbose=False):
-    """
-    :param confirmation_required: [bool] (default: True)
-    :param verbose: [bool] (default: False)
+        return subregion_downloads_catalogue
 
-    Example:
-        confirmation_required = True
-        verbose               = True
-        collect_bbbike_download_catalogue(confirmation_required, verbose)
-    """
     if confirmed("To collect BBBike download dictionary? ", confirmation_required=confirmation_required):
         try:
-            bbbike_subregion_names = fetch_bbbike_subregion_catalogue("BBBike-subregion-name-list", update=True)
-            download_catalogue = [
-                fetch_bbbike_subregion_download_catalogue(subregion_name, update=True, confirmation_required=False)
-                for subregion_name in bbbike_subregion_names]
+            bbbike_subregion_names = fetch_bbbike_subregion_catalogue("BBBike-subregion-name-list", verbose=verbose)
+            print("Collecting BBBike download catalogue for: ") if verbose else ""
+            download_catalogue = [collect_bbbike_subregion_download_catalogue(subregion_name)
+                                  for subregion_name in bbbike_subregion_names]
 
-            subregion_name, subregion_download_catalogue = bbbike_subregion_names[0], download_catalogue[0]
+            sr_name, sr_download_catalogue = bbbike_subregion_names[0], download_catalogue[0]
 
             # Available file formats
-            file_fmt = [re.sub('{}|CHECKSUM'.format(subregion_name), '', f)
-                        for f in subregion_download_catalogue.Filename]
+            file_fmt = [re.sub('{}|CHECKSUM'.format(sr_name), '', f) for f in sr_download_catalogue.Filename]
             save_pickle(file_fmt[:-2], cd_dat("BBBike-osm-file-formats.pickle"), verbose=verbose)
 
             # Available data types
-            data_typ = subregion_download_catalogue.DataType.tolist()
+            data_typ = sr_download_catalogue.DataType.tolist()
             save_pickle(data_typ[:-2], cd_dat("BBBike-osm-data-types.pickle"), verbose=verbose)
 
             # available_file_formats = dict(zip(file_fmt, file_ext))
 
             downloads_dictionary = dict(zip(bbbike_subregion_names, download_catalogue))
             save_pickle(downloads_dictionary, cd_dat("BBBike-download-catalogue.pickle"), verbose=verbose)
+
         except Exception as e:
-            print("Failed to collect BBBike download dictionary. {}".format(e))
-    else:
-        print("The information collection process was not activated. The existing local copy will be loaded instead.")
+            print("Failed to collect BBBike download dictionary. {}".format(e)) if verbose else ""
 
 
 #
@@ -242,6 +213,7 @@ def fetch_bbbike_download_catalogue(catalogue_name, update=False, verbose=False)
         print(e)
 
 
+#
 def regulate_bbbike_input_osm_file_format(osm_file_format):
     """
     :param osm_file_format: [str]