Fix meta_index

Fix meta_index for 1minute precipitation historical, by replacing regular mp, which led to a unknown bug, by concurrency mp
panodata · Jul 1, 2020 · 1ad2744 · 1ad2744
1 parent 78ec74d
commit 1ad2744
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 9 deletions.
diff --git a/python_dwd/indexing/meta_index_creation.py b/python_dwd/indexing/meta_index_creation.py
@@ -4,7 +4,7 @@
 from pathlib import PurePosixPath
 from typing import Tuple
 import pandas as pd
-from multiprocessing import Pool
+from multiprocessing.dummy import Pool
 import functools
 import datetime as dt
 import requests
@@ -149,11 +149,13 @@ def _create_meta_index_for_1minute__historical_precipitation() -> pd.DataFrame:
 
     meta_index_df = pd.DataFrame(None, columns=METADATA_COLUMNS)
 
-    metadata_files = Pool().map(
-        _download_metadata_file_for_1minute_precipitation, metadata_file_paths)
+    with Pool() as p:
+        metadata_files = p.map(
+            _download_metadata_file_for_1minute_precipitation, metadata_file_paths)
 
-    metadata_dfs = Pool().map(
-        _parse_geo_metadata, zip(metadata_files, station_ids))
+    with Pool() as p:
+        metadata_dfs = p.map(
+            _parse_geo_metadata, zip(metadata_files, station_ids))
 
     meta_index_df = meta_index_df.append(other=metadata_dfs, ignore_index=True)
 

diff --git a/tests/indexing/test_meta_index_creation.py b/tests/indexing/test_meta_index_creation.py
@@ -36,10 +36,6 @@ def test_meta_index_creation():
 
 
 @pytest.mark.remote
-@patch(
-    "python_dwd.file_path_handling.path_handling.list_files_of_climate_observations",
-    mock.MagicMock(return_value=["1_minute/precipitation/meta_data/Meta_Daten_ein_min_rr_00003.zip"])
-)
 def test_meta_index_1mph_creation():
     reset_meta_index_cache()
     meta_index_1mph = create_meta_index_for_dwd_data(