Improve tidying function

earthobservations · Jun 24, 2021 · 0fa168a · 0fa168a
1 parent 782fc12
commit 0fa168a
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 28 deletions.
diff --git a/wetterdienst/core/scalar/values.py b/wetterdienst/core/scalar/values.py
@@ -7,6 +7,7 @@
 from enum import Enum
 from typing import Dict, Generator, List, Tuple, Union
 
+import numpy as np
 import pandas as pd
 from pint import Quantity
 from pytz import timezone
@@ -539,10 +540,15 @@ def tidy_up_df(self, df: pd.DataFrame, dataset: Enum) -> pd.DataFrame:
 
         df[Columns.VALUE.value] = pd.to_numeric(df[Columns.VALUE.value]).astype(float)
 
+        if Columns.QUALITY.value not in df:
+            df[Columns.QUALITY.value] = np.nan
+
         df[Columns.QUALITY.value] = pd.to_numeric(df[Columns.QUALITY.value]).astype(
             float
         )
 
+        df.loc[df[Columns.VALUE.value].isna(), Columns.QUALITY.value] = np.NaN
+
         return df
 
     @abstractmethod
@@ -702,7 +708,6 @@ def all(self) -> ValuesResult:
 
         tqdm_out = TqdmToLogger(log, level=logging.INFO)
 
-        # TODO: write tqdm progressbar to logging stream
         for result in tqdm(
             self.query(), total=len(self.stations.station_id), file=tqdm_out
         ):
@@ -716,18 +721,6 @@ def all(self) -> ValuesResult:
         # Have to reapply category dtype after concatenation
         df = self._coerce_meta_fields(df)
 
-        # for column in (
-        #     Columns.STATION_ID.value,
-        #     Columns.PARAMETER.value,
-        #     Columns.QUALITY.value,
-        # ):
-        #     try:
-        #         df[column] = df[column].astype("category")
-        #     except KeyError:
-        #         pass
-
-        # df.attrs["tidy"] = self.stations.tidy
-
         return ValuesResult(stations=self.stations, df=df)
 
     def _humanize(self, df: pd.DataFrame) -> pd.DataFrame:

diff --git a/wetterdienst/provider/dwd/forecast/api.py b/wetterdienst/provider/dwd/forecast/api.py
@@ -75,10 +75,6 @@ class DwdMosmixValues(ScalarValuesCore):
     _data_tz = Timezone.UTC
     _has_quality = False
 
-    # @property
-    # def _tidy(self) -> bool:
-    #     return self.stations.tidy
-
     _irregular_parameters = tuple()
     _integer_parameters = INTEGER_PARAMETERS
     _string_parameters = tuple()
@@ -136,14 +132,16 @@ def query(self) -> Generator[ValuesResult, None, None]:
             df = self._coerce_parameter_types(df)
 
             if self.stations.stations.tidy:
-                df = self._tidy_up_df(df)
+                df = self.tidy_up_df(df, self.stations.stations.mosmix_type)
+
+                # df = self._tidy_up_df(df)
 
-                df[
-                    Columns.DATASET.value
-                ] = self.stations.stations.mosmix_type.value.lower()
-                df[Columns.VALUE.value] = pd.to_numeric(
-                    df[Columns.VALUE.value], errors="coerce"
-                ).astype(float)
+                # df[
+                #     Columns.DATASET.value
+                # ] = self.stations.stations.mosmix_type.value.lower()
+                # df[Columns.VALUE.value] = pd.to_numeric(
+                #     df[Columns.VALUE.value], errors="coerce"
+                # ).astype(float)
 
             df = self._coerce_meta_fields(df)
 
@@ -179,7 +177,7 @@ def _collect_station_parameter(self) -> Generator[pd.DataFrame, None, None]:
                     log.warning(e)
                     continue
 
-    def _tidy_up_df(self, df: pd.DataFrame) -> pd.DataFrame:
+    def _tidy_up_df(self, df: pd.DataFrame, dataset) -> pd.DataFrame:
         """
 
         :param df:

diff --git a/wetterdienst/provider/dwd/observation/api.py b/wetterdienst/provider/dwd/observation/api.py
@@ -6,7 +6,6 @@
 from itertools import repeat
 from typing import Dict, List, Optional, Tuple, Union
 
-import numpy as np
 import pandas as pd
 
 from wetterdienst.core.scalar.request import ScalarRequestCore
@@ -303,8 +302,6 @@ def _tidy_up_df(self, df: pd.DataFrame, dataset) -> pd.DataFrame:
 
         df_tidy[Columns.QUALITY.value] = quality.reset_index(drop=True)
 
-        df_tidy.loc[df_tidy[Columns.VALUE.value].isna(), Columns.QUALITY.value] = np.NaN
-
         return df_tidy
 
     def _coerce_dates(self, series: pd.Series) -> pd.Series: