Refactor station list filter

CHANGES.rst

@@ -9,6 +9,7 @@ in progress
 - Set default format for "stream://" targets to "json"
 - Fix published messages getting lost when not starting
   the MQTT main loop after connecting to MQTT broker
+- Refactor station list filter
 
 
 2019-01-22 0.8.2

luftdatenpumpe/core.py

@@ -220,9 +220,9 @@ def request_live_data(self):
         timestamp = self.convert_timestamp(data[0]['timestamp'])
         log.info('Timestamp of first record: {}'.format(timestamp))
 
-        iterator = data
+        iterator = self.apply_filter(data)
         if self.progressbar:
-            iterator = tqdm(data)
+            iterator = tqdm(list(iterator))
 
         for item in iterator:
             try:
@@ -235,6 +235,33 @@ def request_live_data(self):
             except Exception as ex:
                 log.warning('Could not make reading from {}.\n{}'.format(item, exception_traceback()))
 
+    def apply_filter(self, data):
+
+        for item in data:
+
+            #log.info('item: %s', item)
+
+            # Decode JSON item
+            station_id = item['location']['id']
+            sensor_id = item['sensor']['id']
+
+            # If there is a filter defined, evaluate it.
+            # Skip further processing for specific country codes, station ids or sensor ids.
+            # TODO: Improve evaluating conditions.
+            if self.filter:
+                skip = False
+                if 'station' in self.filter:
+                    if station_id not in self.filter['station']:
+                        skip = True
+                if 'sensor' in self.filter:
+                    if sensor_id not in self.filter['sensor']:
+                        skip = True
+
+                if skip:
+                    continue
+
+            yield item
+
     def make_reading(self, item):
 
         #log.info('item: %s', item)
@@ -244,16 +271,6 @@ def make_reading(self, item):
         sensor_id = item['sensor']['id']
         sensor_type = item['sensor']['sensor_type']['name']
 
-        # If there is a filter defined, evaluate it.
-        # For specific location|sensor ids, skip further processing.
-        if self.filter:
-            if 'station' in self.filter:
-                if station_id not in self.filter['station']:
-                    return
-            if 'sensor' in self.filter:
-                if sensor_id not in self.filter['sensor']:
-                    return
-
         # Build reading
         reading = Munch(
             station=Munch(),