Merge pull request #8 from wtfuii/master

Python 3 compatibility / Removed UPSERT

CHANGES.rst

@@ -6,9 +6,17 @@ in progress
 ===========
 - Make README.rst ASCII-clean re. #5
 
+2019-06-027 0.8.3
+=================
+- Python 3.6 compatibility
+- Running two consecutive INSERT / UPDATE statements instead of a single
+  UPSERT statement as the sqlite version delivered with Python does not
+  support this feature.
+
 2019-06-03 0.8.2
 ================
 - Reestablish Python 2.7 compatibility for ``setup.py``.
+-
 
 2019-06-03 0.8.1
 ================

README.rst

@@ -151,7 +151,7 @@ and
 Status
 ******
 This piece of software is in a very early stage. No test cases yet. Only
-tested with Python 2.7. Use at your own risk.
+tested with Python 3.6. Use at your own risk.
 
 Credits
 =======

dwdweather/__init__.py

@@ -1,5 +1,5 @@
 """dwdweather2: Python client to access weather data from Deutscher Wetterdienst (DWD)."""
 __appname__ = 'dwdweather2'
-__version__ = '0.8.2'
+__version__ = '0.8.3'
 
 from .core import DwdWeather

dwdweather/client.py

@@ -4,7 +4,7 @@
 import io
 import os
 import logging
-from urlparse import urlparse
+from urllib.parse import urlparse
 from zipfile import ZipFile
 
 from requests_cache import CachedSession

dwdweather/commands.py

@@ -49,7 +49,7 @@ def get_weather(args):
         categories = args.categories
         log.info('Querying data for station "{station_id}" and categories "{categories}" at "{timestamp}"'.format(**locals()))
         results = dw.query(station_id, timestamp)
-        print json.dumps(results, indent=4, sort_keys=True)
+        print(json.dumps(results, indent=4, sort_keys=True))
 
     argparser = argparse.ArgumentParser(prog="dwdweather", description="Get weather information for Germany.")
 

dwdweather/core.py

@@ -9,7 +9,7 @@
 import math
 import logging
 import sqlite3
-import StringIO
+from io import StringIO
 import traceback
 
 from tqdm import tqdm
@@ -20,7 +20,6 @@
 from dwdweather.knowledge import DwdCdcKnowledge
 
 from dwdweather import __appname__ as APP_NAME
-from dwdweather import __version__ as APP_VERSION
 
 """
 Python client to access weather data from Deutscher Wetterdienst (DWD),
@@ -188,10 +187,10 @@ def import_station(self, content):
         Takes the content of one station metadata file
         and imports it into the database.
         """
+        content = content.decode("latin1")
         content = content.strip()
         content = content.replace("\r", "")
         content = content.replace("\n\n", "\n")
-        content = content.decode("latin1")
 
         table = self.get_stations_table()
 
@@ -290,6 +289,45 @@ def import_measures(self, station_id, latest=True, historic=False):
                 #log.warning("No files to import for station %s" % station_id)
                 self.import_measures_textfile(result)
 
+    def datetime_to_int(self, datetime):
+        return int(datetime.replace('T', '').replace(':', ''))
+
+    def get_measurement(self, station_id, date):
+        tablename = self.get_measurement_table()
+        sql = 'SELECT * FROM {tablename} WHERE station_id = {station_id} AND datetime = {datetime}'.format(
+            tablename=tablename, station_id=station_id, datetime=date
+        )
+
+        c = self.db.cursor()
+        c.execute(sql)
+
+        result = []
+        for row in c.execute(sql):
+            result.append(row)
+
+        if len(result) > 0:
+            return result[0]
+        else:
+            return None
+
+    def insert_measurement(self, tablename, fields, value_placeholders, dataset):
+        sql = 'INSERT INTO {tablename} ({fields}) VALUES ({value_placeholders})'.format(
+            tablename=tablename, fields=', '.join(fields), value_placeholders=', '.join(value_placeholders)
+        )
+
+        c = self.db.cursor()
+        c.execute(sql, dataset)
+        #self.db.commit()
+
+    def update_measurement(self, tablename, sets, dataset):
+        sql = 'UPDATE {tablename} SET {sets} WHERE station_id = ? AND datetime = ?'.format(
+            tablename=tablename, sets=', '.join(sets)
+        )
+
+        c = self.db.cursor()
+        c.execute(sql, dataset)
+        #self.db.commit()
+
     def import_measures_textfile(self, result):
         """
         Import content of source text file into database.
@@ -320,17 +358,9 @@ def import_measures_textfile(self, result):
             fieldnames.append(fieldname)
             value_placeholders.append('?')
 
-        # Build UPSERT SQL statement.
-        # https://www.sqlite.org/lang_UPSERT.html
-        sql_template = "INSERT INTO {table} ({fields}) VALUES ({value_placeholders}) " \
-                       "ON CONFLICT (station_id, datetime) DO UPDATE SET {sets} WHERE station_id=? AND datetime=?".format(
-                        table=tablename, fields=', '.join(fieldnames),
-                        value_placeholders=', '.join(value_placeholders), sets=', '.join(sets))
-
         # Create data rows.
-        c = self.db.cursor()
         count = 0
-        items = result.payload.split("\n")
+        items = result.payload.decode("latin-1").split("\n")
         for line in tqdm(items, ncols=79):
             count += 1
             line = line.strip()
@@ -371,7 +401,7 @@ def import_measures_textfile(self, result):
                             traceback.print_tb(trace)
                             sys.exit()
                     elif fieldtype == "datetime":
-                        parts[n] = int(parts[n].replace('T', '').replace(':', ''))
+                        parts[n] = self.datetime_to_int(parts[n])
 
                     dataset.append(parts[n])
 
@@ -382,8 +412,10 @@ def import_measures_textfile(self, result):
                 #log.debug('SQL template: %s', sql_template)
                 #log.debug('Dataset: %s', dataset)
 
-                c.execute(sql_template, dataset + dataset)
-
+                if self.get_measurement(parts[0], parts[1]):
+                    self.update_measurement(tablename, sets, dataset)
+                else:
+                    self.insert_measurement(tablename, fieldnames, value_placeholders, dataset)
         self.db.commit()
 
     def get_data_age(self):
@@ -508,7 +540,7 @@ def stations_csv(self, delimiter=","):
         """
         Return stations list as CSV.
         """
-        csvfile = StringIO.StringIO()
+        csvfile = StringIO()
         # assemble field list
         headers = ["station_id", "date_start", "date_end", "geo_lon", "geo_lat", "height", "name"]
         writer = csv.writer(csvfile, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL)
@@ -524,8 +556,6 @@ def stations_csv(self, delimiter=","):
                     val = str(val)
                 elif type(val) == float:
                     val = "%.4f" % val
-                elif type(val) == unicode:
-                    val = val.encode("utf8")
                 row.append(val)
             writer.writerow(row)
         contents = csvfile.getvalue()

dwdweather/knowledge.py

@@ -530,15 +530,15 @@ class minutes_10:
         def get_resolutions(cls):
             resolutions_map = OrderedDict()
             resolutions = DwdCdcKnowledge.as_dict(cls.resolutions)
-            for name, class_ in resolutions.iteritems():
+            for name, class_ in resolutions.items():
                 folder = class_.__folder__
                 resolutions_map[folder] = class_
             return resolutions_map
 
         @classmethod
         def get_resolution_by_name(cls, resolution):
             resolutions_map = cls.get_resolutions()
-            return resolutions_map[resolution]
+            return resolutions_map.get(resolution)
 
     @classmethod
     def as_dict(cls, what):

setup.py

@@ -7,7 +7,7 @@
 README = open(os.path.join(here, 'README.rst'), encoding='UTF-8').read()
 
 setup(name='dwdweather2',
-      version='0.8.2',
+      version='0.8.3',
       description='Python client to access weather data from Deutscher Wetterdienst (DWD), '
                   'the federal meteorological service in Germany.',
       long_description=README,