From 1368b6549096d9a47f90206a59f401cce5ed312f Mon Sep 17 00:00:00 2001 From: Klaus Deja Date: Sun, 23 Jun 2019 22:35:44 +0200 Subject: [PATCH 1/3] Python3 Progress --- dwdweather/client.py | 2 +- dwdweather/commands.py | 2 +- dwdweather/core.py | 39 ++++++++++++++++++++++++++++++++------- dwdweather/knowledge.py | 4 ++-- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/dwdweather/client.py b/dwdweather/client.py index 82f7659..5ac0170 100644 --- a/dwdweather/client.py +++ b/dwdweather/client.py @@ -4,7 +4,7 @@ import io import os import logging -from urlparse import urlparse +from urllib.parse import urlparse from zipfile import ZipFile from requests_cache import CachedSession diff --git a/dwdweather/commands.py b/dwdweather/commands.py index 3d41582..1da60cc 100644 --- a/dwdweather/commands.py +++ b/dwdweather/commands.py @@ -49,7 +49,7 @@ def get_weather(args): categories = args.categories log.info('Querying data for station "{station_id}" and categories "{categories}" at "{timestamp}"'.format(**locals())) results = dw.query(station_id, timestamp) - print json.dumps(results, indent=4, sort_keys=True) + print(json.dumps(results, indent=4, sort_keys=True)) argparser = argparse.ArgumentParser(prog="dwdweather", description="Get weather information for Germany.") diff --git a/dwdweather/core.py b/dwdweather/core.py index a282edd..cd0fddf 100644 --- a/dwdweather/core.py +++ b/dwdweather/core.py @@ -9,7 +9,7 @@ import math import logging import sqlite3 -import StringIO +from io import StringIO import traceback from tqdm import tqdm @@ -188,10 +188,10 @@ def import_station(self, content): Takes the content of one station metadata file and imports it into the database. """ + content = content.decode("latin1") content = content.strip() content = content.replace("\r", "") content = content.replace("\n\n", "\n") - content = content.decode("latin1") table = self.get_stations_table() @@ -323,14 +323,14 @@ def import_measures_textfile(self, result): # Build UPSERT SQL statement. # https://www.sqlite.org/lang_UPSERT.html sql_template = "INSERT INTO {table} ({fields}) VALUES ({value_placeholders}) " \ - "ON CONFLICT (station_id, datetime) DO UPDATE SET {sets} WHERE station_id=? AND datetime=?".format( + "ON CONFLICT (station_id, datetime) DO UPDATE SET {sets}C".format( table=tablename, fields=', '.join(fieldnames), value_placeholders=', '.join(value_placeholders), sets=', '.join(sets)) # Create data rows. c = self.db.cursor() count = 0 - items = result.payload.split("\n") + items = result.payload.decode("latin-1").split("\n") for line in tqdm(items, ncols=79): count += 1 line = line.strip() @@ -386,6 +386,33 @@ def import_measures_textfile(self, result): self.db.commit() + def datetime_to_string(self, datetime): + return int(datetime.replace('T', '').replace(':', '')) + + def get_measurement(self, station_id, date): + tablename = self.get_measurement_table() + sql = 'SELECT TOP 1 FROM {tablename} WHERE station_id = {station_id}, datetime = {datetime}'.format( + tablename=tablename, station_id=station_id, datetime=self.datetime_to_string(date) + ) + + c = self.db.cursor() + c.execute(sql) + + result = [] + for row in c.execute(sql): + result.append(row) + + if len(result) > 0: + return result[0] + else: + return None + + def insert_measurement(self): + return None + + def update_measurement(self): + return None + def get_data_age(self): """ Return age of latest dataset as ``datetime.timedelta``. @@ -508,7 +535,7 @@ def stations_csv(self, delimiter=","): """ Return stations list as CSV. """ - csvfile = StringIO.StringIO() + csvfile = StringIO() # assemble field list headers = ["station_id", "date_start", "date_end", "geo_lon", "geo_lat", "height", "name"] writer = csv.writer(csvfile, delimiter=delimiter, quoting=csv.QUOTE_MINIMAL) @@ -524,8 +551,6 @@ def stations_csv(self, delimiter=","): val = str(val) elif type(val) == float: val = "%.4f" % val - elif type(val) == unicode: - val = val.encode("utf8") row.append(val) writer.writerow(row) contents = csvfile.getvalue() diff --git a/dwdweather/knowledge.py b/dwdweather/knowledge.py index a2860ef..242d661 100644 --- a/dwdweather/knowledge.py +++ b/dwdweather/knowledge.py @@ -530,7 +530,7 @@ class minutes_10: def get_resolutions(cls): resolutions_map = OrderedDict() resolutions = DwdCdcKnowledge.as_dict(cls.resolutions) - for name, class_ in resolutions.iteritems(): + for name, class_ in resolutions.items(): folder = class_.__folder__ resolutions_map[folder] = class_ return resolutions_map @@ -538,7 +538,7 @@ def get_resolutions(cls): @classmethod def get_resolution_by_name(cls, resolution): resolutions_map = cls.get_resolutions() - return resolutions_map[resolution] + return resolutions_map.get(resolution) @classmethod def as_dict(cls, what): From f48a794c239561edd35201e40bfc8c1bb6389ed9 Mon Sep 17 00:00:00 2001 From: Klaus Deja Date: Thu, 27 Jun 2019 00:18:29 +0200 Subject: [PATCH 2/3] Upsert SQL --- dwdweather/core.py | 83 ++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/dwdweather/core.py b/dwdweather/core.py index cd0fddf..776b86e 100644 --- a/dwdweather/core.py +++ b/dwdweather/core.py @@ -20,7 +20,6 @@ from dwdweather.knowledge import DwdCdcKnowledge from dwdweather import __appname__ as APP_NAME -from dwdweather import __version__ as APP_VERSION """ Python client to access weather data from Deutscher Wetterdienst (DWD), @@ -290,6 +289,45 @@ def import_measures(self, station_id, latest=True, historic=False): #log.warning("No files to import for station %s" % station_id) self.import_measures_textfile(result) + def datetime_to_int(self, datetime): + return int(datetime.replace('T', '').replace(':', '')) + + def get_measurement(self, station_id, date): + tablename = self.get_measurement_table() + sql = 'SELECT * FROM {tablename} WHERE station_id = {station_id} AND datetime = {datetime}'.format( + tablename=tablename, station_id=station_id, datetime=date + ) + + c = self.db.cursor() + c.execute(sql) + + result = [] + for row in c.execute(sql): + result.append(row) + + if len(result) > 0: + return result[0] + else: + return None + + def insert_measurement(self, tablename, fields, value_placeholders, dataset): + sql = 'INSERT INTO {tablename} ({fields}) VALUES ({value_placeholders})'.format( + tablename=tablename, fields=', '.join(fields), value_placeholders=', '.join(value_placeholders) + ) + + c = self.db.cursor() + c.execute(sql, dataset) + #self.db.commit() + + def update_measurement(self, tablename, sets, dataset): + sql = 'UPDATE {tablename} SET {sets} WHERE station_id = ? AND datetime = ?'.format( + tablename=tablename, sets=', '.join(sets) + ) + + c = self.db.cursor() + c.execute(sql, dataset) + #self.db.commit() + def import_measures_textfile(self, result): """ Import content of source text file into database. @@ -320,15 +358,7 @@ def import_measures_textfile(self, result): fieldnames.append(fieldname) value_placeholders.append('?') - # Build UPSERT SQL statement. - # https://www.sqlite.org/lang_UPSERT.html - sql_template = "INSERT INTO {table} ({fields}) VALUES ({value_placeholders}) " \ - "ON CONFLICT (station_id, datetime) DO UPDATE SET {sets}C".format( - table=tablename, fields=', '.join(fieldnames), - value_placeholders=', '.join(value_placeholders), sets=', '.join(sets)) - # Create data rows. - c = self.db.cursor() count = 0 items = result.payload.decode("latin-1").split("\n") for line in tqdm(items, ncols=79): @@ -371,7 +401,7 @@ def import_measures_textfile(self, result): traceback.print_tb(trace) sys.exit() elif fieldtype == "datetime": - parts[n] = int(parts[n].replace('T', '').replace(':', '')) + parts[n] = self.datetime_to_int(parts[n]) dataset.append(parts[n]) @@ -382,37 +412,12 @@ def import_measures_textfile(self, result): #log.debug('SQL template: %s', sql_template) #log.debug('Dataset: %s', dataset) - c.execute(sql_template, dataset + dataset) - + if self.get_measurement(parts[0], parts[1]): + self.update_measurement(tablename, sets, dataset) + else: + self.insert_measurement(tablename, fieldnames, value_placeholders, dataset) self.db.commit() - def datetime_to_string(self, datetime): - return int(datetime.replace('T', '').replace(':', '')) - - def get_measurement(self, station_id, date): - tablename = self.get_measurement_table() - sql = 'SELECT TOP 1 FROM {tablename} WHERE station_id = {station_id}, datetime = {datetime}'.format( - tablename=tablename, station_id=station_id, datetime=self.datetime_to_string(date) - ) - - c = self.db.cursor() - c.execute(sql) - - result = [] - for row in c.execute(sql): - result.append(row) - - if len(result) > 0: - return result[0] - else: - return None - - def insert_measurement(self): - return None - - def update_measurement(self): - return None - def get_data_age(self): """ Return age of latest dataset as ``datetime.timedelta``. From c6cb1747156ecdaa007caa45f50e3e4454c6272d Mon Sep 17 00:00:00 2001 From: Klaus Deja Date: Thu, 27 Jun 2019 00:28:37 +0200 Subject: [PATCH 3/3] Updated documentation and version numbers --- CHANGES.rst | 8 ++++++++ README.rst | 2 +- dwdweather/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 760c252..69cab65 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,9 +6,17 @@ in progress =========== - Make README.rst ASCII-clean re. #5 +2019-06-027 0.8.3 +================= +- Python 3.6 compatibility +- Running two consecutive INSERT / UPDATE statements instead of a single + UPSERT statement as the sqlite version delivered with Python does not + support this feature. + 2019-06-03 0.8.2 ================ - Reestablish Python 2.7 compatibility for ``setup.py``. +- 2019-06-03 0.8.1 ================ diff --git a/README.rst b/README.rst index a489541..f947126 100644 --- a/README.rst +++ b/README.rst @@ -151,7 +151,7 @@ and Status ****** This piece of software is in a very early stage. No test cases yet. Only -tested with Python 2.7. Use at your own risk. +tested with Python 3.6. Use at your own risk. Credits ======= diff --git a/dwdweather/__init__.py b/dwdweather/__init__.py index e2771d8..55c4b92 100644 --- a/dwdweather/__init__.py +++ b/dwdweather/__init__.py @@ -1,5 +1,5 @@ """dwdweather2: Python client to access weather data from Deutscher Wetterdienst (DWD).""" __appname__ = 'dwdweather2' -__version__ = '0.8.2' +__version__ = '0.8.3' from .core import DwdWeather diff --git a/setup.py b/setup.py index 1887049..f287325 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ README = open(os.path.join(here, 'README.rst'), encoding='UTF-8').read() setup(name='dwdweather2', - version='0.8.2', + version='0.8.3', description='Python client to access weather data from Deutscher Wetterdienst (DWD), ' 'the federal meteorological service in Germany.', long_description=README,