From c52fee6512c198397f110a37b2d772764a82956d Mon Sep 17 00:00:00 2001 From: "Steven D. Lander" Date: Wed, 6 May 2015 11:15:47 -0400 Subject: [PATCH 1/3] Adding functions to gpkg for with-statments --- Packaging/tiles2gpkg_parallel.py | 103 ++++++++++++++++--------------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/Packaging/tiles2gpkg_parallel.py b/Packaging/tiles2gpkg_parallel.py index e631dac..51b856f 100755 --- a/Packaging/tiles2gpkg_parallel.py +++ b/Packaging/tiles2gpkg_parallel.py @@ -479,6 +479,10 @@ def matrix_height(self, value): class Geopackage(object): + def __enter__(self): + """With-statement caller""" + return self + def __init__(self, file_path, srs): """Constructor.""" self.__file_path = file_path @@ -774,10 +778,8 @@ def assimilate(self, source): raise remove(source) - def close(self): - """ - Closes the sqlite3 db handle for this object. - """ + def __exit__(self, type, value, traceback): + """Resource cleanup on destruction.""" self.db_con.close() @@ -1194,54 +1196,53 @@ def main(arg_list): # Build the tile matrix info object tile_info = build_lut(files, lower_left, arg_list.srs) # Initialize the output file - output_geopackage = Geopackage(arg_list.output_file, arg_list.srs) - if arg_list.threading: - # Enable tiling on multiple CPU cores - cores = cpu_count() - pool = Pool(cores) - # Build allocate dictionary - extra_args = dict(root_dir=root_dir, tile_info=tile_info, - lower_left=lower_left, srs=arg_list.srs, - imagery=arg_list.imagery, jpeg_quality=arg_list.q) - results = allocate(cores, pool, files, extra_args) - status = ["|", "/", "-", "\\"] - counter = 0 - try: - while True: - rem = sum([1 for item in results if not item.ready()]) - if rem == 0: - stdout.write("\r[X] Progress: [" + "=="*(cores-rem) + - " "*rem + "]") - stdout.flush() - print(" All Done!") - break - else: - stdout.write("\r[" + status[counter] + "] Progress: [" + - "=="*(cores-rem) + " "*rem + "]") - stdout.flush() - if counter != len(status)-1: - counter += 1 + with Geopackage(arg_list.output_file, arg_list.srs) as gpkg: + if arg_list.threading: + # Enable tiling on multiple CPU cores + cores = cpu_count() + pool = Pool(cores) + # Build allocate dictionary + extra_args = dict(root_dir=root_dir, tile_info=tile_info, + lower_left=lower_left, srs=arg_list.srs, + imagery=arg_list.imagery, jpeg_quality=arg_list.q) + results = allocate(cores, pool, files, extra_args) + status = ["|", "/", "-", "\\"] + counter = 0 + try: + while True: + rem = sum([1 for item in results if not item.ready()]) + if rem == 0: + stdout.write("\r[X] Progress: [" + "=="*(cores-rem) + + " "*rem + "]") + stdout.flush() + print(" All Done!") + break else: - counter = 0 - sleep(.25) - pool.close() - pool.join() - except KeyboardInterrupt: - print(" Interrupted!") - pool.terminate() - exit(1) - else: - # Debugging call to bypass multiprocessing (-T) - extra_args = dict(root_dir=root_dir, tile_info=tile_info, - lower_left=lower_left, srs=arg_list.srs, - imagery=arg_list.imagery, jpeg_quality=arg_list.q) - sqlite_worker(files, extra_args) - # Combine the individual temp databases into the output file - combine_worker_dbs(output_geopackage) - # Using the data in the output file, create the metadata for it - output_geopackage.update_metadata(tile_info) - output_geopackage.close() - print("Complete") + stdout.write("\r[" + status[counter] + "] Progress: [" + + "=="*(cores-rem) + " "*rem + "]") + stdout.flush() + if counter != len(status)-1: + counter += 1 + else: + counter = 0 + sleep(.25) + pool.close() + pool.join() + except KeyboardInterrupt: + print(" Interrupted!") + pool.terminate() + exit(1) + else: + # Debugging call to bypass multiprocessing (-T) + extra_args = dict(root_dir=root_dir, tile_info=tile_info, + lower_left=lower_left, srs=arg_list.srs, + imagery=arg_list.imagery, jpeg_quality=arg_list.q) + sqlite_worker(files, extra_args) + # Combine the individual temp databases into the output file + combine_worker_dbs(gpkg) + # Using the data in the output file, create the metadata for it + gpkg.update_metadata(tile_info) + print("Complete") if __name__ == '__main__': print(""" From 4d941a6856df418f5cddaf916ffd4fb03b5c90d2 Mon Sep 17 00:00:00 2001 From: "Steven D. Lander" Date: Wed, 6 May 2015 11:24:46 -0400 Subject: [PATCH 2/3] modified location of with statement --- Packaging/tiles2gpkg_parallel.py | 86 ++++++++++++++++---------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/Packaging/tiles2gpkg_parallel.py b/Packaging/tiles2gpkg_parallel.py index 51b856f..9ef14ea 100755 --- a/Packaging/tiles2gpkg_parallel.py +++ b/Packaging/tiles2gpkg_parallel.py @@ -1196,53 +1196,53 @@ def main(arg_list): # Build the tile matrix info object tile_info = build_lut(files, lower_left, arg_list.srs) # Initialize the output file - with Geopackage(arg_list.output_file, arg_list.srs) as gpkg: - if arg_list.threading: - # Enable tiling on multiple CPU cores - cores = cpu_count() - pool = Pool(cores) - # Build allocate dictionary - extra_args = dict(root_dir=root_dir, tile_info=tile_info, - lower_left=lower_left, srs=arg_list.srs, - imagery=arg_list.imagery, jpeg_quality=arg_list.q) - results = allocate(cores, pool, files, extra_args) - status = ["|", "/", "-", "\\"] - counter = 0 - try: - while True: - rem = sum([1 for item in results if not item.ready()]) - if rem == 0: - stdout.write("\r[X] Progress: [" + "=="*(cores-rem) + - " "*rem + "]") - stdout.flush() - print(" All Done!") - break + if arg_list.threading: + # Enable tiling on multiple CPU cores + cores = cpu_count() + pool = Pool(cores) + # Build allocate dictionary + extra_args = dict(root_dir=root_dir, tile_info=tile_info, + lower_left=lower_left, srs=arg_list.srs, + imagery=arg_list.imagery, jpeg_quality=arg_list.q) + results = allocate(cores, pool, files, extra_args) + status = ["|", "/", "-", "\\"] + counter = 0 + try: + while True: + rem = sum([1 for item in results if not item.ready()]) + if rem == 0: + stdout.write("\r[X] Progress: [" + "=="*(cores-rem) + + " "*rem + "]") + stdout.flush() + print(" All Done!") + break + else: + stdout.write("\r[" + status[counter] + "] Progress: [" + + "=="*(cores-rem) + " "*rem + "]") + stdout.flush() + if counter != len(status)-1: + counter += 1 else: - stdout.write("\r[" + status[counter] + "] Progress: [" + - "=="*(cores-rem) + " "*rem + "]") - stdout.flush() - if counter != len(status)-1: - counter += 1 - else: - counter = 0 - sleep(.25) - pool.close() - pool.join() - except KeyboardInterrupt: - print(" Interrupted!") - pool.terminate() - exit(1) - else: - # Debugging call to bypass multiprocessing (-T) - extra_args = dict(root_dir=root_dir, tile_info=tile_info, - lower_left=lower_left, srs=arg_list.srs, - imagery=arg_list.imagery, jpeg_quality=arg_list.q) - sqlite_worker(files, extra_args) - # Combine the individual temp databases into the output file + counter = 0 + sleep(.25) + pool.close() + pool.join() + except KeyboardInterrupt: + print(" Interrupted!") + pool.terminate() + exit(1) + else: + # Debugging call to bypass multiprocessing (-T) + extra_args = dict(root_dir=root_dir, tile_info=tile_info, + lower_left=lower_left, srs=arg_list.srs, + imagery=arg_list.imagery, jpeg_quality=arg_list.q) + sqlite_worker(files, extra_args) + # Combine the individual temp databases into the output file + with Geopackage(arg_list.output_file, arg_list.srs) as gpkg: combine_worker_dbs(gpkg) # Using the data in the output file, create the metadata for it gpkg.update_metadata(tile_info) - print("Complete") + print("Complete") if __name__ == '__main__': print(""" From ffd4b76b64adaf964d87728b24f77e0094481f37 Mon Sep 17 00:00:00 2001 From: "Steven D. Lander" Date: Wed, 6 May 2015 12:21:16 -0400 Subject: [PATCH 3/3] added enter and exit methods to TempDB --- Packaging/tiles2gpkg_parallel.py | 56 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/Packaging/tiles2gpkg_parallel.py b/Packaging/tiles2gpkg_parallel.py index 9ef14ea..7819458 100755 --- a/Packaging/tiles2gpkg_parallel.py +++ b/Packaging/tiles2gpkg_parallel.py @@ -495,12 +495,12 @@ def __init__(self, file_path, srs): self.__projection = ScaledWorldMercator() else: self.__projection = Geodetic() - self.db_con = connect(self.__file_path) + self.__db_con = connect(self.__file_path) self.__create_schema() def __create_schema(self): """Create default geopackage schema on the database.""" - with self.db_con as db_con: + with self.__db_con as db_con: cursor = db_con.cursor() cursor.execute(""" CREATE TABLE gpkg_contents ( @@ -688,7 +688,7 @@ def file_path(self): def update_metadata(self, metadata): """Update the metadata of the geopackage database after tile merge.""" # initialize a new projection - with self.db_con as db_con: + with self.__db_con as db_con: cursor = db_con.cursor() tile_matrix_stmt = """ INSERT OR REPLACE INTO gpkg_tile_matrix ( @@ -746,7 +746,7 @@ def update_metadata(self, metadata): def execute(self, statement, inputs=None): """Execute a prepared SQL statement on this geopackage database.""" - with self.db_con as db_con: + with self.__db_con as db_con: cursor = db_con.cursor() if inputs is not None: result_cursor = cursor.execute(statement, inputs) @@ -758,7 +758,7 @@ def assimilate(self, source): """Assimilate .gpkg.part tiles into this geopackage database.""" if not exists(source): raise IOError - with self.db_con as db_con: + with self.__db_con as db_con: cursor = db_con.cursor() cursor.execute("pragma synchronous = off;") cursor.execute("pragma journal_mode = off;") @@ -780,7 +780,7 @@ def assimilate(self, source): def __exit__(self, type, value, traceback): """Resource cleanup on destruction.""" - self.db_con.close() + self.__db_con.close() class TempDB(object): @@ -789,6 +789,10 @@ class TempDB(object): Has a .gpkg.part file format. """ + def __enter__(self): + """With-statement caller.""" + return self + def __init__(self, filename): """ Constructor. @@ -799,8 +803,8 @@ def __init__(self, filename): uid = uuid4() self.name = uid.hex + '.gpkg.part' self.__file_path = join(filename, self.name) - self.db_con = connect(self.__file_path) - with self.db_con as db_con: + self.__db_con = connect(self.__file_path) + with self.__db_con as db_con: cursor = db_con.cursor() stmt = """ CREATE TABLE tiles ( @@ -824,7 +828,7 @@ def __init__(self, filename): """ def execute(self, statement, inputs=None): - with self.db_con as db_con: + with self.__db_con as db_con: cursor = db_con.cursor() if inputs is not None: result_cursor = cursor.execute(statement, inputs) @@ -843,15 +847,13 @@ def insert_image_blob(self, z, x, y, data): y -- the column number of the data data -- the image data containing in a binary array """ - with self.db_con as db_con: + with self.__db_con as db_con: cursor = db_con.cursor() cursor.execute(self.image_blob_stmt, (z, x, y, data)) - def close(self): - """ - Closes this sqlite3 database handle. - """ - self.db_con.close() + def __exit__(self, type, value, traceback): + """Resource cleanup on destruction.""" + self.__db_con.close() def img_to_buf(img, img_type, jpeg_quality=75): @@ -1015,18 +1017,18 @@ def sqlite_worker(file_list, extra_args): the tiles in the TMS directory """ temp_db = TempDB(extra_args['root_dir']) - invert_y = None - if extra_args['lower_left']: - if extra_args['srs'] == 3857: - invert_y = Mercator.invert_y - elif extra_args['srs'] == 4326: - invert_y = Geodetic.invert_y - elif extra_args['srs'] == 3395: - invert_y = EllipsoidalMercator.invert_y - elif extra_args['srs'] == 9804: - invert_y = ScaledWorldMercator.invert_y - [worker_map(temp_db, item, extra_args, invert_y) for item in file_list] - temp_db.close() + with TempDB(extra_args['root_dir']) as temp_db: + invert_y = None + if extra_args['lower_left']: + if extra_args['srs'] == 3857: + invert_y = Mercator.invert_y + elif extra_args['srs'] == 4326: + invert_y = Geodetic.invert_y + elif extra_args['srs'] == 3395: + invert_y = EllipsoidalMercator.invert_y + elif extra_args['srs'] == 9804: + invert_y = ScaledWorldMercator.invert_y + [worker_map(temp_db, item, extra_args, invert_y) for item in file_list] def allocate(cores, pool, file_list, extra_args):