diff --git a/CHANGES.txt b/CHANGES.txt index 743a674..b0973bc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1 +1,8 @@ -v0.0.1, March 14, 2011 -- Initial release +### v0.1.0 + +* Switch tests to nosetest +* Switch default output to XYZ + +### v0.0.1, March 14, 2011 + +* Initial release diff --git a/Makefile b/Makefile deleted file mode 100644 index 1b2b3ff..0000000 --- a/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -test/tmp: - mkdir test/tmp - -test/data: - mkdir test/data - wget http://mapbox-tilesets.s3.amazonaws.com/mbtiles/World_Glass_1.1.mbtiles.zip -o test/data/World_Glass_1.1.mbtiles.zip - unzip test/data/World_Glass_1.1.mbtiles.zip - mv World_Glass_1.1.mbtiles test/data - -test/tmp/world_glass: - time ./mb-util test/data/World_Glass_1.1.mbtiles test/tmp/world_glass - -test/tmp/world_glass_out.mbtiles: - time ./mb-util test/tmp/world_glass test/tmp/world_glass_out.mbtiles - -test: test/data test/tmp test/tmp/world_glass test/tmp/world_glass_out.mbtiles diff --git a/README.md b/README.md index bd769e0..2aafc0c 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,20 @@ Import a directory into a `mbtiles` file MBUtil imports and exports metadata as JSON, in the root of the tile directory, as a file named `metadata.json`. - { - "name": "World Light", - "description": "A Test Metadata", - "version": "3" - } +```javascript +{ + "name": "World Light", + "description": "A Test Metadata", + "version": "3" +} +``` + +## Testing + +This project uses [nosetests](http://readthedocs.org/docs/nose/en/latest/) for testing. Install nosetests +and run + + nosetests ## Authors diff --git a/mb-util b/mb-util index 49dbfbd..d9d9ec5 100755 --- a/mb-util +++ b/mb-util @@ -1,16 +1,16 @@ #!/usr/bin/env python + # MBUtil: a tool for MBTiles files # Supports importing, exporting, and more # -# (c) Development Seed 2011 +# (c) Development Seed 2012 # Licensed under BSD -import logging -import os, sys + +import logging, os, sys from optparse import OptionParser from mbutil import mbtiles_to_disk, disk_to_mbtiles - if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) @@ -20,42 +20,39 @@ if __name__ == '__main__': Examples: Export an mbtiles file to a directory of files: - $ mb-util world.mbtiles tiles # tiles must not alredy exist + $ mb-util world.mbtiles tiles # tiles must not already exist Import a directory of tiles into an mbtiles file: $ mb-util tiles world.mbtiles # mbtiles file must not already exist""") - # todo - add support for compression - #parser.add_option('-w', '--window', dest='window', - # help='compression window size. larger values faster, dangerouser', - # type='int', - # default=2000) - parser.add_option('--scheme', dest='scheme', - help='Tiling scheme for exporting tiles. Default is "tms" (z/x/y), other option is "osm" which is also z/x/y but uses a flipped y coordinate', + help='''Tiling scheme for exporting tiles. Default is "xyz" (z/x/y), + other option is "tms" which is also z/x/y + but uses a flipped y coordinate''', type='string', - default='tms') + default='xyz') (options, args) = parser.parse_args() # Transfer operations - if len(args) == 2: - if os.path.isfile(args[0]) and os.path.exists(args[1]): - sys.stderr.write('To export MBTiles to disk, specify a directory that does not yet exist\n') - sys.exit(1) - - # to disk - if os.path.isfile(args[0]) and not os.path.exists(args[1]): - mbtiles_file, directory_path = args - mbtiles_to_disk(mbtiles_file, directory_path, **options.__dict__) - - if os.path.isdir(args[0]) and os.path.isfile(args[1]): - sys.stderr.write('Importing tiles into already-existing MBTiles is not yet supported\n') - sys.exit(1) - - # to mbtiles - if os.path.isdir(args[0]) and not os.path.isfile(args[0]): - directory_path, mbtiles_file = args - disk_to_mbtiles(directory_path, mbtiles_file, **options.__dict__) - else: + if len(args) != 2: parser.print_help() + sys.exit(1) + + if os.path.isfile(args[0]) and os.path.exists(args[1]): + sys.stderr.write('To export MBTiles to disk, specify a directory that does not yet exist\n') + sys.exit(1) + + # to disk + if os.path.isfile(args[0]) and not os.path.exists(args[1]): + mbtiles_file, directory_path = args + mbtiles_to_disk(mbtiles_file, directory_path, **options.__dict__) + + if os.path.isdir(args[0]) and os.path.isfile(args[1]): + sys.stderr.write('Importing tiles into already-existing MBTiles is not yet supported\n') + sys.exit(1) + + # to mbtiles + if os.path.isdir(args[0]) and not os.path.isfile(args[0]): + directory_path, mbtiles_file = args + disk_to_mbtiles(directory_path, mbtiles_file, **options.__dict__) diff --git a/mbutil/util.py b/mbutil/util.py index 4974c5e..5f823e7 100755 --- a/mbutil/util.py +++ b/mbutil/util.py @@ -3,10 +3,10 @@ # MBUtil: a tool for MBTiles files # Supports importing, exporting, and more # -# (c) Development Seed 2011 +# (c) Development Seed 2012 # Licensed under BSD -import sqlite3, uuid, sys, logging, time, os, json, zlib, glob, shutil +import sqlite3, uuid, sys, logging, time, os, json, zlib logger = logging.getLogger(__name__) @@ -21,10 +21,10 @@ def mbtiles_setup(cur): tile_row integer, tile_data blob); """) - cur.execute("""create table metadata + cur.execute("""create table metadata (name text, value text);""") cur.execute("""create unique index name on metadata (name);""") - cur.execute("""create unique index tile_index on tiles + cur.execute("""create unique index tile_index on tiles (zoom_level, tile_column, tile_row);""") def mbtiles_connect(mbtiles_file): @@ -39,7 +39,7 @@ def mbtiles_connect(mbtiles_file): def optimize_connection(cur): cur.execute("""PRAGMA synchronous=0""") cur.execute("""PRAGMA locking_mode=EXCLUSIVE""") - cur.execute("""PRAGMA journal_mode=TRUNCATE""") + cur.execute("""PRAGMA journal_mode=DELETE""") def compression_prepare(cur, con): cur.execute(""" @@ -49,9 +49,9 @@ def compression_prepare(cur, con): """) cur.execute(""" CREATE TABLE if not exists map ( - zoom_level integer, - tile_column integer, - tile_row integer, + zoom_level integer, + tile_column integer, + tile_row integer, tile_id VARCHAR(256)); """) @@ -83,8 +83,8 @@ def compression_do(cur, con, chunk): if r[3] in files: overlapping = overlapping + 1 start = time.time() - query = """insert into map - (zoom_level, tile_column, tile_row, tile_id) + query = """insert into map + (zoom_level, tile_column, tile_row, tile_id) values (?, ?, ?, ?)""" logger.debug("insert: %s" % (time.time() - start)) cur.execute(query, (r[0], r[1], r[2], ids[files.index(r[3])])) @@ -96,14 +96,14 @@ def compression_do(cur, con, chunk): files.append(r[3]) start = time.time() - query = """insert into images - (tile_id, tile_data) + query = """insert into images + (tile_id, tile_data) values (?, ?)""" cur.execute(query, (str(id), sqlite3.Binary(r[3]))) logger.debug("insert into images: %s" % (time.time() - start)) start = time.time() - query = """insert into map - (zoom_level, tile_column, tile_row, tile_id) + query = """insert into map + (zoom_level, tile_column, tile_row, tile_id) values (?, ?, ?, ?)""" cur.execute(query, (r[0], r[1], r[2], id)) logger.debug("insert into map: %s" % (time.time() - start)) @@ -118,10 +118,10 @@ def compression_finalize(cur): images.tile_data as tile_data FROM map JOIN images on images.tile_id = map.tile_id;""") cur.execute(""" - CREATE UNIQUE INDEX map_index on map + CREATE UNIQUE INDEX map_index on map (zoom_level, tile_column, tile_row);""") cur.execute(""" - CREATE UNIQUE INDEX images_id on images + CREATE UNIQUE INDEX images_id on images (tile_id);""") cur.execute("""vacuum;""") cur.execute("""analyze;""") @@ -177,13 +177,12 @@ def mbtiles_to_disk(mbtiles_file, directory_path, **kwargs): logger.debug("Exporting MBTiles to disk") logger.debug("%s --> %s" % (mbtiles_file, directory_path)) con = mbtiles_connect(mbtiles_file) - cur = con.cursor() os.mkdir("%s" % directory_path) metadata = dict(con.execute('select name, value from metadata;').fetchall()) - json.dump(metadata, open('%s/metadata.json' % directory_path, 'w'),indent=4) + json.dump(metadata, open('%s/metadata.json' % directory_path, 'w'), indent=4) count = con.execute('select count(zoom_level) from tiles;').fetchone()[0] done = 0 - msg ='' + msg = '' service_version = metadata.get('version', '1.0.0') base_path = os.path.join(directory_path, service_version, @@ -222,7 +221,7 @@ def mbtiles_to_disk(mbtiles_file, directory_path, **kwargs): # grids done = 0 - msg ='' + msg = '' try: count = con.execute('select count(zoom_level) from grids;').fetchone()[0] grids = con.execute('select zoom_level, tile_column, tile_row, grid from grids;') @@ -232,8 +231,7 @@ def mbtiles_to_disk(mbtiles_file, directory_path, **kwargs): while g: zoom_level = g[0] # z tile_column = g[1] # x - tile_row = g[2] # y - y = g[2] + y = g[2] # y if kwargs.get('scheme') == 'osm': y = flip_y(zoom_level,y) grid_dir = os.path.join(base_path, str(zoom_level), str(tile_column)) @@ -243,7 +241,11 @@ def mbtiles_to_disk(mbtiles_file, directory_path, **kwargs): f = open(grid, 'w') grid_json = json.loads(zlib.decompress(g[3])) # join up with the grid 'data' which is in pieces when stored in mbtiles file - grid_data_cursor = con.execute('select key_name, key_json FROM grid_data WHERE zoom_level = %(zoom_level)d and tile_column = %(tile_column)d and tile_row = %(tile_row)d;' % locals()) + grid_data_cursor = con.execute('''select key_name, key_json FROM + grid_data WHERE + zoom_level = %(zoom_level)d and + tile_column = %(tile_column)d and + tile_row = %(tile_row)d;''' % locals()) grid_data = grid_data_cursor.fetchone() data = {} while grid_data: diff --git a/setup.py b/setup.py index f6d347e..c7e03a3 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ setup( name='mbutil', - version='0.0.2', + version='0.1.0', author='Tom MacWright', - author_email='macwright@gmail.com', + author_email='tom@macwright.org', packages=['mbutil'], scripts=['mb-util'], url='https://github.com/mapbox/mbutil', diff --git a/test/data/one_tile.mbtiles b/test/data/one_tile.mbtiles new file mode 100644 index 0000000..70b87e0 Binary files /dev/null and b/test/data/one_tile.mbtiles differ diff --git a/test/test.py b/test/test.py new file mode 100644 index 0000000..6f4f94b --- /dev/null +++ b/test/test.py @@ -0,0 +1,22 @@ +import os, shutil +from nose import with_setup +from mbutil import mbtiles_to_disk, disk_to_mbtiles + +def clear_data(): + try: shutil.rmtree('test/output') + except Exception: pass + + try: os.path.mkdir('test/output') + except Exception: pass + +@with_setup(clear_data, clear_data) +def test_mbtiles_to_disk(): + mbtiles_to_disk('test/data/one_tile.mbtiles', 'test/output') + assert os.path.exists('test/output/1.0.0/shadowplay/0/0/0.png') + +@with_setup(clear_data, clear_data) +def test_mbtiles_to_disk_and_back(): + mbtiles_to_disk('test/data/one_tile.mbtiles', 'test/output') + assert os.path.exists('test/output/1.0.0/shadowplay/0/0/0.png') + disk_to_mbtiles('test/output/one', 'test/output/one.mbtiles') + assert os.path.exists('test/output/one.mbtiles')