test/verify_download.py @@ -5,17 +5,22 @@ http://wiki.openstreetmap.org/index.php/Route_altitude_profiles_SRTM#SRTM_import Usage: First run the tests that come with this script: -$ cd test -$ python test_download.py -$ cd .. +python test/test_download.py -Then download the SRTM source files (635 MB): +Then download the SRTM source files (635 MB for Australia): -$ python download.py +$ python download.py continent + +(replace continent by Africa, Australia, Eurasia, Islands, +North_America or South_America. + +Second argument (optional) specifies from which tile to resume. +Use full file name e.g. 'N36W004.hgt.zip'. Set to 0 start at the first file. +Argument 3-6 optionally specify a bounding box: north, south, west, east. Verify that the download went correctly: -$ python verify_download.py +python test/verify_download.py continent Unzip the files (adds 3 GB): $ for f in `ls *.zip`; do unzip $f; done @@ -39,10 +44,10 @@ Run script that reads files and puts them in the database. Test it first: $ python test/test_read_data.py -$ python read_data.py +$ python read_data.py continent Verify the result -$ python read_data.py verify +$ python read_data.py continent verify All altitude data should now be in the table 'altitude'. Enjoy. README @@ -1,3 +1,5 @@ -db='srtm' -db_user='postgres' -db_pass='' +# For Postgres, uncomment the lines below and fill in the correct values + +#db='srtm' +#db_user='postgres' +#db_pass='' src/database_template.py @@ -1,16 +1,10 @@ # Downloads the SRTM data for Australia -# -# Loosely based on example at: -# http://postneo.com/stories/2003/01/01/beyondTheBasicPythonFtplibExample.html # Import some libraries: from ftplib import FTP import urllib import re - -# First we get the list of files through an FTP connection. - -ftp = FTP('e0srp01u.ecs.nasa.gov') +import sys def handleDownload(block): file.write(block) @@ -35,30 +29,78 @@ def getLatLonFromFileName(name): return [lat,lon] -ftp.login() - -ftp.cwd("srtm/version2/SRTM3/Eurasia") -#ftp.cwd("srtm/version2/SRTM3/Australia") - -# Now list all Australian tiles. -# See ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/Australia/ - -# List all files (should be 1060) -files = ftp.nlst() - -# And close connection. - -ftp.close() - -# Now download all files using urllib.urlretrieve - -for i in range(len(files)): - [lat,lon] = getLatLonFromFileName(files[i]) - # Only download if in Germany or NL: - # if(47 <= lat and lat <=54 and 0 <= lon and lon <= 16): - # if(51 <= lat and lat <=51 and 8 <= lon and lon <= 16): - if(53 <= lat and lat <=54 and 0 <= lon and lon <= 16): - print "Downloading " + files[i] + " (lat = " + str(lat) + " , lon = " + str(lon) + " )... (" + str(i + 1) + " of " + str(len(files)) +")" - urllib.urlretrieve("ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/Eurasia/" + files[i],"data/Eurasia/" + files[i]) - #urllib.urlretrieve("ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/Australia/" + files[i],"data/Australia/" + files[i]) - +def main(): + # First we make a list of all files that need to be download. This depends + # on the arguments given to the program. + # The first argument should be the continent: + # * Africa + # * Australia + # * Eurasia + # * Islands + # * North_America + # * South_America + + if len(sys.argv) > 1: + continent = sys.argv[1] + if not continent in ["Africa", "Australia", "Eurasia", "Islands", "North_America", "South_America"]: + print "First argument should be Africa, Australia, Eurasia, Islands, North_America or South_America." + exit() + else: + print "Please provide arguments \n",\ + "First argument should be Africa, Australia, Eurasia, Islands, North_America or South_America.\n",\ + "Second argument (optional) specifies from which tile to resume. Use full file name e.g. \n",\ + "'N36W004.hgt.zip'. Set to 0 start at the first file. \n",\ + "Argument 3-6 optionally specify a bounding box: north, south, west, east" + exit() + + # First we get the list of files through an FTP connection. + ftp = FTP('e0srp01u.ecs.nasa.gov') + + ftp.login() + + ftp.cwd("srtm/version2/SRTM3/" + continent) + + # Now list all tiles of that continent. + # See ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/[continent]/ + + files = ftp.nlst() + + # And close connection. + + ftp.close() + + print len(files) + exit() + + # Now download all files using urllib.urlretrieve + + # Determine if we need to resume at a certain point + if(len(sys.argv) > 1): + resume = argv[1] + skip = True + + # Do we have a bounding box? + if len(sys.argv) == 6: + north = sys.argv[2] + south = sys.argv[3] + west = sys.argv[4] + east = sys.argv[5] + else: + north = 90 + south = -90 + east = -180 + west = 180 + + for i in range(len(files)): + if skip: + if files[i] == resume: + skip = False + + if not(skip): + [lat,lon] = getLatLonFromFileName(files[i]) + if(south <= lat and lat <=north and west <= lon and lon <= east): + print "Downloading " + files[i] + " (lat = " + str(lat) + " , lon = " + str(lon) + " )... (" + str(i + 1) + " of " + str(len(files)) +")" + urllib.urlretrieve("ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/Eurasia/" + files[i],"data/" + continent + "/" + files[i]) + +if __name__ == '__main__': + main() src/download.py @@ -8,7 +8,7 @@ from math import sqrt # Main functions -def loadTile(filename): +def loadTile(continent, filename): srtm = gdal.Open('data/Australia/' + filename + '.hgt') return gdal_array.DatasetReadAsArray(srtm) src/read_data.py @@ -1,3 +1,2 @@ *.swp -database.py database_test.py test/.gitignore @@ -1,3 +1,5 @@ -db='srtm_test' -db_user='postgres' -db_pass='' +# For Postgres, uncomment the lines below and fill in the correct values + +#db='srtm_test' +#db_user='postgres' +#db_pass='' test/database_test_template.py @@ -4,6 +4,8 @@ import pg import unittest sys.path += [os.path.abspath('.')] +sys.path += [os.path.abspath('.') + "/src"] + import database_test from read_data import * @@ -161,4 +163,10 @@ class TestDatabase(unittest.TestCase): dropAllTables(self.db); if __name__ == '__main__': + # We will only do this for a PostGIS database: + + try: + file("POSTGIS") unittest.main() + except: + print "Only tests for PostGIS database at the moment." test/test_read_data.py src/verify_download.py e67aec6e52ba2dfe09c0b9a3c1f76c09b170f6b8 Sjors sjors@sprovoost.nl http://github.com/Sjors/srtm2postgis/commit/23bb37004394f18a4e3f895450468b92ad5e7297 23bb37004394f18a4e3f895450468b92ad5e7297 2008-07-12T01:33:25-07:00 2008-07-12T01:33:25-07:00 Made some changes. 6c8fbe98b742d1b89d15dff215899e6e901855ab Sjors sjors@sprovoost.nl