Skip to content

Commit

Permalink
new wrapper scripts + removed GEOGRID.TBL from repository as it
Browse files Browse the repository at this point in the history 
will be modified by the scripts
  • Loading branch information
@jbeezley
jbeezley committed Apr 27, 2009
1 parent afd4f91 commit 3bebc4f
Show file tree
Hide file tree
Showing 11 changed files with 924 additions and 142 deletions.
5 changes: 5 additions & 0 deletions WPS/configure
View file
Expand Up @@ -2,6 +2,11 @@

arg1=$1

# link GEOGRID.TBL
pushd geogrid >& /dev/null
ln -sf GEOGRID.TBL.NED GEOGRID.TBL
popd >& /dev/null

# Look for netcdf
if test -z "$NETCDF" ; then
for p in /usr/local/netcdf
Expand Down
1 change: 0 additions & 1 deletion WPS/geogrid/GEOGRID.TBL
View file

This file was deleted.

26 changes: 26 additions & 0 deletions WPS/geogrid_wrapper.sh
View file
@@ -0,0 +1,26 @@
#!/bin/bash

##############################################################################
#
# geogrid_wrapper.sh
#
# A simple wrapper script for geogrid.exe that attempts to fetch data from
# a usgs server if it fails from a halt_on_missing error. This script relies
# on a number of python scripts assumed to be in '../other/convert_to_geogrid'.
# It needs python 2.5+ as well as two external python modules:
#
# twill (http://twill.idyll.org/)
# gdal (http://www.gdal.org/)
#
# twill can simply be extracted into a directory inside your PYTHONPATH, but
# gdal requires a number of C libraries. I recommend finding a binary
# distribution from 'http://trac.osgeo.org/gdal/wiki/DownloadingGdalBinaries'
# rather than compiling your own.
#
# Author: Jonathan Beezley (jon.beezley.math@gmail.com)
# Date: April 26, 2009
#
##############################################################################

pypath="../other/convert_to_geogrid"
PYTHONPATH="${PYTHONPATH}:${pypath}" python ${pypath}/geogrid_wrapper.py ${@}
282 changes: 142 additions & 140 deletions other/convert_to_geogrid/fetch_data.py
View file
Expand Up @@ -38,152 +38,154 @@
"http://twill.idyll.org/ or `easy_install twill`"
sys.exit(3)

# a simple class with a write method
class WritableObject:
def __init__(self):
self.content = []
def write(self, string):
self.content.append(string)
def clear(self):
self.content=[]

def code(g):
if g.get_code() != 200:
return False
else:
return True

if __name__ != "__main__":
print "This code is not meant to be a module!!!"

usage = "usage: %prog [options] -- north south east west\n"
usage += "coordinates given in lat/lon decimal degrees\n"
usage += "('--' is required!)"
parse=OptionParser(usage)
parse.add_option("-v","--verbose",action="store_true",
help="set verbose output for debugging")
parse.add_option("-d","--dataset",action="store",
help="set usgs data set name [default: ND301HZ (1/3 arcsec elevation)]")
parse.set_defaults(verbose=False,dataset="ND301HZ")
(opts,args)=parse.parse_args(sys.argv[1:])
verbose=opts.verbose
if len(args) != 4:
#args=["39.9","39.","-106","-107"]
parse.print_usage()
print "Must supply coordinate information."
sys.exit(2)
args=[ float(a) for a in args ]
args.append(opt.dataset)
u="http://extract.cr.usgs.gov/Website/distreq/RequestSummary.jsp?AL=%9.6f,%9.6f,%9.6f,%9.6f&PL=%s," % \
tuple(args)
print "Generating data from:"
print u
print "This could take a few minutes"
foo = WritableObject()
sys.stdout = foo
g=twill.browser.TwillBrowser()
g.go(u)
if not code(g):
sys.stdout = sys.__stdout__
raise Exception("Couldn't open webpage")
a=g.get_all_forms()
v=[]
r=re.compile('http://\S*')
for f in a:
doclick=False
for i in f.controls:
# possibly collect other data here
if i.attrs['type'] == 'submit' and i.attrs['value'] == 'Download':
doclick=True
if doclick:
g._browser.form=f
foo.clear()

try:
g.submit()
except:
pass

if verbose:
sys.stdout = sys.__stdout__
print ""
print "Generated this output:"
print ""
print "".join(foo.content)
print ""
sys.stdout = foo

def main(argv):
class WritableObject:
def __init__(self):
self.content = []
def write(self, string):
self.content.append(string)
def clear(self):
self.content=[]

def code(g):
if g.get_code() != 200:
return False
else:
return True


usage = "usage: %prog [options] -- north south east west\n"
usage += "coordinates given in lat/lon decimal degrees\n"
usage += "('--' is required!)"
parse=OptionParser(usage)
parse.add_option("-v","--verbose",action="store_true",
help="set verbose output for debugging")
parse.add_option("-d","--dataset",action="store",
help="set usgs data set name [default: ND301HZ (1/3 arcsec elevation)]")
parse.set_defaults(verbose=False,dataset="ND302HT")
(opts,args)=parse.parse_args(argv)
verbose=opts.verbose
if len(args) != 4:
#args=["39.9","39.","-106","-107"]
parse.print_usage()
print "Must supply coordinate information."
sys.exit(2)
args=[ float(a) for a in args ]
args.append(opt.dataset)
u="http://extract.cr.usgs.gov/Website/distreq/RequestSummary.jsp?AL=%9.6f,%9.6f,%9.6f,%9.6f&PL=%s," % \
tuple(args)
print "Generating data from:"
print u
print "This could take a few minutes"
foo = WritableObject()
sys.stdout = foo
g=twill.browser.TwillBrowser()
g.go(u)
if not code(g):
sys.stdout = sys.__stdout__
raise Exception("Couldn't open webpage")
a=g.get_all_forms()
v=[]
r=re.compile('http://\S*')
for f in a:
doclick=False
for i in f.controls:
# possibly collect other data here
if i.attrs['type'] == 'submit' and i.attrs['value'] == 'Download':
doclick=True
if doclick:
g._browser.form=f
foo.clear()

if g.get_url() is not None:
sys.stdout = sys.__stdout__
print ""
print "WARNING"
print "The query redirected correctly... this is not expected."
print "Something might be going wrong."
if not verbose:
print "If the files didn't download correctly, rerun with '-v'"
print "Maybe the website has changed."
try:
g.submit()
except:
pass

if verbose:
sys.stdout = sys.__stdout__
print ""
print "Generated this output:"
print ""
print "".join(foo.content)
print ""
sys.stdout = foo


if g.get_url() is not None:
sys.stdout = sys.__stdout__
print ""
print "WARNING"
print "The query redirected correctly... this is not expected."
print "Something might be going wrong."
if not verbose:
print "If the files didn't download correctly, rerun with '-v'"
print "Maybe the website has changed."
if verbose:
print "Check the output below for error messages."
print "The page returned has a title: ",g.get_title()
print "and here is the full html:"
print g.get_html()
print ""
sys.stdout = foo
else:
#reopen the url because g.submit closes it!!!
g.go(u)

for vv in foo.content:
m=r.search(vv)
if m:
v.append(m.group())
break


sys.stdout = sys.__stdout__

print "Received the following URL's:"
print "\n".join(v)

print ""
print "Waiting for 1 minute for the source files to be generated."
time.sleep(60)
print "Downloading"
i=0
files=[]
maxretries=25
pause=60
for u in v:
i+=1
print u," ..."
sname="datafile%02i.tgz" % i
for i in range(maxretries):
(fname,finfo)=urllib.urlretrieve(u,sname)
if finfo.dict['content-type'] != 'text/html':
break
if verbose:
print "Check the output below for error messages."
print "The page returned has a title: ",g.get_title()
print "and here is the full html:"
print "Received: ", g.get_title()
print ""
print g.get_html()
print ""
sys.stdout = foo
print ""
print "Not finished yet, wait some more..."
else:
#reopen the url because g.submit closes it!!!
g.go(u)

for vv in foo.content:
m=r.search(vv)
if m:
v.append(m.group())
break


sys.stdout = sys.__stdout__

print "Received the following URL's:"
print "\n".join(v)

print ""
print "Waiting for 1 minute for the source files to be generated."
time.sleep(60)
print "Downloading"
i=0
files=[]
maxretries=25
pause=60
for u in v:
i+=1
print u," ..."
sname="datafile%02i.zip" % i
for i in range(maxretries):
(fname,finfo)=urllib.urlretrieve(u,sname)
if finfo.dict['content-type'] != 'text/html':
break
print "The data should be available by now..."
print "Something has gone wrong."
print "Try downloading the files manually."
raise Exception("Couldn't download data files!")
if verbose:
print "Received: ", g.get_title()
print ""
print g.get_html()
print "File meta information:"
print finfo.dict
print ""
print "Not finished yet, wait some more..."
else:
print "The data should be available by now..."
print "Something has gone wrong."
print "Try downloading the files manually."
raise Exception("Couldn't download data files!")
if verbose:
if finfo.dict['content-type'] != "application/x-compressed":
print "Strange, this doesn't seem to be a tgz file."
print "Saving with .tgz extension anyway."
print "saved to: "+fname
print ""
print "File meta information:"
print finfo.dict
print ""
if finfo.dict['content-type'] != "application/x-zip-compressed":
print "Strange, this doesn't seem to be a zip file."
print "Saving with .zip extension anyway."
print "saved to: "+fname
print ""
files.append(fname)


files.append(fname)

return files


if __name__ == "__main__":
f=main(sys.argv[1:])

3 changes: 2 additions & 1 deletion other/convert_to_geogrid/geogrid.py
View file
Expand Up @@ -292,7 +292,8 @@ def mainprog(argv):
print web
print "for a full listing of options."
else:
print g.info.dirname
#print g.info.dirname
return g.info.dirname

if __name__ == '__main__':
# mainprog(["-f","-v","testdir","24/w001001.adf"])
Expand Down

0 comments on commit 3bebc4f

Please sign in to comment.