From 173bfe96b0e20aca360163e66de6206188577e82 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 11:32:38 +0200 Subject: [PATCH 01/23] Added a progress bar... forgot to add to this commit though, so next time. Cleaned up URL retrieval process (no force write to HD) --- astrodata/magpis/magpis.py | 5 +- astrodata/ukidss/ukidss.py | 108 +++++++++++++++++++----------------- astrodata/utils/__init__.py | 2 + 3 files changed, 63 insertions(+), 52 deletions(-) diff --git a/astrodata/magpis/magpis.py b/astrodata/magpis/magpis.py index 74e24e8a32..b75dd96919 100644 --- a/astrodata/magpis/magpis.py +++ b/astrodata/magpis/magpis.py @@ -18,6 +18,7 @@ import time import tempfile import StringIO +from astrodata.utils import progressbar url_gpscutout = "http://third.ucllnl.org/cgi-bin/gpscutout" @@ -92,8 +93,10 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, request = urllib.urlencode(request) # load the URL as text U = urllib.urlopen(url_gpscutout, request) + # read results with progressbar + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) # turn the text into a StringIO object for FITS reading - S = StringIO.StringIO(U.read()) + S = StringIO.StringIO(results) fitsfile = pyfits.open(S) if save: diff --git a/astrodata/ukidss/ukidss.py b/astrodata/ukidss/ukidss.py index 339bb94f1a..8c46aed5c3 100644 --- a/astrodata/ukidss/ukidss.py +++ b/astrodata/ukidss/ukidss.py @@ -18,9 +18,10 @@ import time import tempfile import numpy as np +import shutil +from astrodata.utils import progressbar - -__all__ = ['UKIDSSQuery'] +__all__ = ['UKIDSSQuery','clean_catalog'] class LinksExtractor(htmllib.HTMLParser): # derive new HTML parser @@ -94,7 +95,7 @@ def logged_in(self): return True def get_image_gal(self, glon, glat, filter='all', frametype='stack', - directory=None, size=1.0, verbose=False, save=True, + directory=None, size=1.0, verbose=True, save=True, savename=None, overwrite=False): """ Get an image at a specified glon/glat. Size can be specified @@ -116,6 +117,11 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', Print out extra error messages? save : bool Save FITS file? + savename : string or None + The file name to save the catalog to. If unspecified, will save as + UKIDSS_[band]_G###.###-###.###_[obj].fits.gz, where the #'s + indicate galactic lon/lat and [band] and [obj] refer to the filter + and the object name overwrite : bool Overwrite if file already exists? @@ -152,7 +158,10 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', # Retrieve page page = self.opener.open(url_getimage, urllib.urlencode(request)) - results = page.read() + if verbose: + results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) + else: + results = page.read() # Parse results for links format = formatter.NullFormatter() @@ -170,20 +179,22 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', if not os.path.exists(directory): os.mkdir(directory) - if not os.path.exists(directory + '/' + frametype): - os.mkdir(directory + '/' + frametype) # Get image filename basename = os.path.basename( link.split("&")[0]).replace('.fit', '.fits.gz') - temp_file = directory + '/' + frametype + '/' + basename + temp_file = tempfile.NamedTemporaryFile() - # Get the file, and store temporarily - urllib.urlretrieve( - link.replace("getImage", "getFImage"), temp_file) + # Get the file + U = self.opener.open(link.replace("getImage", "getFImage")) + if verbose: + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = U.read() + S = StringIO.StringIO(results) + fitsfile = pyfits.open(S) # Get Multiframe ID from the header - fitsfile = pyfits.open(temp_file) images.append(fitsfile) if save: @@ -191,26 +202,24 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', filt = str(h0['FILTER']).strip() obj = filt + "_" + str(h0['OBJECT']).strip().replace(":", ".") - # Set final directory and file names - final_dir = directory + '/' + frametype + '/' + obj - final_file = final_dir + '/' + basename + if savename is None: + savename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj) - # Create MFID directory if not existent - if not os.path.exists(final_dir): - os.mkdir(final_dir) + # Set final directory and file names + final_file = directory + '/' + savename if not overwrite: # Check that the final file doesn't already exist if os.path.exists(final_file): raise IOError("File exists : " + final_file) - os.rename(temp_file, final_file) + shutil.copy(temp_file.name, final_file) return fitsfile def get_images_radius(self, ra, dec, radius, filter='all', frametype='stack', directory=None, n_concurrent=1, save=True, - verbose=False, overwrite=False): + verbose=True, overwrite=False): """ Get all images within some radius of a specified RA/Dec @@ -287,7 +296,10 @@ def get_images_radius(self, ra, dec, radius, filter='all', # Retrieve page page = self.opener.open(url_getimages, urllib.urlencode(request)) - results = page.read() + if verbose: + results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) + else: + results = page.read() # Parse results for links format = formatter.NullFormatter() @@ -310,7 +322,8 @@ def get_images_radius(self, ra, dec, radius, filter='all', basename = os.path.basename(link.split("&")[0]) temp_file = directory + '/' + frametype + '/' + basename - print "Downloading %s..." % basename + if verbose: + print "Downloading %s..." % basename p = mp.Process( target=urllib.urlretrieve, args=(link, temp_file)) @@ -321,28 +334,8 @@ def get_images_radius(self, ra, dec, radius, filter='all', break time.sleep(0.1) - # urllib.urlretrieve(link, temp_file) - - # # Get Multiframe ID from the header - # h0 = pyfits.getheader(temp_file) - # filt = str(h0['FILTER']).strip() - # obj = filt+"_"+str(h0['OBJECT']).strip().replace(":",".") - # - # # Set final directory and file names - # final_dir = directory+'/'+frametype+'/'+obj - # final_file = final_dir+'/'+basename - # - # # Create MFID directory if not existent - # if not os.path.exists(final_dir): - # os.mkdir(final_dir) - # - # # Check that the final file doesn't already exist - # if os.path.exists(final_file): - # sys.exit("File exists : "+final_file) - # - # os.rename(temp_file,final_file) - - def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False): + def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False, + verbose=True, savename=None): """ Get all sources in the catalog within some radius @@ -355,6 +348,10 @@ def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False): Directory to download files into. Defaults to self.directory radius : float Radius in which to search for catalog entries in arcminutes + savename : string or None + The file name to save the catalog to. If unspecified, will save as + UKIDSS_catalog_G###.###-###.###_r###.fits.gz, where the #'s indicate + galactic lon/lat and radius Returns ------- @@ -392,7 +389,10 @@ def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False): # Retrieve page page = self.opener.open(url_getcatalog + urllib.urlencode(request)) - results = page.read() + if verbose: + results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) + else: + results = page.read() # Parse results for links format = formatter.NullFormatter() # create default formatter @@ -412,15 +412,21 @@ def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False): os.mkdir(directory) if save: - filename = directory + "/catalog_" + str(c) + ".fits.gz" + if savename is None: + savename = "UKIDSS_catalog_G%07.3f%+08.3f_r%03i.fits.gz" % (glon,glat,radius) + filename = directory + "/" + savename + + U = self.opener.open(link) + if verbose: + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: - outfile = tempfile.NamedTemporaryFile() - filename = outfile.name - urllib.urlretrieve(link, filename) + results = U.read() + S = StringIO.StringIO(results) + fitsfile = pyfits.open(S) - data.append(pyfits.open(filename)) - if not save: - outfile.close() + data.append(fitsfile) + if save: + fitsfile.writeto(filename, clobber=overwrite) return data diff --git a/astrodata/utils/__init__.py b/astrodata/utils/__init__.py index c750873c05..7e01721fcf 100644 --- a/astrodata/utils/__init__.py +++ b/astrodata/utils/__init__.py @@ -2,3 +2,5 @@ # This sub-module is destined for common non-package specific utility # functions that will ultimately be merged into `astropy.utils` + +from .progressbar import * From 326089decf565a1c03c6f15a253c7ab30fc59199 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 11:33:15 +0200 Subject: [PATCH 02/23] added progressbar code --- astrodata/utils/progressbar.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 astrodata/utils/progressbar.py diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py new file mode 100644 index 0000000000..c67d260831 --- /dev/null +++ b/astrodata/utils/progressbar.py @@ -0,0 +1,48 @@ +import urllib2, sys + +__all__ = ['chunk_report','chunk_read'] + +def chunk_report(bytes_so_far, chunk_size, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % + (bytes_so_far, total_size, percent)) + else: + sys.stdout.write("Downloaded %10.2g Mb\r" % + (bytes_so_far / 1024.**2)) + + +def chunk_read(response, chunk_size=1024, report_hook=None): + content_length = response.info().getheader('Content-Length') + if content_length is None: + total_size = 0 + else: + total_size = content_length.strip() + total_size = int(total_size) + + bytes_so_far = 0 + + result_string = "" + + #sys.stdout.write("Beginning download.\n") + + while 1: + chunk = response.read(chunk_size) + result_string += chunk + bytes_so_far += len(chunk) + + if not chunk: + if report_hook: + sys.stdout.write('\n') + break + + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + + return result_string + +if __name__ == '__main__': + response = urllib2.urlopen('http://www.ebay.com') + C = chunk_read(response, report_hook=chunk_report) + From 2b3abd3a19afe6ed94d1176246b83d163f0bc966 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 21:39:58 +0200 Subject: [PATCH 03/23] UKIDSS catalog getting fixes - there were problems with gziping, fitsfile writing, stringio objects... etc. --- astrodata/ukidss/ukidss.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/astrodata/ukidss/ukidss.py b/astrodata/ukidss/ukidss.py index 8c46aed5c3..2a31dffc96 100644 --- a/astrodata/ukidss/ukidss.py +++ b/astrodata/ukidss/ukidss.py @@ -10,6 +10,7 @@ import urllib import htmllib import formatter +import gzip import os import sys import astropy.io.fits as pyfits @@ -19,6 +20,7 @@ import tempfile import numpy as np import shutil +import StringIO from astrodata.utils import progressbar __all__ = ['UKIDSSQuery','clean_catalog'] @@ -159,6 +161,7 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', # Retrieve page page = self.opener.open(url_getimage, urllib.urlencode(request)) if verbose: + print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() @@ -188,11 +191,12 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', # Get the file U = self.opener.open(link.replace("getImage", "getFImage")) if verbose: + print "Downloading image from %s" % link results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = U.read() S = StringIO.StringIO(results) - fitsfile = pyfits.open(S) + fitsfile = pyfits.open(S,ignore_missing_end=True) # Get Multiframe ID from the header images.append(fitsfile) @@ -297,6 +301,7 @@ def get_images_radius(self, ra, dec, radius, filter='all', # Retrieve page page = self.opener.open(url_getimages, urllib.urlencode(request)) if verbose: + print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() @@ -335,7 +340,7 @@ def get_images_radius(self, ra, dec, radius, filter='all', time.sleep(0.1) def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False, - verbose=True, savename=None): + verbose=True, savename=None, overwrite=False): """ Get all sources in the catalog within some radius @@ -390,6 +395,7 @@ def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False, # Retrieve page page = self.opener.open(url_getcatalog + urllib.urlencode(request)) if verbose: + print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() @@ -418,15 +424,22 @@ def get_catalog_gal(self, glon, glat, directory=None, radius=1, save=False, U = self.opener.open(link) if verbose: + print "Downloading catalog %s" % link results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = U.read() S = StringIO.StringIO(results) - fitsfile = pyfits.open(S) + try: + fitsfile = pyfits.open(S,ignore_missing_end=True) + except IOError: + S.seek(0) + G = gzip.GzipFile(fileobj=S) + fitsfile = pyfits.open(G,ignore_missing_end=True) + data.append(fitsfile) if save: - fitsfile.writeto(filename, clobber=overwrite) + fitsfile.writeto(filename.rstrip(".gz"), clobber=overwrite) return data From 36ca8fa8eceb71ca94cb59359932bac226746b9a Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 21:40:34 +0200 Subject: [PATCH 04/23] progress bar tweaks --- astrodata/utils/progressbar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index c67d260831..a913d13a7f 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,7 +6,7 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % + sys.stdout.write("Downloaded %12d of %12d bytes (%6.2f%%)\r" % (bytes_so_far, total_size, percent)) else: sys.stdout.write("Downloaded %10.2g Mb\r" % From 31efeeb2be4724cc59923072e8d4a799fccff85e Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:44:01 +0200 Subject: [PATCH 05/23] added the besancon query tool --- astrodata/besancon/besancon.py | 265 +++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 astrodata/besancon/besancon.py diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py new file mode 100644 index 0000000000..656360c7a1 --- /dev/null +++ b/astrodata/besancon/besancon.py @@ -0,0 +1,265 @@ +import urllib,urllib2 +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[20]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
    \s* + (
  1. ([a-zA-Z0-9 \s_-]*)
    2. \s*)*\s* +
\s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} +mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), + 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude upper,lower limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Upper and Lower magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in (mag_limits): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s" % filename + + if retrieve_file: + return get_besancon_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except URLError: + sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return results + + From 53a34f72655e4ab41afa6ee0545eb466eee6fc36 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:50:32 +0200 Subject: [PATCH 06/23] documentation updates --- astrodata/besancon/besancon.py | 8 ++++++++ astrodata/magpis/magpis.py | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 656360c7a1..6ff69097d8 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -1,3 +1,11 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" import urllib,urllib2 import time import copy diff --git a/astrodata/magpis/magpis.py b/astrodata/magpis/magpis.py index b75dd96919..0533271abc 100644 --- a/astrodata/magpis/magpis.py +++ b/astrodata/magpis/magpis.py @@ -2,7 +2,6 @@ MAGPIS Image and Catalog Query Tool ----------------------------------- -:Author: Thomas Robitalle (thomas.robitaille@gmail.com) :Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) """ import cookielib From 096a7ba740db4c73d79601327d72be193287d0a4 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:54:42 +0200 Subject: [PATCH 07/23] added __init__ to besancon --- astrodata/besancon/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 astrodata/besancon/__init__.py diff --git a/astrodata/besancon/__init__.py b/astrodata/besancon/__init__.py new file mode 100644 index 0000000000..60c2920749 --- /dev/null +++ b/astrodata/besancon/__init__.py @@ -0,0 +1 @@ +from .besancon import * From cfe992e4f0cb2562e7dafe2866cbd5264cdb9562 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:27 +0200 Subject: [PATCH 08/23] documentation added, and it works (assuming you have the dependencies...) --- astrodata/besancon/besancon.py | 41 ++++++++++++------- docs/astrodata/besancon.rst | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 docs/astrodata/besancon.rst diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 6ff69097d8..a4fbb187ce 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -6,13 +6,17 @@ :Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) """ -import urllib,urllib2 +import urllib +import urllib2 +import socket import time import copy from astrodata.utils import progressbar import sys import re +__all__ = ['get_besancon_model_file','request_besancon'] + keyword_defaults = { 'rinf':0.000000, 'rsup':50.000000, @@ -34,7 +38,7 @@ 'ev':[""]*24, 'di':[""]*24, 'oo':[-7]+[-99]*12, - 'ff':[20]+[99]*12, + 'ff':[15]+[99]*12, 'spectyp_min':1, 'subspectyp_min': 0, 'spectyp_max':9, @@ -93,11 +97,12 @@ def parse_errors(text): try: errors = re.compile(r"""\s*
      \s* - (
    1. ([a-zA-Z0-9 \s_-]*)
      2. \s*)*\s* + (
    2. ([a-zA-Z0-9):( \s_-]*)
      3. \s*)*\s*
    \s* """, re.X) text = errors.search(text).group() except AttributeError: + likely_errors = text.split('\n')[132:150] raise ValueError("Regular expression matching to error message failed.") text_items = re.split("
  • |
    • |\n",errors.search(text).group()) text_items = [t for t in text_items if t != ""] @@ -105,13 +110,13 @@ def parse_errors(text): return error_list -colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} -mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), - 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} mag_order = "U","B","V","R","I","J","H","K","L" def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, - area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), mag_limits=copy.copy(mag_limits), colors_limits=copy.copy(colors_limits), retrieve_file=True, **kwargs): @@ -134,12 +139,12 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, area : float Area in square degrees absmag_limits : (float,float) - Absolute magnitude upper,lower limits + Absolute magnitude lower,upper limits colors_limits : dict of (float,float) Should contain 4 elements listing color differences in the valid bands, e.g.: {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} mag_limits = dict of (float,float) - Upper and Lower magnitude difference limits for each magnitude band + Lower and Upper magnitude difference limits for each magnitude band U B V R I J H K L clouds : list of 2-tuples Up to 25 line-of-sight clouds can be specified in pairs of (A_V, @@ -173,7 +178,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, kwd['oo'][0] = absmag_limits[0] kwd['ff'][0] = absmag_limits[1] - for ii,(key,val) in enumerate(colors_limits): + for ii,(key,val) in enumerate(colors_limits.items()): if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: kwd['colind'][ii] = key kwd['oo'][ii+9] = val[0] @@ -181,7 +186,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, else: raise ValueError('Invalid color %s' % key) - for (key,val) in (mag_limits): + for (key,val) in mag_limits.iteritems(): if key in mag_order: kwd['band0'][mag_order.index(key)] = val[0] kwd['bandf'][mag_order.index(key)] = val[1] @@ -213,7 +218,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, print "File is %s" % filename if retrieve_file: - return get_besancon_file(filename) + return get_besancon_model_file(filename) else: return filename @@ -244,6 +249,7 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov sys.stdout.write("\n") while 1: + sys.stdout.write(u"\r") try: U = urllib2.urlopen(url,timeout=5) if verbose: @@ -253,12 +259,18 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov else: results = page.read() break - except URLError: - sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) time.sleep(30) elapsed_time += 30 continue + if save: if savename is None: savename = filename @@ -270,4 +282,3 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov return results - diff --git a/docs/astrodata/besancon.rst b/docs/astrodata/besancon.rst new file mode 100644 index 0000000000..280826e249 --- /dev/null +++ b/docs/astrodata/besancon.rst @@ -0,0 +1,72 @@ +.. _astrodata.besancon: + +***************************************** +Besancon Queries (`astrodata.besancon`) +***************************************** + +Getting started +=============== + +The following example illustrates an Besancon catalog/image query. The API describes the +relevant keyword arguments (of which there are many) :: + + >>> from astrodata import besancon + >>> besancon_model = besancon.request_besancon('your@email.net',10.5,0.0) + >>> import astropy.io.ascii as asciitable + >>> from astropy.io.ascii import besancon as besancon_reader + >>> B = asciitable.read(besancon_model,Reader=asciitable.besancon.BesanconFixed,guess=False) + >>> B.pprint() + +A successful run should look something like this:: + >>> besancon_model = request_besancon('your@email.net',10.5,0.0) + + Waiting 30s for model to finish (elapsed wait time 30s, total 32) + Loading page... + Waiting 30s for model to finish (elapsed wait time 60s, total 198) + Loading page... + Waiting 30s for model to finish (elapsed wait time 90s, total 362) + Loading page... + Waiting 30s for model to finish (elapsed wait time 120s, total 456) + Loading page... + Downloaded 74752 of 11029183 bytes ( 0.68%) + + >>> B.pprint() + Dist Mv CL Typ LTef logg Age Mass J-K J-H V-K H-K K [Fe/H] l b Av Mbol + ----- ---- --- ---- ----- ---- --- ---- ----- ----- ------ ----- ------ ------ ----- ----- ----- ------ + 0.87 12.5 5 7.5 3.515 4.99 5 0.24 0.966 0.649 5.408 0.318 17.44 -0.02 10.62 -0.38 0.637 10.844 + 0.91 13.0 5 7.5 3.506 5.03 1 0.21 0.976 0.671 5.805 0.305 17.726 0.13 10.62 -0.38 0.669 11.312 + 0.97 18.5 5 7.9 3.39 5.32 7 0.08 0.804 0.634 8.634 0.17 20.518 -0.46 10.62 -0.38 0.716 0.0 + 1.01 15.2 5 7.9 3.465 5.14 1 0.13 1.045 0.649 7.015 0.396 18.957 0.22 10.62 -0.38 0.748 13.353 + 1.01 16.5 5 7.8 3.435 5.27 5 0.09 1.024 0.701 7.545 0.323 19.642 -0.09 10.62 -0.38 0.748 0.0 + 1.03 17.0 5 7.85 3.424 5.29 1 0.09 1.133 0.701 8.132 0.432 19.631 0.07 10.62 -0.38 0.764 0.0 + 1.09 13.5 5 7.6 3.497 5.05 7 0.18 0.995 0.69 5.829 0.305 18.629 -0.43 10.62 -0.38 0.812 11.78 + 1.17 13.7 5 7.65 3.493 5.0 2 0.17 1.025 0.68 6.319 0.345 18.552 0.2 10.62 -0.38 0.876 11.948 + 1.17 18.5 5 7.9 3.39 5.32 5 0.08 0.927 0.69 8.876 0.237 20.763 -0.27 10.62 -0.38 0.876 0.0 + 1.25 20.0 5 7.9 3.353 5.36 3 0.08 1.533 0.883 10.202 0.65 21.215 0.15 10.62 -0.38 0.941 0.0 + 1.29 9.3 5 7.1 3.58 4.74 2 0.56 0.997 0.777 4.592 0.219 16.263 0.21 10.62 -0.38 0.974 7.834 + 1.29 13.5 6 9.0 3.853 8.0 7 0.6 0.349 0.283 1.779 0.066 23.266 -0.17 10.62 -0.38 0.974 0.0 + 1.33 6.9 5 6.4 3.656 4.62 5 0.77 0.857 0.69 3.604 0.167 14.889 0.25 10.62 -0.38 1.006 5.795 + 1.35 7.5 5 6.5 3.633 4.62 5 0.7 0.902 0.729 3.885 0.172 15.22 0.08 10.62 -0.38 1.023 6.225 + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + 40.19 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.816 -1.99 10.62 -0.38 11.8 0.0 + 41.01 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.899 -2.14 10.62 -0.38 11.8 0.0 + 41.21 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.312 -1.36 10.62 -0.38 11.8 0.0 + 41.41 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.358 -0.92 10.62 -0.38 11.8 0.0 + 41.87 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.917 -1.79 10.62 -0.38 11.8 0.0 + 42.05 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.936 -2.06 10.62 -0.38 11.8 0.0 + 44.19 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.494 -3.04 10.62 -0.38 11.8 0.0 + 45.39 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.497 -1.28 10.62 -0.38 11.8 0.0 + 46.01 18.5 6 9.0 3.297 8.19 9 0.7 0.813 1.381 8.016 0.568 40.611 -2.01 10.62 -0.38 11.8 0.0 + 46.71 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 35.087 -2.59 10.62 -0.38 11.8 0.0 + 46.97 17.9 6 9.0 3.389 8.19 9 0.7 1.233 1.161 9.536 0.072 38.563 -1.42 10.62 -0.38 11.8 0.0 + 47.45 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.579 -2.25 10.62 -0.38 11.8 0.0 + 48.05 5.2 5 4.82 3.786 4.54 9 0.74 2.42 1.563 11.919 0.857 23.548 -1.45 10.62 -0.38 11.8 5.08 + 49.39 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.813 -1.19 10.62 -0.38 11.8 0.0 + + + +Reference/API +============= + +.. automodapi:: astrodata.besancon + :no-inheritance-diagram: From ddfcf1f9e5ea60a1c04700a79c37e98c2e9253ee Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:54 +0200 Subject: [PATCH 09/23] progressbar tweaks --- astrodata/utils/progressbar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index a913d13a7f..2a670db8f5 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,10 +6,10 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write("Downloaded %12d of %12d bytes (%6.2f%%)\r" % + sys.stdout.write(u"Downloaded %12d of %12d bytes (%6.2f%%)\r" % (bytes_so_far, total_size, percent)) else: - sys.stdout.write("Downloaded %10.2g Mb\r" % + sys.stdout.write(u"Downloaded %10.2g Mb\r" % (bytes_so_far / 1024.**2)) From 86aee81c98ca70514710c641393f6847157540ca Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 11:33:15 +0200 Subject: [PATCH 10/23] added progressbar code --- astrodata/utils/progressbar.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 astrodata/utils/progressbar.py diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py new file mode 100644 index 0000000000..c67d260831 --- /dev/null +++ b/astrodata/utils/progressbar.py @@ -0,0 +1,48 @@ +import urllib2, sys + +__all__ = ['chunk_report','chunk_read'] + +def chunk_report(bytes_so_far, chunk_size, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % + (bytes_so_far, total_size, percent)) + else: + sys.stdout.write("Downloaded %10.2g Mb\r" % + (bytes_so_far / 1024.**2)) + + +def chunk_read(response, chunk_size=1024, report_hook=None): + content_length = response.info().getheader('Content-Length') + if content_length is None: + total_size = 0 + else: + total_size = content_length.strip() + total_size = int(total_size) + + bytes_so_far = 0 + + result_string = "" + + #sys.stdout.write("Beginning download.\n") + + while 1: + chunk = response.read(chunk_size) + result_string += chunk + bytes_so_far += len(chunk) + + if not chunk: + if report_hook: + sys.stdout.write('\n') + break + + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + + return result_string + +if __name__ == '__main__': + response = urllib2.urlopen('http://www.ebay.com') + C = chunk_read(response, report_hook=chunk_report) + From 48d519d4c07c5eaeba357721e82fcd1787d52b76 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 21:40:34 +0200 Subject: [PATCH 11/23] progress bar tweaks --- astrodata/utils/progressbar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index c67d260831..a913d13a7f 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,7 +6,7 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % + sys.stdout.write("Downloaded %12d of %12d bytes (%6.2f%%)\r" % (bytes_so_far, total_size, percent)) else: sys.stdout.write("Downloaded %10.2g Mb\r" % From 99e75e562c3c5871672d7a3d661820e07af6da19 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:44:01 +0200 Subject: [PATCH 12/23] added the besancon query tool --- astrodata/besancon/besancon.py | 265 +++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 astrodata/besancon/besancon.py diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py new file mode 100644 index 0000000000..656360c7a1 --- /dev/null +++ b/astrodata/besancon/besancon.py @@ -0,0 +1,265 @@ +import urllib,urllib2 +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[20]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
      \s* + (
    1. ([a-zA-Z0-9 \s_-]*)
      2. \s*)*\s* +
    \s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} +mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), + 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude upper,lower limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Upper and Lower magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in (mag_limits): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s" % filename + + if retrieve_file: + return get_besancon_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except URLError: + sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return results + + From c84101f412fc6954464b1b98249467e20d20b75e Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:50:32 +0200 Subject: [PATCH 13/23] besancon documentation updates --- astrodata/besancon/besancon.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 656360c7a1..6ff69097d8 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -1,3 +1,11 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" import urllib,urllib2 import time import copy From a745ff662d0fb0381a5f3b1ef7ce781c8e27170b Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:54:42 +0200 Subject: [PATCH 14/23] added __init__ to besancon --- astrodata/besancon/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 astrodata/besancon/__init__.py diff --git a/astrodata/besancon/__init__.py b/astrodata/besancon/__init__.py new file mode 100644 index 0000000000..60c2920749 --- /dev/null +++ b/astrodata/besancon/__init__.py @@ -0,0 +1 @@ +from .besancon import * From 9a0065c6943194362817410c7db8e76655aacb86 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:27 +0200 Subject: [PATCH 15/23] documentation added, and it works (assuming you have the dependencies...) --- astrodata/besancon/besancon.py | 41 ++++++++++++------- docs/astrodata/besancon.rst | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 docs/astrodata/besancon.rst diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 6ff69097d8..a4fbb187ce 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -6,13 +6,17 @@ :Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) """ -import urllib,urllib2 +import urllib +import urllib2 +import socket import time import copy from astrodata.utils import progressbar import sys import re +__all__ = ['get_besancon_model_file','request_besancon'] + keyword_defaults = { 'rinf':0.000000, 'rsup':50.000000, @@ -34,7 +38,7 @@ 'ev':[""]*24, 'di':[""]*24, 'oo':[-7]+[-99]*12, - 'ff':[20]+[99]*12, + 'ff':[15]+[99]*12, 'spectyp_min':1, 'subspectyp_min': 0, 'spectyp_max':9, @@ -93,11 +97,12 @@ def parse_errors(text): try: errors = re.compile(r"""\s*
      \s* - (
    1. ([a-zA-Z0-9 \s_-]*)
      2. \s*)*\s* + (
    2. ([a-zA-Z0-9):( \s_-]*)
      3. \s*)*\s*
    \s* """, re.X) text = errors.search(text).group() except AttributeError: + likely_errors = text.split('\n')[132:150] raise ValueError("Regular expression matching to error message failed.") text_items = re.split("
  • |
    • |\n",errors.search(text).group()) text_items = [t for t in text_items if t != ""] @@ -105,13 +110,13 @@ def parse_errors(text): return error_list -colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} -mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), - 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} mag_order = "U","B","V","R","I","J","H","K","L" def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, - area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), mag_limits=copy.copy(mag_limits), colors_limits=copy.copy(colors_limits), retrieve_file=True, **kwargs): @@ -134,12 +139,12 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, area : float Area in square degrees absmag_limits : (float,float) - Absolute magnitude upper,lower limits + Absolute magnitude lower,upper limits colors_limits : dict of (float,float) Should contain 4 elements listing color differences in the valid bands, e.g.: {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} mag_limits = dict of (float,float) - Upper and Lower magnitude difference limits for each magnitude band + Lower and Upper magnitude difference limits for each magnitude band U B V R I J H K L clouds : list of 2-tuples Up to 25 line-of-sight clouds can be specified in pairs of (A_V, @@ -173,7 +178,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, kwd['oo'][0] = absmag_limits[0] kwd['ff'][0] = absmag_limits[1] - for ii,(key,val) in enumerate(colors_limits): + for ii,(key,val) in enumerate(colors_limits.items()): if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: kwd['colind'][ii] = key kwd['oo'][ii+9] = val[0] @@ -181,7 +186,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, else: raise ValueError('Invalid color %s' % key) - for (key,val) in (mag_limits): + for (key,val) in mag_limits.iteritems(): if key in mag_order: kwd['band0'][mag_order.index(key)] = val[0] kwd['bandf'][mag_order.index(key)] = val[1] @@ -213,7 +218,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, print "File is %s" % filename if retrieve_file: - return get_besancon_file(filename) + return get_besancon_model_file(filename) else: return filename @@ -244,6 +249,7 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov sys.stdout.write("\n") while 1: + sys.stdout.write(u"\r") try: U = urllib2.urlopen(url,timeout=5) if verbose: @@ -253,12 +259,18 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov else: results = page.read() break - except URLError: - sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) time.sleep(30) elapsed_time += 30 continue + if save: if savename is None: savename = filename @@ -270,4 +282,3 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov return results - diff --git a/docs/astrodata/besancon.rst b/docs/astrodata/besancon.rst new file mode 100644 index 0000000000..280826e249 --- /dev/null +++ b/docs/astrodata/besancon.rst @@ -0,0 +1,72 @@ +.. _astrodata.besancon: + +***************************************** +Besancon Queries (`astrodata.besancon`) +***************************************** + +Getting started +=============== + +The following example illustrates an Besancon catalog/image query. The API describes the +relevant keyword arguments (of which there are many) :: + + >>> from astrodata import besancon + >>> besancon_model = besancon.request_besancon('your@email.net',10.5,0.0) + >>> import astropy.io.ascii as asciitable + >>> from astropy.io.ascii import besancon as besancon_reader + >>> B = asciitable.read(besancon_model,Reader=asciitable.besancon.BesanconFixed,guess=False) + >>> B.pprint() + +A successful run should look something like this:: + >>> besancon_model = request_besancon('your@email.net',10.5,0.0) + + Waiting 30s for model to finish (elapsed wait time 30s, total 32) + Loading page... + Waiting 30s for model to finish (elapsed wait time 60s, total 198) + Loading page... + Waiting 30s for model to finish (elapsed wait time 90s, total 362) + Loading page... + Waiting 30s for model to finish (elapsed wait time 120s, total 456) + Loading page... + Downloaded 74752 of 11029183 bytes ( 0.68%) + + >>> B.pprint() + Dist Mv CL Typ LTef logg Age Mass J-K J-H V-K H-K K [Fe/H] l b Av Mbol + ----- ---- --- ---- ----- ---- --- ---- ----- ----- ------ ----- ------ ------ ----- ----- ----- ------ + 0.87 12.5 5 7.5 3.515 4.99 5 0.24 0.966 0.649 5.408 0.318 17.44 -0.02 10.62 -0.38 0.637 10.844 + 0.91 13.0 5 7.5 3.506 5.03 1 0.21 0.976 0.671 5.805 0.305 17.726 0.13 10.62 -0.38 0.669 11.312 + 0.97 18.5 5 7.9 3.39 5.32 7 0.08 0.804 0.634 8.634 0.17 20.518 -0.46 10.62 -0.38 0.716 0.0 + 1.01 15.2 5 7.9 3.465 5.14 1 0.13 1.045 0.649 7.015 0.396 18.957 0.22 10.62 -0.38 0.748 13.353 + 1.01 16.5 5 7.8 3.435 5.27 5 0.09 1.024 0.701 7.545 0.323 19.642 -0.09 10.62 -0.38 0.748 0.0 + 1.03 17.0 5 7.85 3.424 5.29 1 0.09 1.133 0.701 8.132 0.432 19.631 0.07 10.62 -0.38 0.764 0.0 + 1.09 13.5 5 7.6 3.497 5.05 7 0.18 0.995 0.69 5.829 0.305 18.629 -0.43 10.62 -0.38 0.812 11.78 + 1.17 13.7 5 7.65 3.493 5.0 2 0.17 1.025 0.68 6.319 0.345 18.552 0.2 10.62 -0.38 0.876 11.948 + 1.17 18.5 5 7.9 3.39 5.32 5 0.08 0.927 0.69 8.876 0.237 20.763 -0.27 10.62 -0.38 0.876 0.0 + 1.25 20.0 5 7.9 3.353 5.36 3 0.08 1.533 0.883 10.202 0.65 21.215 0.15 10.62 -0.38 0.941 0.0 + 1.29 9.3 5 7.1 3.58 4.74 2 0.56 0.997 0.777 4.592 0.219 16.263 0.21 10.62 -0.38 0.974 7.834 + 1.29 13.5 6 9.0 3.853 8.0 7 0.6 0.349 0.283 1.779 0.066 23.266 -0.17 10.62 -0.38 0.974 0.0 + 1.33 6.9 5 6.4 3.656 4.62 5 0.77 0.857 0.69 3.604 0.167 14.889 0.25 10.62 -0.38 1.006 5.795 + 1.35 7.5 5 6.5 3.633 4.62 5 0.7 0.902 0.729 3.885 0.172 15.22 0.08 10.62 -0.38 1.023 6.225 + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + 40.19 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.816 -1.99 10.62 -0.38 11.8 0.0 + 41.01 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.899 -2.14 10.62 -0.38 11.8 0.0 + 41.21 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.312 -1.36 10.62 -0.38 11.8 0.0 + 41.41 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.358 -0.92 10.62 -0.38 11.8 0.0 + 41.87 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.917 -1.79 10.62 -0.38 11.8 0.0 + 42.05 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.936 -2.06 10.62 -0.38 11.8 0.0 + 44.19 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.494 -3.04 10.62 -0.38 11.8 0.0 + 45.39 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.497 -1.28 10.62 -0.38 11.8 0.0 + 46.01 18.5 6 9.0 3.297 8.19 9 0.7 0.813 1.381 8.016 0.568 40.611 -2.01 10.62 -0.38 11.8 0.0 + 46.71 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 35.087 -2.59 10.62 -0.38 11.8 0.0 + 46.97 17.9 6 9.0 3.389 8.19 9 0.7 1.233 1.161 9.536 0.072 38.563 -1.42 10.62 -0.38 11.8 0.0 + 47.45 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.579 -2.25 10.62 -0.38 11.8 0.0 + 48.05 5.2 5 4.82 3.786 4.54 9 0.74 2.42 1.563 11.919 0.857 23.548 -1.45 10.62 -0.38 11.8 5.08 + 49.39 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.813 -1.19 10.62 -0.38 11.8 0.0 + + + +Reference/API +============= + +.. automodapi:: astrodata.besancon + :no-inheritance-diagram: From bc8f03def086db09bfee7895e487b41f93ecc851 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:54 +0200 Subject: [PATCH 16/23] progressbar tweaks --- astrodata/utils/progressbar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index a913d13a7f..2a670db8f5 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,10 +6,10 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write("Downloaded %12d of %12d bytes (%6.2f%%)\r" % + sys.stdout.write(u"Downloaded %12d of %12d bytes (%6.2f%%)\r" % (bytes_so_far, total_size, percent)) else: - sys.stdout.write("Downloaded %10.2g Mb\r" % + sys.stdout.write(u"Downloaded %10.2g Mb\r" % (bytes_so_far / 1024.**2)) From 0d5bc24d0c5644f977088c1ea2bfa2d2835f6cdf Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 30 Jun 2012 12:54:32 +0200 Subject: [PATCH 17/23] minor change to progressbar formatting --- astrodata/utils/progressbar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index 2a670db8f5..14f29c3388 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,8 +6,8 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write(u"Downloaded %12d of %12d bytes (%6.2f%%)\r" % - (bytes_so_far, total_size, percent)) + sys.stdout.write(u"Downloaded %12.2g of %12.2g Mb (%6.2f%%)\r" % + (bytes_so_far / 1024.**2, total_size / 1024.**2, percent)) else: sys.stdout.write(u"Downloaded %10.2g Mb\r" % (bytes_so_far / 1024.**2)) From 12797012a99acad3ae30588a4c88ccdd20cddf78 Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Fri, 6 Jul 2012 14:08:54 -0600 Subject: [PATCH 18/23] fixed FITS reading error (gzip'd fits look corrupted) --- astrodata/ukidss/ukidss.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/astrodata/ukidss/ukidss.py b/astrodata/ukidss/ukidss.py index 2a31dffc96..a71c68d151 100644 --- a/astrodata/ukidss/ukidss.py +++ b/astrodata/ukidss/ukidss.py @@ -196,7 +196,12 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', else: results = U.read() S = StringIO.StringIO(results) - fitsfile = pyfits.open(S,ignore_missing_end=True) + try: + fitsfile = pyfits.open(S,ignore_missing_end=True) + except IOError: + S.seek(0) + G = gzip.GzipFile(fileobj=S) + fitsfile = pyfits.open(G,ignore_missing_end=True) # Get Multiframe ID from the header images.append(fitsfile) From 856636b051eb742686debfce9a965017a9de70dd Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Mon, 9 Jul 2012 15:19:27 -0600 Subject: [PATCH 19/23] git diff claims there are no changes. --- astrodata/magpis/magpis.py | 8 ++++++-- astrodata/ukidss/ukidss.py | 17 ++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/astrodata/magpis/magpis.py b/astrodata/magpis/magpis.py index 0533271abc..517dff6f2f 100644 --- a/astrodata/magpis/magpis.py +++ b/astrodata/magpis/magpis.py @@ -40,7 +40,7 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, verbose=False, savename=None, save=True, - overwrite=False): + overwrite=False, directory='./'): """ Get an image at a specified glon/glat. Size can be specified WARNING: MAGPIS has a maxmimum image size of about 2048x2048 @@ -64,6 +64,8 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, Save FITS file? overwrite : bool Overwrite if file already exists? + directory : string + Directory to store file in. Defaults to './'. Examples -------- @@ -101,7 +103,9 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, if save: if savename is None: savename = "G%08.4f%+09.4f_%s.fits" % (glon,glat,survey) - fitsfile.writeto(savename, clobber=overwrite) + if directory[-1] != '/': + directory += '/' + fitsfile.writeto(directory+savename, clobber=overwrite) return fitsfile diff --git a/astrodata/ukidss/ukidss.py b/astrodata/ukidss/ukidss.py index a71c68d151..78952ca27a 100644 --- a/astrodata/ukidss/ukidss.py +++ b/astrodata/ukidss/ukidss.py @@ -186,7 +186,6 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', # Get image filename basename = os.path.basename( link.split("&")[0]).replace('.fit', '.fits.gz') - temp_file = tempfile.NamedTemporaryFile() # Get the file U = self.opener.open(link.replace("getImage", "getFImage")) @@ -212,19 +211,19 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', obj = filt + "_" + str(h0['OBJECT']).strip().replace(":", ".") if savename is None: - savename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj) + filename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj) + else: + filename = savename # Set final directory and file names - final_file = directory + '/' + savename + final_file = directory + '/' + filename - if not overwrite: - # Check that the final file doesn't already exist - if os.path.exists(final_file): - raise IOError("File exists : " + final_file) + if verbose: + print "Saving file %s" % final_file - shutil.copy(temp_file.name, final_file) + fitsfile.writeto(final_file, clobber=overwrite) - return fitsfile + return images def get_images_radius(self, ra, dec, radius, filter='all', frametype='stack', directory=None, n_concurrent=1, save=True, From 981d59c02b0f434bc003b4a37a0fa62941d7bbc2 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 11:33:15 +0200 Subject: [PATCH 20/23] added progressbar code (actually removed it) --- astrodata/utils/progressbar.py | 48 ---------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 astrodata/utils/progressbar.py diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py deleted file mode 100644 index 14f29c3388..0000000000 --- a/astrodata/utils/progressbar.py +++ /dev/null @@ -1,48 +0,0 @@ -import urllib2, sys - -__all__ = ['chunk_report','chunk_read'] - -def chunk_report(bytes_so_far, chunk_size, total_size): - if total_size > 0: - percent = float(bytes_so_far) / total_size - percent = round(percent*100, 2) - sys.stdout.write(u"Downloaded %12.2g of %12.2g Mb (%6.2f%%)\r" % - (bytes_so_far / 1024.**2, total_size / 1024.**2, percent)) - else: - sys.stdout.write(u"Downloaded %10.2g Mb\r" % - (bytes_so_far / 1024.**2)) - - -def chunk_read(response, chunk_size=1024, report_hook=None): - content_length = response.info().getheader('Content-Length') - if content_length is None: - total_size = 0 - else: - total_size = content_length.strip() - total_size = int(total_size) - - bytes_so_far = 0 - - result_string = "" - - #sys.stdout.write("Beginning download.\n") - - while 1: - chunk = response.read(chunk_size) - result_string += chunk - bytes_so_far += len(chunk) - - if not chunk: - if report_hook: - sys.stdout.write('\n') - break - - if report_hook: - report_hook(bytes_so_far, chunk_size, total_size) - - return result_string - -if __name__ == '__main__': - response = urllib2.urlopen('http://www.ebay.com') - C = chunk_read(response, report_hook=chunk_report) - From bfc061e6a47dbb7600f45fedbb12edc44b6ddd47 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:54 +0200 Subject: [PATCH 21/23] progressbar tweaks re-added progressbar --- astrodata/utils/progressbar.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 astrodata/utils/progressbar.py diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py new file mode 100644 index 0000000000..2a670db8f5 --- /dev/null +++ b/astrodata/utils/progressbar.py @@ -0,0 +1,48 @@ +import urllib2, sys + +__all__ = ['chunk_report','chunk_read'] + +def chunk_report(bytes_so_far, chunk_size, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write(u"Downloaded %12d of %12d bytes (%6.2f%%)\r" % + (bytes_so_far, total_size, percent)) + else: + sys.stdout.write(u"Downloaded %10.2g Mb\r" % + (bytes_so_far / 1024.**2)) + + +def chunk_read(response, chunk_size=1024, report_hook=None): + content_length = response.info().getheader('Content-Length') + if content_length is None: + total_size = 0 + else: + total_size = content_length.strip() + total_size = int(total_size) + + bytes_so_far = 0 + + result_string = "" + + #sys.stdout.write("Beginning download.\n") + + while 1: + chunk = response.read(chunk_size) + result_string += chunk + bytes_so_far += len(chunk) + + if not chunk: + if report_hook: + sys.stdout.write('\n') + break + + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + + return result_string + +if __name__ == '__main__': + response = urllib2.urlopen('http://www.ebay.com') + C = chunk_read(response, report_hook=chunk_report) + From 67984a822fc390e5d457309e5c4c06b7c06d941b Mon Sep 17 00:00:00 2001 From: Adam Date: Sat, 30 Jun 2012 12:54:32 +0200 Subject: [PATCH 22/23] minor change to progressbar formatting --- astrodata/utils/progressbar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index 2a670db8f5..14f29c3388 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,8 +6,8 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write(u"Downloaded %12d of %12d bytes (%6.2f%%)\r" % - (bytes_so_far, total_size, percent)) + sys.stdout.write(u"Downloaded %12.2g of %12.2g Mb (%6.2f%%)\r" % + (bytes_so_far / 1024.**2, total_size / 1024.**2, percent)) else: sys.stdout.write(u"Downloaded %10.2g Mb\r" % (bytes_so_far / 1024.**2)) From 4abdd962c460ead78e3e611d63eb0a7b3d1d3848 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 9 Jul 2012 15:40:10 -0600 Subject: [PATCH 23/23] Added progressbar --- astrodata/utils/__init__.py | 6 +++++ astrodata/utils/progressbar.py | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 astrodata/utils/__init__.py create mode 100644 astrodata/utils/progressbar.py diff --git a/astrodata/utils/__init__.py b/astrodata/utils/__init__.py new file mode 100644 index 0000000000..7e01721fcf --- /dev/null +++ b/astrodata/utils/__init__.py @@ -0,0 +1,6 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +# This sub-module is destined for common non-package specific utility +# functions that will ultimately be merged into `astropy.utils` + +from .progressbar import * diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py new file mode 100644 index 0000000000..14f29c3388 --- /dev/null +++ b/astrodata/utils/progressbar.py @@ -0,0 +1,48 @@ +import urllib2, sys + +__all__ = ['chunk_report','chunk_read'] + +def chunk_report(bytes_so_far, chunk_size, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write(u"Downloaded %12.2g of %12.2g Mb (%6.2f%%)\r" % + (bytes_so_far / 1024.**2, total_size / 1024.**2, percent)) + else: + sys.stdout.write(u"Downloaded %10.2g Mb\r" % + (bytes_so_far / 1024.**2)) + + +def chunk_read(response, chunk_size=1024, report_hook=None): + content_length = response.info().getheader('Content-Length') + if content_length is None: + total_size = 0 + else: + total_size = content_length.strip() + total_size = int(total_size) + + bytes_so_far = 0 + + result_string = "" + + #sys.stdout.write("Beginning download.\n") + + while 1: + chunk = response.read(chunk_size) + result_string += chunk + bytes_so_far += len(chunk) + + if not chunk: + if report_hook: + sys.stdout.write('\n') + break + + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + + return result_string + +if __name__ == '__main__': + response = urllib2.urlopen('http://www.ebay.com') + C = chunk_read(response, report_hook=chunk_report) +