diff --git a/astrodata/besancon/__init__.py b/astrodata/besancon/__init__.py new file mode 100644 index 0000000000..60c2920749 --- /dev/null +++ b/astrodata/besancon/__init__.py @@ -0,0 +1 @@ +from .besancon import * diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py new file mode 100644 index 0000000000..a4fbb187ce --- /dev/null +++ b/astrodata/besancon/besancon.py @@ -0,0 +1,284 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" +import urllib +import urllib2 +import socket +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +__all__ = ['get_besancon_model_file','request_besancon'] + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[15]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
    \s* + (
  1. ([a-zA-Z0-9):( \s_-]*)
    2. \s*)*\s* +
\s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + likely_errors = text.split('\n')[132:150] + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude lower,upper limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Lower and Upper magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits.items()): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in mag_limits.iteritems(): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s" % filename + + if retrieve_file: + return get_besancon_model_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + sys.stdout.write(u"\r") + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return results + diff --git a/astrodata/utils/__init__.py b/astrodata/utils/__init__.py new file mode 100644 index 0000000000..7e01721fcf --- /dev/null +++ b/astrodata/utils/__init__.py @@ -0,0 +1,6 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +# This sub-module is destined for common non-package specific utility +# functions that will ultimately be merged into `astropy.utils` + +from .progressbar import * diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py new file mode 100644 index 0000000000..14f29c3388 --- /dev/null +++ b/astrodata/utils/progressbar.py @@ -0,0 +1,48 @@ +import urllib2, sys + +__all__ = ['chunk_report','chunk_read'] + +def chunk_report(bytes_so_far, chunk_size, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write(u"Downloaded %12.2g of %12.2g Mb (%6.2f%%)\r" % + (bytes_so_far / 1024.**2, total_size / 1024.**2, percent)) + else: + sys.stdout.write(u"Downloaded %10.2g Mb\r" % + (bytes_so_far / 1024.**2)) + + +def chunk_read(response, chunk_size=1024, report_hook=None): + content_length = response.info().getheader('Content-Length') + if content_length is None: + total_size = 0 + else: + total_size = content_length.strip() + total_size = int(total_size) + + bytes_so_far = 0 + + result_string = "" + + #sys.stdout.write("Beginning download.\n") + + while 1: + chunk = response.read(chunk_size) + result_string += chunk + bytes_so_far += len(chunk) + + if not chunk: + if report_hook: + sys.stdout.write('\n') + break + + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + + return result_string + +if __name__ == '__main__': + response = urllib2.urlopen('http://www.ebay.com') + C = chunk_read(response, report_hook=chunk_report) + diff --git a/astroquery/magpis/magpis.py b/astroquery/magpis/magpis.py index 45dd263941..550e8020d6 100644 --- a/astroquery/magpis/magpis.py +++ b/astroquery/magpis/magpis.py @@ -40,7 +40,7 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, verbose=False, savename=None, save=True, - overwrite=False): + overwrite=False, directory='./'): """ Get an image at a specified glon/glat. Size can be specified WARNING: MAGPIS has a maxmimum image size of about 2048x2048 @@ -64,6 +64,8 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, Save FITS file? overwrite : bool Overwrite if file already exists? + directory : string + Directory to store file in. Defaults to './'. Examples -------- @@ -101,7 +103,9 @@ def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, if save: if savename is None: savename = "G%08.4f%+09.4f_%s.fits" % (glon,glat,survey) - fitsfile.writeto(savename, clobber=overwrite) + if directory[-1] != '/': + directory += '/' + fitsfile.writeto(directory+savename, clobber=overwrite) return fitsfile diff --git a/astroquery/ukidss/ukidss.py b/astroquery/ukidss/ukidss.py index 07c3623584..95387148c8 100644 --- a/astroquery/ukidss/ukidss.py +++ b/astroquery/ukidss/ukidss.py @@ -186,7 +186,6 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', # Get image filename basename = os.path.basename( link.split("&")[0]).replace('.fit', '.fits.gz') - temp_file = tempfile.NamedTemporaryFile() # Get the file U = self.opener.open(link.replace("getImage", "getFImage")) @@ -196,7 +195,12 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', else: results = U.read() S = StringIO.StringIO(results) - fitsfile = pyfits.open(S,ignore_missing_end=True) + try: + fitsfile = pyfits.open(S,ignore_missing_end=True) + except IOError: + S.seek(0) + G = gzip.GzipFile(fileobj=S) + fitsfile = pyfits.open(G,ignore_missing_end=True) # Get Multiframe ID from the header images.append(fitsfile) @@ -207,19 +211,19 @@ def get_image_gal(self, glon, glat, filter='all', frametype='stack', obj = filt + "_" + str(h0['OBJECT']).strip().replace(":", ".") if savename is None: - savename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj) + filename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj) + else: + filename = savename # Set final directory and file names - final_file = directory + '/' + savename + final_file = directory + '/' + filename - if not overwrite: - # Check that the final file doesn't already exist - if os.path.exists(final_file): - raise IOError("File exists : " + final_file) + if verbose: + print "Saving file %s" % final_file - shutil.copy(temp_file.name, final_file) + fitsfile.writeto(final_file, clobber=overwrite) - return fitsfile + return images def get_images_radius(self, ra, dec, radius, filter='all', frametype='stack', directory=None, n_concurrent=1, save=True, diff --git a/docs/astrodata/besancon.rst b/docs/astrodata/besancon.rst new file mode 100644 index 0000000000..280826e249 --- /dev/null +++ b/docs/astrodata/besancon.rst @@ -0,0 +1,72 @@ +.. _astrodata.besancon: + +***************************************** +Besancon Queries (`astrodata.besancon`) +***************************************** + +Getting started +=============== + +The following example illustrates an Besancon catalog/image query. The API describes the +relevant keyword arguments (of which there are many) :: + + >>> from astrodata import besancon + >>> besancon_model = besancon.request_besancon('your@email.net',10.5,0.0) + >>> import astropy.io.ascii as asciitable + >>> from astropy.io.ascii import besancon as besancon_reader + >>> B = asciitable.read(besancon_model,Reader=asciitable.besancon.BesanconFixed,guess=False) + >>> B.pprint() + +A successful run should look something like this:: + >>> besancon_model = request_besancon('your@email.net',10.5,0.0) + + Waiting 30s for model to finish (elapsed wait time 30s, total 32) + Loading page... + Waiting 30s for model to finish (elapsed wait time 60s, total 198) + Loading page... + Waiting 30s for model to finish (elapsed wait time 90s, total 362) + Loading page... + Waiting 30s for model to finish (elapsed wait time 120s, total 456) + Loading page... + Downloaded 74752 of 11029183 bytes ( 0.68%) + + >>> B.pprint() + Dist Mv CL Typ LTef logg Age Mass J-K J-H V-K H-K K [Fe/H] l b Av Mbol + ----- ---- --- ---- ----- ---- --- ---- ----- ----- ------ ----- ------ ------ ----- ----- ----- ------ + 0.87 12.5 5 7.5 3.515 4.99 5 0.24 0.966 0.649 5.408 0.318 17.44 -0.02 10.62 -0.38 0.637 10.844 + 0.91 13.0 5 7.5 3.506 5.03 1 0.21 0.976 0.671 5.805 0.305 17.726 0.13 10.62 -0.38 0.669 11.312 + 0.97 18.5 5 7.9 3.39 5.32 7 0.08 0.804 0.634 8.634 0.17 20.518 -0.46 10.62 -0.38 0.716 0.0 + 1.01 15.2 5 7.9 3.465 5.14 1 0.13 1.045 0.649 7.015 0.396 18.957 0.22 10.62 -0.38 0.748 13.353 + 1.01 16.5 5 7.8 3.435 5.27 5 0.09 1.024 0.701 7.545 0.323 19.642 -0.09 10.62 -0.38 0.748 0.0 + 1.03 17.0 5 7.85 3.424 5.29 1 0.09 1.133 0.701 8.132 0.432 19.631 0.07 10.62 -0.38 0.764 0.0 + 1.09 13.5 5 7.6 3.497 5.05 7 0.18 0.995 0.69 5.829 0.305 18.629 -0.43 10.62 -0.38 0.812 11.78 + 1.17 13.7 5 7.65 3.493 5.0 2 0.17 1.025 0.68 6.319 0.345 18.552 0.2 10.62 -0.38 0.876 11.948 + 1.17 18.5 5 7.9 3.39 5.32 5 0.08 0.927 0.69 8.876 0.237 20.763 -0.27 10.62 -0.38 0.876 0.0 + 1.25 20.0 5 7.9 3.353 5.36 3 0.08 1.533 0.883 10.202 0.65 21.215 0.15 10.62 -0.38 0.941 0.0 + 1.29 9.3 5 7.1 3.58 4.74 2 0.56 0.997 0.777 4.592 0.219 16.263 0.21 10.62 -0.38 0.974 7.834 + 1.29 13.5 6 9.0 3.853 8.0 7 0.6 0.349 0.283 1.779 0.066 23.266 -0.17 10.62 -0.38 0.974 0.0 + 1.33 6.9 5 6.4 3.656 4.62 5 0.77 0.857 0.69 3.604 0.167 14.889 0.25 10.62 -0.38 1.006 5.795 + 1.35 7.5 5 6.5 3.633 4.62 5 0.7 0.902 0.729 3.885 0.172 15.22 0.08 10.62 -0.38 1.023 6.225 + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + 40.19 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.816 -1.99 10.62 -0.38 11.8 0.0 + 41.01 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.899 -2.14 10.62 -0.38 11.8 0.0 + 41.21 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.312 -1.36 10.62 -0.38 11.8 0.0 + 41.41 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.358 -0.92 10.62 -0.38 11.8 0.0 + 41.87 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.917 -1.79 10.62 -0.38 11.8 0.0 + 42.05 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.936 -2.06 10.62 -0.38 11.8 0.0 + 44.19 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.494 -3.04 10.62 -0.38 11.8 0.0 + 45.39 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.497 -1.28 10.62 -0.38 11.8 0.0 + 46.01 18.5 6 9.0 3.297 8.19 9 0.7 0.813 1.381 8.016 0.568 40.611 -2.01 10.62 -0.38 11.8 0.0 + 46.71 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 35.087 -2.59 10.62 -0.38 11.8 0.0 + 46.97 17.9 6 9.0 3.389 8.19 9 0.7 1.233 1.161 9.536 0.072 38.563 -1.42 10.62 -0.38 11.8 0.0 + 47.45 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.579 -2.25 10.62 -0.38 11.8 0.0 + 48.05 5.2 5 4.82 3.786 4.54 9 0.74 2.42 1.563 11.919 0.857 23.548 -1.45 10.62 -0.38 11.8 5.08 + 49.39 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.813 -1.19 10.62 -0.38 11.8 0.0 + + + +Reference/API +============= + +.. automodapi:: astrodata.besancon + :no-inheritance-diagram: