From a76e3bdb9b59b377610f3a2c8c8c63c5110ca41c Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 11:33:15 +0200 Subject: [PATCH 01/14] added progressbar code --- astrodata/utils/progressbar.py | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 astrodata/utils/progressbar.py diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py new file mode 100644 index 0000000000..c67d260831 --- /dev/null +++ b/astrodata/utils/progressbar.py @@ -0,0 +1,48 @@ +import urllib2, sys + +__all__ = ['chunk_report','chunk_read'] + +def chunk_report(bytes_so_far, chunk_size, total_size): + if total_size > 0: + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % + (bytes_so_far, total_size, percent)) + else: + sys.stdout.write("Downloaded %10.2g Mb\r" % + (bytes_so_far / 1024.**2)) + + +def chunk_read(response, chunk_size=1024, report_hook=None): + content_length = response.info().getheader('Content-Length') + if content_length is None: + total_size = 0 + else: + total_size = content_length.strip() + total_size = int(total_size) + + bytes_so_far = 0 + + result_string = "" + + #sys.stdout.write("Beginning download.\n") + + while 1: + chunk = response.read(chunk_size) + result_string += chunk + bytes_so_far += len(chunk) + + if not chunk: + if report_hook: + sys.stdout.write('\n') + break + + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + + return result_string + +if __name__ == '__main__': + response = urllib2.urlopen('http://www.ebay.com') + C = chunk_read(response, report_hook=chunk_report) + From dff9f7e6cd65969ae7da5acf9fd303c6b6be1a39 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 27 Jun 2012 21:40:34 +0200 Subject: [PATCH 02/14] progress bar tweaks --- astrodata/utils/progressbar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index c67d260831..a913d13a7f 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,7 +6,7 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" % + sys.stdout.write("Downloaded %12d of %12d bytes (%6.2f%%)\r" % (bytes_so_far, total_size, percent)) else: sys.stdout.write("Downloaded %10.2g Mb\r" % From e536b3eeab3934da17e7796424b1c3246cfffaa7 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:44:01 +0200 Subject: [PATCH 03/14] added the besancon query tool --- astrodata/besancon/besancon.py | 265 +++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 astrodata/besancon/besancon.py diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py new file mode 100644 index 0000000000..656360c7a1 --- /dev/null +++ b/astrodata/besancon/besancon.py @@ -0,0 +1,265 @@ +import urllib,urllib2 +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[20]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
    \s* + (
  1. ([a-zA-Z0-9 \s_-]*)
    2. \s*)*\s* +
\s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} +mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), + 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude upper,lower limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Upper and Lower magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in (mag_limits): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s" % filename + + if retrieve_file: + return get_besancon_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except URLError: + sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return results + + From 78c951328f3adaa24765ac4458c4e974dea13c1f Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:50:32 +0200 Subject: [PATCH 04/14] besancon documentation updates --- astrodata/besancon/besancon.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 656360c7a1..6ff69097d8 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -1,3 +1,11 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" import urllib,urllib2 import time import copy From cb1e1e19939119e5670836cb85fdf1cdb5876dac Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:54:42 +0200 Subject: [PATCH 05/14] added __init__ to besancon --- astrodata/besancon/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 astrodata/besancon/__init__.py diff --git a/astrodata/besancon/__init__.py b/astrodata/besancon/__init__.py new file mode 100644 index 0000000000..60c2920749 --- /dev/null +++ b/astrodata/besancon/__init__.py @@ -0,0 +1 @@ +from .besancon import * From bdb15d723539e523b4299439b446a30b1c2820f6 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:27 +0200 Subject: [PATCH 06/14] documentation added, and it works (assuming you have the dependencies...) --- astrodata/besancon/besancon.py | 41 ++++++++++++------- docs/astrodata/besancon.rst | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 docs/astrodata/besancon.rst diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 6ff69097d8..a4fbb187ce 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -6,13 +6,17 @@ :Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) """ -import urllib,urllib2 +import urllib +import urllib2 +import socket import time import copy from astrodata.utils import progressbar import sys import re +__all__ = ['get_besancon_model_file','request_besancon'] + keyword_defaults = { 'rinf':0.000000, 'rsup':50.000000, @@ -34,7 +38,7 @@ 'ev':[""]*24, 'di':[""]*24, 'oo':[-7]+[-99]*12, - 'ff':[20]+[99]*12, + 'ff':[15]+[99]*12, 'spectyp_min':1, 'subspectyp_min': 0, 'spectyp_max':9, @@ -93,11 +97,12 @@ def parse_errors(text): try: errors = re.compile(r"""\s*
      \s* - (
    1. ([a-zA-Z0-9 \s_-]*)
      2. \s*)*\s* + (
    2. ([a-zA-Z0-9):( \s_-]*)
      3. \s*)*\s*
    \s* """, re.X) text = errors.search(text).group() except AttributeError: + likely_errors = text.split('\n')[132:150] raise ValueError("Regular expression matching to error message failed.") text_items = re.split("
  • |
    • |\n",errors.search(text).group()) text_items = [t for t in text_items if t != ""] @@ -105,13 +110,13 @@ def parse_errors(text): return error_list -colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} -mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), - 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} mag_order = "U","B","V","R","I","J","H","K","L" def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, - area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), mag_limits=copy.copy(mag_limits), colors_limits=copy.copy(colors_limits), retrieve_file=True, **kwargs): @@ -134,12 +139,12 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, area : float Area in square degrees absmag_limits : (float,float) - Absolute magnitude upper,lower limits + Absolute magnitude lower,upper limits colors_limits : dict of (float,float) Should contain 4 elements listing color differences in the valid bands, e.g.: {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} mag_limits = dict of (float,float) - Upper and Lower magnitude difference limits for each magnitude band + Lower and Upper magnitude difference limits for each magnitude band U B V R I J H K L clouds : list of 2-tuples Up to 25 line-of-sight clouds can be specified in pairs of (A_V, @@ -173,7 +178,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, kwd['oo'][0] = absmag_limits[0] kwd['ff'][0] = absmag_limits[1] - for ii,(key,val) in enumerate(colors_limits): + for ii,(key,val) in enumerate(colors_limits.items()): if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: kwd['colind'][ii] = key kwd['oo'][ii+9] = val[0] @@ -181,7 +186,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, else: raise ValueError('Invalid color %s' % key) - for (key,val) in (mag_limits): + for (key,val) in mag_limits.iteritems(): if key in mag_order: kwd['band0'][mag_order.index(key)] = val[0] kwd['bandf'][mag_order.index(key)] = val[1] @@ -213,7 +218,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, print "File is %s" % filename if retrieve_file: - return get_besancon_file(filename) + return get_besancon_model_file(filename) else: return filename @@ -244,6 +249,7 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov sys.stdout.write("\n") while 1: + sys.stdout.write(u"\r") try: U = urllib2.urlopen(url,timeout=5) if verbose: @@ -253,12 +259,18 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov else: results = page.read() break - except URLError: - sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) time.sleep(30) elapsed_time += 30 continue + if save: if savename is None: savename = filename @@ -270,4 +282,3 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov return results - diff --git a/docs/astrodata/besancon.rst b/docs/astrodata/besancon.rst new file mode 100644 index 0000000000..280826e249 --- /dev/null +++ b/docs/astrodata/besancon.rst @@ -0,0 +1,72 @@ +.. _astrodata.besancon: + +***************************************** +Besancon Queries (`astrodata.besancon`) +***************************************** + +Getting started +=============== + +The following example illustrates an Besancon catalog/image query. The API describes the +relevant keyword arguments (of which there are many) :: + + >>> from astrodata import besancon + >>> besancon_model = besancon.request_besancon('your@email.net',10.5,0.0) + >>> import astropy.io.ascii as asciitable + >>> from astropy.io.ascii import besancon as besancon_reader + >>> B = asciitable.read(besancon_model,Reader=asciitable.besancon.BesanconFixed,guess=False) + >>> B.pprint() + +A successful run should look something like this:: + >>> besancon_model = request_besancon('your@email.net',10.5,0.0) + + Waiting 30s for model to finish (elapsed wait time 30s, total 32) + Loading page... + Waiting 30s for model to finish (elapsed wait time 60s, total 198) + Loading page... + Waiting 30s for model to finish (elapsed wait time 90s, total 362) + Loading page... + Waiting 30s for model to finish (elapsed wait time 120s, total 456) + Loading page... + Downloaded 74752 of 11029183 bytes ( 0.68%) + + >>> B.pprint() + Dist Mv CL Typ LTef logg Age Mass J-K J-H V-K H-K K [Fe/H] l b Av Mbol + ----- ---- --- ---- ----- ---- --- ---- ----- ----- ------ ----- ------ ------ ----- ----- ----- ------ + 0.87 12.5 5 7.5 3.515 4.99 5 0.24 0.966 0.649 5.408 0.318 17.44 -0.02 10.62 -0.38 0.637 10.844 + 0.91 13.0 5 7.5 3.506 5.03 1 0.21 0.976 0.671 5.805 0.305 17.726 0.13 10.62 -0.38 0.669 11.312 + 0.97 18.5 5 7.9 3.39 5.32 7 0.08 0.804 0.634 8.634 0.17 20.518 -0.46 10.62 -0.38 0.716 0.0 + 1.01 15.2 5 7.9 3.465 5.14 1 0.13 1.045 0.649 7.015 0.396 18.957 0.22 10.62 -0.38 0.748 13.353 + 1.01 16.5 5 7.8 3.435 5.27 5 0.09 1.024 0.701 7.545 0.323 19.642 -0.09 10.62 -0.38 0.748 0.0 + 1.03 17.0 5 7.85 3.424 5.29 1 0.09 1.133 0.701 8.132 0.432 19.631 0.07 10.62 -0.38 0.764 0.0 + 1.09 13.5 5 7.6 3.497 5.05 7 0.18 0.995 0.69 5.829 0.305 18.629 -0.43 10.62 -0.38 0.812 11.78 + 1.17 13.7 5 7.65 3.493 5.0 2 0.17 1.025 0.68 6.319 0.345 18.552 0.2 10.62 -0.38 0.876 11.948 + 1.17 18.5 5 7.9 3.39 5.32 5 0.08 0.927 0.69 8.876 0.237 20.763 -0.27 10.62 -0.38 0.876 0.0 + 1.25 20.0 5 7.9 3.353 5.36 3 0.08 1.533 0.883 10.202 0.65 21.215 0.15 10.62 -0.38 0.941 0.0 + 1.29 9.3 5 7.1 3.58 4.74 2 0.56 0.997 0.777 4.592 0.219 16.263 0.21 10.62 -0.38 0.974 7.834 + 1.29 13.5 6 9.0 3.853 8.0 7 0.6 0.349 0.283 1.779 0.066 23.266 -0.17 10.62 -0.38 0.974 0.0 + 1.33 6.9 5 6.4 3.656 4.62 5 0.77 0.857 0.69 3.604 0.167 14.889 0.25 10.62 -0.38 1.006 5.795 + 1.35 7.5 5 6.5 3.633 4.62 5 0.7 0.902 0.729 3.885 0.172 15.22 0.08 10.62 -0.38 1.023 6.225 + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + 40.19 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.816 -1.99 10.62 -0.38 11.8 0.0 + 41.01 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.899 -2.14 10.62 -0.38 11.8 0.0 + 41.21 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.312 -1.36 10.62 -0.38 11.8 0.0 + 41.41 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.358 -0.92 10.62 -0.38 11.8 0.0 + 41.87 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.917 -1.79 10.62 -0.38 11.8 0.0 + 42.05 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.936 -2.06 10.62 -0.38 11.8 0.0 + 44.19 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.494 -3.04 10.62 -0.38 11.8 0.0 + 45.39 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.497 -1.28 10.62 -0.38 11.8 0.0 + 46.01 18.5 6 9.0 3.297 8.19 9 0.7 0.813 1.381 8.016 0.568 40.611 -2.01 10.62 -0.38 11.8 0.0 + 46.71 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 35.087 -2.59 10.62 -0.38 11.8 0.0 + 46.97 17.9 6 9.0 3.389 8.19 9 0.7 1.233 1.161 9.536 0.072 38.563 -1.42 10.62 -0.38 11.8 0.0 + 47.45 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.579 -2.25 10.62 -0.38 11.8 0.0 + 48.05 5.2 5 4.82 3.786 4.54 9 0.74 2.42 1.563 11.919 0.857 23.548 -1.45 10.62 -0.38 11.8 5.08 + 49.39 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.813 -1.19 10.62 -0.38 11.8 0.0 + + + +Reference/API +============= + +.. automodapi:: astrodata.besancon + :no-inheritance-diagram: From f4eb665d256fc6f9b9ff30184e33975c04b286ef Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:54 +0200 Subject: [PATCH 07/14] progressbar tweaks --- astrodata/utils/progressbar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astrodata/utils/progressbar.py b/astrodata/utils/progressbar.py index a913d13a7f..2a670db8f5 100644 --- a/astrodata/utils/progressbar.py +++ b/astrodata/utils/progressbar.py @@ -6,10 +6,10 @@ def chunk_report(bytes_so_far, chunk_size, total_size): if total_size > 0: percent = float(bytes_so_far) / total_size percent = round(percent*100, 2) - sys.stdout.write("Downloaded %12d of %12d bytes (%6.2f%%)\r" % + sys.stdout.write(u"Downloaded %12d of %12d bytes (%6.2f%%)\r" % (bytes_so_far, total_size, percent)) else: - sys.stdout.write("Downloaded %10.2g Mb\r" % + sys.stdout.write(u"Downloaded %10.2g Mb\r" % (bytes_so_far / 1024.**2)) From 944517ab8dd93534e042027e2c78f5dc17ec4ba1 Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Mon, 16 Jul 2012 13:46:09 -0600 Subject: [PATCH 08/14] added parser to besancon query tool --- astroquery/besancon/besancon.py | 307 ++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 astroquery/besancon/besancon.py diff --git a/astroquery/besancon/besancon.py b/astroquery/besancon/besancon.py new file mode 100644 index 0000000000..72ccef8f11 --- /dev/null +++ b/astroquery/besancon/besancon.py @@ -0,0 +1,307 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" +import urllib +import urllib2 +import socket +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +__all__ = ['get_besancon_model_file','request_besancon'] + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[15]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
      \s* + (
    1. ([a-zA-Z0-9):( \s_-]*)
      2. \s*)*\s* +
    \s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + likely_errors = text.split('\n')[132:150] + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude lower,upper limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Lower and Upper magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + Returns + ------- + Either the filename or the table depending on whether 'retrieve file' is + specified + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits.items()): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in mag_limits.iteritems(): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s and can be aquired from %s" % (filename, url_download+filename) + + if retrieve_file: + return get_besancon_model_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + sys.stdout.write(u"\r") + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return parse_besancon_model_string(results) + +def parse_besancon_model_string(bms,): + import astropy.io.ascii as asciireader + + besancon_table = asciireader.read(bms, Reader=asciireader.FixedWidth, + header_start=None, data_start=81, + names=bms.split('\n')[80].split(), + col_starts=(0,7,13,16,21,27,33,36,41,49,56,62,69,76,82,92,102,109), + col_ends=(6,12,15,20,26,32,35,39,48,55,61,68,75,81,91,101,108,115), + data_end=-7) + + for cn in besancon_table.columns: + if besancon_table[cn].dtype.kind in ('s','S'): + print "WARNING: The Besancon table did not parse properly. Some columns are likely to have invalid values and others incorrect values. Please report this error." + break + + return besancon_table + + From bfda0ac4af1af272d0d9f754295c76cf556301b3 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:44:01 +0200 Subject: [PATCH 09/14] added the besancon query tool --- astrodata/besancon/besancon.py | 265 +++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 astrodata/besancon/besancon.py diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py new file mode 100644 index 0000000000..656360c7a1 --- /dev/null +++ b/astrodata/besancon/besancon.py @@ -0,0 +1,265 @@ +import urllib,urllib2 +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[20]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
      \s* + (
    1. ([a-zA-Z0-9 \s_-]*)
      2. \s*)*\s* +
    \s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} +mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), + 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude upper,lower limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Upper and Lower magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in (mag_limits): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s" % filename + + if retrieve_file: + return get_besancon_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except URLError: + sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return results + + From baab1797c2464405ff6e3672dd473d75749dbdfc Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:50:32 +0200 Subject: [PATCH 10/14] besancon documentation updates --- astrodata/besancon/besancon.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 656360c7a1..6ff69097d8 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -1,3 +1,11 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" import urllib,urllib2 import time import copy From d38d1fa642bf1aa14973076e778885eaa44840ac Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Mon, 16 Jul 2012 13:46:09 -0600 Subject: [PATCH 11/14] added parser to besancon query tool --- astroquery/besancon/besancon.py | 307 ++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 astroquery/besancon/besancon.py diff --git a/astroquery/besancon/besancon.py b/astroquery/besancon/besancon.py new file mode 100644 index 0000000000..72ccef8f11 --- /dev/null +++ b/astroquery/besancon/besancon.py @@ -0,0 +1,307 @@ +""" +Besancon Query Tool +------------------- +A tool to query the Besancon model of the galaxy +http://model.obs-besancon.fr/ + +:Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) +""" +import urllib +import urllib2 +import socket +import time +import copy +from astrodata.utils import progressbar +import sys +import re + +__all__ = ['get_besancon_model_file','request_besancon'] + +keyword_defaults = { + 'rinf':0.000000, + 'rsup':50.000000, + 'dist_step_mode':0, + 'dlr': 0.000, + 'kleg':1, + 'longit': 10.62, + 'latit':-0.38, + 'soli':0.0003, # degrees. 0.00027777 = 1 arcmin + 'kleh':1, + 'eq1': 2000.0, + 'al0': 200.00, + 'alm': 200.00, + 'dl': 1.00, + 'ab0': 59.00, + 'abm': 59.00, + 'db': 1.00, + 'adif': 0.700, + 'ev':[""]*24, + 'di':[""]*24, + 'oo':[-7]+[-99]*12, + 'ff':[15]+[99]*12, + 'spectyp_min':1, + 'subspectyp_min': 0, + 'spectyp_max':9, + 'subspectyp_max': 5, + 'lumi[]':range(1,8), + 'sous_pop[]':range(1,11), + 'iband':8, + 'band0':[8]*9, + 'bandf':[25]*9, + 'colind':["J-H","H-K","J-K","V-K",], + 'nic': 4, + 'klea':1, + 'sc':[[0,0,0]]*9, + 'klee':0, + 'throughform':'ok', + 'kleb':3, + 'klec':1, + 'cinem':0, + 'outmod':"", + } + +url_download = "ftp://sasftp.obs-besancon.fr/modele/" +url_request = "http://model.obs-besancon.fr/modele_form.php" +# sample file: 1340900648.230224.resu +result_re = re.compile("[0-9]{10}\.[0-9]{6}\.resu") + +def parse_besancon_dict(bd): + """ + Turn a dict like default_keys into a list of tuples (must be a list of + tuples because there are some repeated entries, which dictionaries do not + support) + """ + + http_dict = [] + for key,val in bd.iteritems(): + if type(val) is list: + if "[]" in key: + for listval in val: + http_dict.append((key,listval)) + else: + for ii,listval in enumerate(val): + if type(listval) is list: + for jj,lv in enumerate(listval): + http_dict.append((key+"[%i][%i]" % (ii,jj),lv)) + else: + http_dict.append((key+"[%i]" % (ii) , listval)) + else: + http_dict.append((key , val)) + + return http_dict + +def parse_errors(text): + """ + Attempt to extract the errors from a Besancon web page with error messages in it + """ + try: + errors = re.compile(r"""\s* +
      \s* + (
    1. ([a-zA-Z0-9):( \s_-]*)
      2. \s*)*\s* +
    \s* + """, re.X) + text = errors.search(text).group() + except AttributeError: + likely_errors = text.split('\n')[132:150] + raise ValueError("Regular expression matching to error message failed.") + text_items = re.split("
  • |
    • |\n",errors.search(text).group()) + text_items = [t for t in text_items if t != ""] + error_list = text_items[2:-2] + return error_list + + +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} +mag_order = "U","B","V","R","I","J","H","K","L" + +def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), + mag_limits=copy.copy(mag_limits), + colors_limits=copy.copy(colors_limits), + retrieve_file=True, **kwargs): + """ + Perform a query on the Besancon model of the galaxy + http://model.obs-besancon.fr/ + + Parameters + ---------- + email : string + A valid e-mail address to send the report of completion to + glon : float + glat : float + Galactic latitude and longitude at the center + smallfield : bool + Small field (True) or Large Field (False) + LARGE FIELD NOT SUPPORTED YET + extinction : float + Extinction per kpc in A_V + area : float + Area in square degrees + absmag_limits : (float,float) + Absolute magnitude lower,upper limits + colors_limits : dict of (float,float) + Should contain 4 elements listing color differences in the valid bands, e.g.: + {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} + mag_limits = dict of (float,float) + Lower and Upper magnitude difference limits for each magnitude band + U B V R I J H K L + clouds : list of 2-tuples + Up to 25 line-of-sight clouds can be specified in pairs of (A_V, + distance in pc) + verbose : bool + Print out extra error messages? + retrieve_file : bool + If True, will try to retrieve the file every 30s until it shows up. + Otherwise, just returns the filename (the job is still executed on + the remote server, though) + kwargs : dict + Can override any argument in the request if you know the name of the + POST keyword. + + Returns + ------- + Either the filename or the table depending on whether 'retrieve file' is + specified + + """ + + # create a new keyword dict based on inputs + defaults + kwd = copy.copy(keyword_defaults) + for key,val in kwargs.iteritems(): + if key in keyword_defaults: + kwd[key] = val + elif verbose: + print "Skipped invalid key %s" % key + + kwd['kleg'] = 1 if smallfield else 2 + if not smallfield: + raise NotImplementedError + + kwd['adif'] = extinction + kwd['soli'] = area + kwd['oo'][0] = absmag_limits[0] + kwd['ff'][0] = absmag_limits[1] + + for ii,(key,val) in enumerate(colors_limits.items()): + if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: + kwd['colind'][ii] = key + kwd['oo'][ii+9] = val[0] + kwd['ff'][ii+9] = val[1] + else: + raise ValueError('Invalid color %s' % key) + + for (key,val) in mag_limits.iteritems(): + if key in mag_order: + kwd['band0'][mag_order.index(key)] = val[0] + kwd['bandf'][mag_order.index(key)] = val[1] + else: + raise ValueError('Invalid band %s' % key) + + if clouds is not None: + for ii,(AV,di) in enumerate(clouds): + kwd[ev][ii] = AV + kwd[di][ii] = di + + # parse the default dictionary + request = parse_besancon_dict(keyword_defaults) + + # an e-mail address is required + request.append(('email',email)) + request = urllib.urlencode(request) + # load the URL as text + U = urllib.urlopen(url_request, request) + # keep the text stored for possible later use + text = U.read() + try: + filename = result_re.search(text).group() + except AttributeError: # if there are no matches + errors = parse_errors(text) + raise ValueError("Errors: "+"\n".join(errors)) + + if verbose: + print "File is %s and can be aquired from %s" % (filename, url_download+filename) + + if retrieve_file: + return get_besancon_model_file(filename) + else: + return filename + +def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): + """ + Download a Besancon model from the website + + Parameters + ---------- + filename : string + The besancon filename, with format ##########.######.resu + verbose : bool + Print details about the download process + save : bool + Save the table after acquiring it? + savename : None or string + If not specified, defaults to the .resu table name + overwrite : bool + Overwrite the file if it exists? Defaults to True because the .resu + tables should have unique names by default, so there's little risk of + accidentally overwriting important information + """ + + url = url_download+filename + + elapsed_time = 0 + t0 = time.time() + + sys.stdout.write("\n") + while 1: + sys.stdout.write(u"\r") + try: + U = urllib2.urlopen(url,timeout=5) + if verbose: + print "" + print "Loading page..." + results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) + else: + results = page.read() + break + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + + + if save: + if savename is None: + savename = filename + if not overwrite and os.path.exists(savename): + raise IOError("File %s already exists." % savename) + outf = open(savename,'w') + print >>outf,results + outf.close() + + return parse_besancon_model_string(results) + +def parse_besancon_model_string(bms,): + import astropy.io.ascii as asciireader + + besancon_table = asciireader.read(bms, Reader=asciireader.FixedWidth, + header_start=None, data_start=81, + names=bms.split('\n')[80].split(), + col_starts=(0,7,13,16,21,27,33,36,41,49,56,62,69,76,82,92,102,109), + col_ends=(6,12,15,20,26,32,35,39,48,55,61,68,75,81,91,101,108,115), + data_end=-7) + + for cn in besancon_table.columns: + if besancon_table[cn].dtype.kind in ('s','S'): + print "WARNING: The Besancon table did not parse properly. Some columns are likely to have invalid values and others incorrect values. Please report this error." + break + + return besancon_table + + From 88249523986caf1b7bd1fa7636a87f705ff33a7d Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:50:32 +0200 Subject: [PATCH 12/14] besancon documentation updates From 5f23ad374687ab4a900f561a0b4a8d016a6dd19c Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 22:54:42 +0200 Subject: [PATCH 13/14] added __init__ to besancon --- astrodata/besancon/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 astrodata/besancon/__init__.py diff --git a/astrodata/besancon/__init__.py b/astrodata/besancon/__init__.py new file mode 100644 index 0000000000..60c2920749 --- /dev/null +++ b/astrodata/besancon/__init__.py @@ -0,0 +1 @@ +from .besancon import * From 0a8143333ad5820a34d8d484536f9c9202cd5fe4 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 28 Jun 2012 23:40:27 +0200 Subject: [PATCH 14/14] documentation added, and it works (assuming you have the dependencies...) --- astrodata/besancon/besancon.py | 41 ++++++++++++------- docs/astrodata/besancon.rst | 72 ++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 docs/astrodata/besancon.rst diff --git a/astrodata/besancon/besancon.py b/astrodata/besancon/besancon.py index 6ff69097d8..a4fbb187ce 100644 --- a/astrodata/besancon/besancon.py +++ b/astrodata/besancon/besancon.py @@ -6,13 +6,17 @@ :Author: Adam Ginsburg (adam.g.ginsburg@gmail.com) """ -import urllib,urllib2 +import urllib +import urllib2 +import socket import time import copy from astrodata.utils import progressbar import sys import re +__all__ = ['get_besancon_model_file','request_besancon'] + keyword_defaults = { 'rinf':0.000000, 'rsup':50.000000, @@ -34,7 +38,7 @@ 'ev':[""]*24, 'di':[""]*24, 'oo':[-7]+[-99]*12, - 'ff':[20]+[99]*12, + 'ff':[15]+[99]*12, 'spectyp_min':1, 'subspectyp_min': 0, 'spectyp_max':9, @@ -93,11 +97,12 @@ def parse_errors(text): try: errors = re.compile(r"""\s*
      \s* - (
    1. ([a-zA-Z0-9 \s_-]*)
      2. \s*)*\s* + (
    2. ([a-zA-Z0-9):( \s_-]*)
      3. \s*)*\s*
    \s* """, re.X) text = errors.search(text).group() except AttributeError: + likely_errors = text.split('\n')[132:150] raise ValueError("Regular expression matching to error message failed.") text_items = re.split("
  • |
    • |\n",errors.search(text).group()) text_items = [t for t in text_items if t != ""] @@ -105,13 +110,13 @@ def parse_errors(text): return error_list -colors_limits = {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} -mag_limits = {'U':(99,-99), 'B':(99,-99), 'V':(99,-99), 'R':(99,-99), - 'I':(99,-99), 'J':(99,-99), 'H':(99,-99), 'K':(99,-99), 'L':(99,-99)} +colors_limits = {"J-H":(-99,99),"H-K":(-99,99),"J-K":(-99,99),"V-K":(-99,99)} +mag_limits = {'U':(-99,99), 'B':(-99,99), 'V':(-5,20), 'R':(-99,99), + 'I':(-99,99), 'J':(-99,99), 'H':(-99,99), 'K':(-99,99), 'L':(-99,99)} mag_order = "U","B","V","R","I","J","H","K","L" def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, - area=0.0001, verbose=False, clouds=None, absmag_limits=(-7,20), + area=0.0001, verbose=True, clouds=None, absmag_limits=(-7,15), mag_limits=copy.copy(mag_limits), colors_limits=copy.copy(colors_limits), retrieve_file=True, **kwargs): @@ -134,12 +139,12 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, area : float Area in square degrees absmag_limits : (float,float) - Absolute magnitude upper,lower limits + Absolute magnitude lower,upper limits colors_limits : dict of (float,float) Should contain 4 elements listing color differences in the valid bands, e.g.: {"J-H":(99,-99),"H-K":(99,-99),"J-K":(99,-99),"V-K":(99,-99)} mag_limits = dict of (float,float) - Upper and Lower magnitude difference limits for each magnitude band + Lower and Upper magnitude difference limits for each magnitude band U B V R I J H K L clouds : list of 2-tuples Up to 25 line-of-sight clouds can be specified in pairs of (A_V, @@ -173,7 +178,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, kwd['oo'][0] = absmag_limits[0] kwd['ff'][0] = absmag_limits[1] - for ii,(key,val) in enumerate(colors_limits): + for ii,(key,val) in enumerate(colors_limits.items()): if key[0] in mag_order and key[1] == '-' and key[2] in mag_order: kwd['colind'][ii] = key kwd['oo'][ii+9] = val[0] @@ -181,7 +186,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, else: raise ValueError('Invalid color %s' % key) - for (key,val) in (mag_limits): + for (key,val) in mag_limits.iteritems(): if key in mag_order: kwd['band0'][mag_order.index(key)] = val[0] kwd['bandf'][mag_order.index(key)] = val[1] @@ -213,7 +218,7 @@ def request_besancon(email, glon, glat, smallfield=True, extinction=0.7, print "File is %s" % filename if retrieve_file: - return get_besancon_file(filename) + return get_besancon_model_file(filename) else: return filename @@ -244,6 +249,7 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov sys.stdout.write("\n") while 1: + sys.stdout.write(u"\r") try: U = urllib2.urlopen(url,timeout=5) if verbose: @@ -253,12 +259,18 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov else: results = page.read() break - except URLError: - sys.stdout.write("Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + except urllib2.URLError: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) + time.sleep(30) + elapsed_time += 30 + continue + except socket.timeout: + sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) time.sleep(30) elapsed_time += 30 continue + if save: if savename is None: savename = filename @@ -270,4 +282,3 @@ def get_besancon_model_file(filename, verbose=True, save=True, savename=None, ov return results - diff --git a/docs/astrodata/besancon.rst b/docs/astrodata/besancon.rst new file mode 100644 index 0000000000..280826e249 --- /dev/null +++ b/docs/astrodata/besancon.rst @@ -0,0 +1,72 @@ +.. _astrodata.besancon: + +***************************************** +Besancon Queries (`astrodata.besancon`) +***************************************** + +Getting started +=============== + +The following example illustrates an Besancon catalog/image query. The API describes the +relevant keyword arguments (of which there are many) :: + + >>> from astrodata import besancon + >>> besancon_model = besancon.request_besancon('your@email.net',10.5,0.0) + >>> import astropy.io.ascii as asciitable + >>> from astropy.io.ascii import besancon as besancon_reader + >>> B = asciitable.read(besancon_model,Reader=asciitable.besancon.BesanconFixed,guess=False) + >>> B.pprint() + +A successful run should look something like this:: + >>> besancon_model = request_besancon('your@email.net',10.5,0.0) + + Waiting 30s for model to finish (elapsed wait time 30s, total 32) + Loading page... + Waiting 30s for model to finish (elapsed wait time 60s, total 198) + Loading page... + Waiting 30s for model to finish (elapsed wait time 90s, total 362) + Loading page... + Waiting 30s for model to finish (elapsed wait time 120s, total 456) + Loading page... + Downloaded 74752 of 11029183 bytes ( 0.68%) + + >>> B.pprint() + Dist Mv CL Typ LTef logg Age Mass J-K J-H V-K H-K K [Fe/H] l b Av Mbol + ----- ---- --- ---- ----- ---- --- ---- ----- ----- ------ ----- ------ ------ ----- ----- ----- ------ + 0.87 12.5 5 7.5 3.515 4.99 5 0.24 0.966 0.649 5.408 0.318 17.44 -0.02 10.62 -0.38 0.637 10.844 + 0.91 13.0 5 7.5 3.506 5.03 1 0.21 0.976 0.671 5.805 0.305 17.726 0.13 10.62 -0.38 0.669 11.312 + 0.97 18.5 5 7.9 3.39 5.32 7 0.08 0.804 0.634 8.634 0.17 20.518 -0.46 10.62 -0.38 0.716 0.0 + 1.01 15.2 5 7.9 3.465 5.14 1 0.13 1.045 0.649 7.015 0.396 18.957 0.22 10.62 -0.38 0.748 13.353 + 1.01 16.5 5 7.8 3.435 5.27 5 0.09 1.024 0.701 7.545 0.323 19.642 -0.09 10.62 -0.38 0.748 0.0 + 1.03 17.0 5 7.85 3.424 5.29 1 0.09 1.133 0.701 8.132 0.432 19.631 0.07 10.62 -0.38 0.764 0.0 + 1.09 13.5 5 7.6 3.497 5.05 7 0.18 0.995 0.69 5.829 0.305 18.629 -0.43 10.62 -0.38 0.812 11.78 + 1.17 13.7 5 7.65 3.493 5.0 2 0.17 1.025 0.68 6.319 0.345 18.552 0.2 10.62 -0.38 0.876 11.948 + 1.17 18.5 5 7.9 3.39 5.32 5 0.08 0.927 0.69 8.876 0.237 20.763 -0.27 10.62 -0.38 0.876 0.0 + 1.25 20.0 5 7.9 3.353 5.36 3 0.08 1.533 0.883 10.202 0.65 21.215 0.15 10.62 -0.38 0.941 0.0 + 1.29 9.3 5 7.1 3.58 4.74 2 0.56 0.997 0.777 4.592 0.219 16.263 0.21 10.62 -0.38 0.974 7.834 + 1.29 13.5 6 9.0 3.853 8.0 7 0.6 0.349 0.283 1.779 0.066 23.266 -0.17 10.62 -0.38 0.974 0.0 + 1.33 6.9 5 6.4 3.656 4.62 5 0.77 0.857 0.69 3.604 0.167 14.889 0.25 10.62 -0.38 1.006 5.795 + 1.35 7.5 5 6.5 3.633 4.62 5 0.7 0.902 0.729 3.885 0.172 15.22 0.08 10.62 -0.38 1.023 6.225 + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + 40.19 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.816 -1.99 10.62 -0.38 11.8 0.0 + 41.01 17.1 6 9.0 3.515 8.19 9 0.7 2.013 1.481 11.166 0.532 35.899 -2.14 10.62 -0.38 11.8 0.0 + 41.21 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.312 -1.36 10.62 -0.38 11.8 0.0 + 41.41 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.358 -0.92 10.62 -0.38 11.8 0.0 + 41.87 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.917 -1.79 10.62 -0.38 11.8 0.0 + 42.05 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 34.936 -2.06 10.62 -0.38 11.8 0.0 + 44.19 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.494 -3.04 10.62 -0.38 11.8 0.0 + 45.39 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.497 -1.28 10.62 -0.38 11.8 0.0 + 46.01 18.5 6 9.0 3.297 8.19 9 0.7 0.813 1.381 8.016 0.568 40.611 -2.01 10.62 -0.38 11.8 0.0 + 46.71 16.7 6 9.0 3.568 8.19 9 0.7 1.783 1.141 11.686 0.642 35.087 -2.59 10.62 -0.38 11.8 0.0 + 46.97 17.9 6 9.0 3.389 8.19 9 0.7 1.233 1.161 9.536 0.072 38.563 -1.42 10.62 -0.38 11.8 0.0 + 47.45 17.3 6 9.0 3.485 8.19 9 0.7 1.933 1.471 10.826 0.462 36.579 -2.25 10.62 -0.38 11.8 0.0 + 48.05 5.2 5 4.82 3.786 4.54 9 0.74 2.42 1.563 11.919 0.857 23.548 -1.45 10.62 -0.38 11.8 5.08 + 49.39 16.9 6 9.0 3.542 8.19 9 0.7 1.893 1.301 11.436 0.592 35.813 -1.19 10.62 -0.38 11.8 0.0 + + + +Reference/API +============= + +.. automodapi:: astrodata.besancon + :no-inheritance-diagram: