From 036fb086664b636d5bf9e5ce02e133ed11f9d66c Mon Sep 17 00:00:00 2001 From: Alexander Urban Date: Sat, 7 Jul 2018 13:52:14 -0500 Subject: [PATCH 01/73] Correcting a typo in the year of Chatterji's thesis --- gwdetchar/omega/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gwdetchar/omega/__init__.py b/gwdetchar/omega/__init__.py index 2a7dfc472..ba4fb3d37 100755 --- a/gwdetchar/omega/__init__.py +++ b/gwdetchar/omega/__init__.py @@ -18,7 +18,7 @@ """Methods and utilties for performing Omega pipline scans -See Chatterji 20015 [thesis] for details on the Q-pipeline. +See Chatterji 2005 [thesis] for details on the Q-pipeline. """ import os From e993d44a6dfedb067909631637008a4831f78442 Mon Sep 17 00:00:00 2001 From: Alexander Urban Date: Tue, 10 Jul 2018 11:01:33 -0500 Subject: [PATCH 02/73] Adding a new executable, gwdetchar-omega, and supporting submodules under gwdetchar.omega. The gwdetchar-omega tool is invoked very similarly to wdq. --- bin/gwdetchar-omega | 195 ++++++++++ gwdetchar/omega/config.py | 87 +++++ gwdetchar/omega/html.py | 796 ++++++++++++++++++++++++++++++++++++++ gwdetchar/omega/plot.py | 44 +++ 4 files changed, 1122 insertions(+) create mode 100644 bin/gwdetchar-omega create mode 100644 gwdetchar/omega/config.py create mode 100644 gwdetchar/omega/html.py create mode 100644 gwdetchar/omega/plot.py diff --git a/bin/gwdetchar-omega b/bin/gwdetchar-omega new file mode 100644 index 000000000..b80ee1595 --- /dev/null +++ b/bin/gwdetchar-omega @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright (C) LIGO Scientific Collaboration (2015-) +# +# This file is part of the GW DetChar python package. +# +# GW DetChar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GW DetChar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GW DetChar. If not, see . + +"""Compute omega scans for a list of channels around the given GPS time(s). +""" + +from __future__ import division + +import os.path +import re +import sys +import warnings +from StringIO import StringIO + +import numpy + +from matplotlib import (use, rcParams) +use('agg') + +from glue.lal import Cache + +from gwpy.utils import gprint +from gwpy.time import tconvert +from gwpy.table import EventTable +from gwpy.timeseries import TimeSeriesDict +from gwpy.detector import (Channel, ChannelList) +from gwpy.plotter import (figure, TimeSeriesPlot) +from gwpy.segments import (DataQualityFlag, DataQualityDict, + Segment, SegmentList) + +from gwdetchar import (cli, const, omega, __version__) +from gwdetchar.omega import (config, html) + +try: + from LDAStools import frameCPP +except ImportError: + io_kw = {'format': 'gwf'} +else: + io_kw = {'type': 'adc', 'format': 'gwf.framecpp'} + +__author__ = 'Alex Urban ' +__credits__ = 'Duncan Macleod ' + + +# -- parse command line ------------------------------------------------------- + +parser = cli.create_parser(description=__doc__) +parser.add_argument('ifo', type=str, help='IFO prefix for this analysis') +parser.add_argument('gpstime', type=str, help='GPS time of scan') +parser.add_argument('-o', '--output-directory', + help='output directory for the omega scan, ' + 'default: ~/public_html/wdq/{IFO}_{gpstime}') +parser.add_argument('-f', '--config-file', action='append', + help='path to configuration file to use, can be given multiple times ' + '(files read in order), default: ' + '~detchar/etc/omega/{epoch}/{OBS}-{IFO}_R-selected.txt') +parser.add_argument('-c', '--cache-file', + help='path to a LAL format data cache file; if not given, data locations ' + 'are found using NDS') +parser.add_argument('--condor', action='store_true', default=False, + help='indicates this job is running under condor, ' + 'only use when running as part of a workflow') +parser.add_argument('--colormap', default='viridis', + help='name of colormap to use, default: %(default)s') + +args = parser.parse_args() + +print("----------------------------------------------\n" + "Creating %s omega scan at GPS second %s..." % (args.ifo, args.gpstime)) + +gps = args.gpstime +gpstime = float(gps) +ifo = args.ifo +obs = ifo[0] + +# update rcParams +rcParams.update({ + 'axes.labelsize': 20, + 'figure.subplot.bottom': 0.17, + 'figure.subplot.left': 0.1, + 'figure.subplot.right': 0.9, + 'figure.subplot.top': 0.90, + 'grid.color': 'gray', + 'image.cmap': args.colormap, + 'svg.fonttype': 'none', +}) + +# set output directory +outdir = args.output_directory +if outdir is None: + outdir = os.path.expanduser('~/public_html/wdq/%s_%s' % (ifo, gps)) +if not os.path.isdir(outdir): + os.makedirs(outdir) +os.chdir(outdir) +print("Output directory created as %s" % outdir) + +# parse configuration file +cp = config.OmegaConfigParser(ifo=ifo) +cp.read(args.config_file) + +# prepare html variables +htmlv = { + 'title': '%s Qscan | %s' % (ifo, gpstime), + 'config': args.config_file, +} + +# -- FIXME: Eventually move these classes to gwdetchar.omega ------------------ + +class OmegaChannel(Channel): + def __init__(self, channelname, section, **params): + self.name = channelname + frametype = params.get('frametype', None) + frequency_range = tuple( + [float(s) for s in params.get('frequency-range', None).split(',')] + ) + super(OmegaChannel, self).__init__(channelname, frametype=frametype, + frequency_range=frequency_range) + self.plots = [ + html.FancyPlot('plots/%s-%s.png' % (self.name[3::], s)) + for s in ['short', 'med', 'long'] + ] + self.section = section + self.params = params.copy() + +class OmegaChannelList(object): + def __init__(self, **params): + self.name = params.get('name', None) + self.key = self.name.lower().replace(' ', '-') + self.resample = int(params.get('resample', None)) + self.frametype = params.get('frametype', None) + chans = params.get('channels', None).split(',\n') + self.channels = [OmegaChannel(c, self.name, **params) for c in chans] + self.params = params.copy() + + +# -- Compute Qscan ------------------------------------------------------------ + +gprint('Computing Q-scans...') + +# make subdirectories +plotdir = 'plots' +aboutdir = 'about' +for d in [plotdir, aboutdir]: + if not os.path.isdir(d): + os.makedirs(d) + +# determine channel blocks +blocks = [OmegaChannelList(**cp[s]) for s in cp.sections()] + +# range over blocks +for block in blocks: + gprint('Processing block %s' % block.name) + chans = [c.name for c in block.channels] + # read in 32 seconds of data + # centered on gpstime + data = TimeSeriesDict.get(chans, start=gpstime-16, end=gpstime+16, + frametype=block.frametype, nproc=8) + # compute qscans + for c in block.channels: + series = data[c.name].resample(block.resample) + qscan = series.q_transform(qrange=(4, 96)) + for span, png in zip([.5, 2, 8], c.plots): + plot = qscan.crop(gpstime-span, gpstime+span).plot(figsize=[7.5, 6.5]) + ax = plot.gca() + ax.set_epoch(gpstime) + ax.set_xlabel('Time [seconds]') + ax.set_yscale('log') + ax.grid(True, axis='y', which='both') + plot.add_colorbar(cmap=args.colormap, clim=(0, 25), + label='Normalized energy') + plot.savefig(str(png)) + + +# -- Prepare HTML ------------------------------------------------------------- + +# write HTML page and finish +gprint('Preparing HTML at %s/index.html...' % outdir) +html.write_qscan_page(ifo, gpstime, blocks, **htmlv) +gprint("-- index.html written, all done --") diff --git a/gwdetchar/omega/config.py b/gwdetchar/omega/config.py new file mode 100644 index 000000000..4c50aed9c --- /dev/null +++ b/gwdetchar/omega/config.py @@ -0,0 +1,87 @@ +# coding=utf-8 +# Copyright (C) Duncan Macleod (2015) +# +# This file is part of the GW DetChar python package. +# +# GW DetChar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GW DetChar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GW DetChar. If not, see . + +""" +Configuration files for Omega Scans +################################### + +How to write a configuration file +================================= +""" + +try: + import configparser +except ImportError: # python 2.x + import ConfigParser as configparser + +__author__ = 'Alex Urban ' +__credits__ = 'Duncan Macleod ' + +OMEGA_DEFAULTS = {} + +class OmegaConfigParser(configparser.ConfigParser): + def __init__(self, ifo=None, defaults=dict(), **kwargs): + if ifo is not None: + defaults.setdefault('IFO', ifo) + configparser.ConfigParser.__init__(self, defaults=defaults, **kwargs) + self.set_omega_defaults() + + def set_omega_defaults(self): + for section in OMEGA_DEFAULTS: + self.add_section(section) + for key, val in OMEGA_DEFAULTS[section].iteritems(): + if key.endswith('channels') and isinstance(val, (tuple, list)): + self.set(section, key, '\n'.join(list(val))) + elif isinstance(val, tuple): + self.set(section, key, ', '.join(map(str, val))) + else: + self.set(section, key, str(val)) + + def read(self, filenames): + readok = configparser.ConfigParser.read(self, filenames) + for f in filenames: + if f not in readok: + raise IOError("Cannot read file %r" % f) + return readok + read.__doc__ = configparser.ConfigParser.read.__doc__ + + def getfloats(self, section, option): + return self._get(section, comma_separated_floats, option) + + def getparams(self, section, prefix): + nchar = len(prefix) + params = dict((key[nchar:], val) for (key, val) in + self.items(section) if key.startswith(prefix)) + # try simple typecasting + for key in params: + if params[key].lower() in ('true', 'false'): + params[key] = bool(params[key]) + if key == 'frequency-range': + params[key] = tuple([float(s) for s in params[key].split(',')]) + if key == 'channels': + params[key] = params[key].split(',\n') + else: + try: + params[key] = float(params[key]) + except ValueError: + pass + return params + + +def comma_separated_floats(string): + return map(float, string.split(',')) diff --git a/gwdetchar/omega/html.py b/gwdetchar/omega/html.py new file mode 100644 index 000000000..759157975 --- /dev/null +++ b/gwdetchar/omega/html.py @@ -0,0 +1,796 @@ +# coding=utf-8 +# Copyright (C) Duncan Macleod (2015) +# +# This file is part of the GW DetChar python package. +# +# GW DetChar is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GW DetChar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GW DetChar. If not, see . + +"""Utilties for writing Omega scan HTML pages +""" + +from __future__ import division + +import os +import sys +import datetime +import subprocess +from functools import wraps +from getpass import getuser +from pytz import timezone + +from glue import markup +from gwpy.time import tconvert +from ..io.html import (JQUERY_JS, BOOTSTRAP_CSS, BOOTSTRAP_JS) +from .. import __version__ + +__author__ = 'Alex Urban ' +__credit__ = 'Duncan Macleod ' + +# -- give context for ifo names + +OBSERVATORY_MAP = { + 'G1': 'GEO', + 'H1': 'LIGO Hanford', + 'K1': 'KAGRA', + 'L1': 'LIGO Livingston', + 'V1': 'Virgo' +} + +OBSERVATORY_ZONE = { + 'G1': 'Europe/Berlin', + 'H1': 'US/Pacific', + 'K1': 'Japan', + 'L1': 'US/Central', + 'V1': 'Europe/Rome' +} + +# -- set up default JS and CSS files + +FANCYBOX_CSS = ( + "//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/jquery.fancybox.min.css") +FANCYBOX_JS = ( + "//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.5/jquery.fancybox.min.js") + +FONT_LATO_CSS = ( + "//fonts.googleapis.com/css?family=Lato:300,700" +) + +CSS_FILES = [BOOTSTRAP_CSS, FANCYBOX_CSS, FONT_LATO_CSS] +JS_FILES = [JQUERY_JS, BOOTSTRAP_JS, FANCYBOX_JS] + +OMEGA_CSS = """ +html { + position: relative; + min-height: 100%; +} +body { + margin-bottom: 120px; + font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; + -webkit-font-smoothing: antialiased; +} +.footer { + position: absolute; + bottom: 0; + width: 100%; + height: 100px; + background-color: #f5f5f5; + padding-top: 20px; + padding-bottom: 20px; +} +.fancybox-skin { + background: white; +} +""" # nopep8 + +OMEGA_JS = """ +$(document).ready(function() { + $(\".fancybox\").fancybox({ + nextEffect: 'none', + prevEffect: 'none', + helpers: {title: {type: 'inside'}} + }); +}); +""" # nopep8 + + +# -- Plot construction -------------------------------------------------------- + +class FancyPlot(object): + """A helpful class of objects that coalesce image links and caption text + for fancybox figures. + + Parameters + ---------- + img : `str` or `FancyPlot` + either a filename (including relative or absolute path) or another + FancyPlot instance + caption : `str` + the text to be displayed in a fancybox as this figure's caption + """ + def __init__(self, img, caption=None): + if isinstance(img, FancyPlot): + caption = caption if caption else img.caption + self.img = str(img) + self.caption = caption if caption else os.path.basename(self.img) + + def __str__(self): + return self.img + + +# -- HTML construction -------------------------------------------------------- + +def write_static_files(static): + """Write static CSS and javascript files into the given directory + + Parameters + ---------- + static : `str` + the target directory for the static files, will be created if + necessary + + Returns + ------- + `/omega.css` + `/omega.js` + the paths of the two files written by this method, which will be + `omega.css` and `omega.js` inside `static` + + Notes + ----- + This method modifies the module-level variables ``CSS_FILES`` and + ``JS_FILES`` to guarantee that the static files are actually only + written once per process. + """ + if not os.path.isdir(static): + os.makedirs(static) + omegacss = os.path.join(static, 'omega.css') + if omegacss not in CSS_FILES: + with open(omegacss, 'w') as f: + f.write(OMEGA_CSS) + CSS_FILES.append(omegacss) + omegajs = os.path.join(static, 'omega.js') + if omegajs not in JS_FILES: + with open(omegajs, 'w') as f: + f.write(OMEGA_JS) + JS_FILES.append(omegajs) + return omegacss, omegajs + + +def init_page(ifo, gpstime, css=[], script=[], base=os.path.curdir, + **kwargs): + """Initialise a new `markup.page` + This method constructs an HTML page with the following structure + .. code-block:: html + + + + + +
+
+

IFO

+

GPSTIME

+
+
+
+ + Parameters + ---------- + ifo : `str` + the interferometer prefix + gpstime : `float` + the central GPS time of the analysis + css : `list`, optional + the list of stylesheets to link in the `` + script : `list`, optional + the list of javascript files to link in the `` + base : `str`, optional, default '.' + the path for the `` tag to link in the `` + + Returns + ------- + page : `markup.page` + the structured markup to open an HTML document + """ + # write CSS to static dir + staticdir = os.path.join(os.path.curdir, 'static') + write_static_files(staticdir) + # create page + page = markup.page() + page.header.append('') + page.html(lang='en') + page.head() + page.base(href=base) + page._full = True + # link stylesheets (appending bootstrap if needed) + css = css[:] + for cssf in CSS_FILES[::-1]: + b = os.path.basename(cssf) + if not any(f.endswith(b) for f in css): + css.insert(0, cssf) + for f in css: + page.link(href=f, rel='stylesheet', type='text/css', media='all') + # link javascript + script = script[:] + for jsf in JS_FILES[::-1]: + b = os.path.basename(jsf) + if not any(f.endswith(b) for f in script): + script.insert(0, jsf) + for f in script: + page.script('', src=f, type='text/javascript') + # add other attributes + for key in kwargs: + getattr(page, key)(kwargs[key]) + # finalize header + page.head.close() + page.body() + # write banner + page.div(class_='container') + page.div(class_='page-header', role='banner') + page.h1("%s Omega Scan" % ifo) + page.h2("%s" % gpstime) + page.div.close() + page.div.close() # container + + # open container + page.div(class_='container') + return page + + +def close_page(page, target, about=None, date=None): + """Close an HTML document with markup then write to disk + This method writes the closing markup to complement the opening + written by `init_page`, something like: + .. code-block:: html +
+
+ +
+ + + + Parameters + ---------- + page : `markup.page` + the markup object to close + target : `str` + the output filename for HTML + about : `str`, optional + the path of the 'about' page to link in the footer + date : `datetime.datetime`, optional + the timestamp to place in the footer, defaults to + `~datetime.datetime.now` + """ + page.div.close() # container + page.add(str(write_footer(about=about, date=date))) + if not page._full: + page.body.close() + page.html.close() + with open(target, 'w') as f: + f.write(page()) + return page + + +def wrap_html(func): + """Decorator to wrap a function with `init_page` and `close_page` calls + This allows inner HTML methods to be written with minimal arguments + and content, hopefully making things simpler + """ + @wraps(func) + def decorated_func(ifo, gpstime, *args, **kwargs): + # set page init args + initargs = { + 'title': '%s Qscan | %s' % (ifo, gpstime), + 'base': os.path.curdir, + } + for key in ['title', 'base']: + if key in kwargs: + initargs[key] = kwargs.pop(key) + # find outdir + outdir = kwargs.pop('outdir', initargs['base']) + if not os.path.isdir(outdir): + os.makedirs(outdir) + # write about page + try: + config = kwargs.pop('config') + except KeyError: + about = None + else: + iargs = initargs.copy() + aboutdir = os.path.join(outdir, 'about') + if iargs['base'] == os.path.curdir: + iargs['base'] = os.path.pardir + about = write_about_page(ifo, gpstime, config, outdir=aboutdir, + **iargs) + if os.path.basename(about) == 'index.html': + about = about[:-10] + # open page + page = init_page(ifo, gpstime, **initargs) + # write analysis summary + # (but only on the main results page) + if about: + page.add(write_summary(ifo, gpstime)) + # write content + contentf = os.path.join(outdir, '_inner.html') + with open(contentf, 'w') as f: + f.write(str(func(*args, **kwargs))) + # embed content + page.div('', id_='content') + page.script("$('#content').load('%s');" % contentf) + # close page + index = os.path.join(outdir, 'index.html') + close_page(page, index, about=about) + return index + return decorated_func + +# -- Utilities ---------------------------------------------------------------- + + +def html_link(href, txt, target="_blank", **params): + """Write an HTML tag + + Parameters + ---------- + href : `str` + the URL to point to + txt : `str` + the text for the link + target : `str`, optional + the ``target`` of this link + **params + other HTML parameters for the ```` tag + + Returns + ------- + html : `str` + """ + if target is not None: + params.setdefault('target', target) + return markup.oneliner.a(txt, href=href, **params) + + +def cis_link(channel, **params): + """Write a channel name as a link to the Channel Information System + + Parameters + ---------- + channel : `str` + the name of the channel to link + **params + other HTML parmeters for the ```` tag + + Returns + ------- + html : `str` + """ + kwargs = { + 'title': "CIS entry for %s" % channel, + 'style': "font-family: Monaco, \"Courier New\", monospace;", + } + kwargs.update(params) + return html_link("https://cis.ligo.org/channel/byname/%s" % channel, + channel, **kwargs) + + +def fancybox_img(img, linkparams=dict(), **params): + """Return the markup to embed an in HTML + + Parameters + ---------- + img : `FancyPlot` + a `FancyPlot` object containing the path of the image to embed + and its caption to be displayed + linkparams : `dict` + the HTML attributes for the ```` tag + **params + the HTML attributes for the ```` tag + + Returns + ------- + html : `str` + Notes + ----- + See `~gwdetchar.omega.plot.FancyPlot` for more about the `FancyPlot` class. + """ + page = markup.page() + aparams = { + 'title': img.caption, + 'class_': 'fancybox', + 'target': '_blank', + 'data-fancybox-group': 'qscan-image', + } + aparams.update(linkparams) + img = str(img) + page.a(href=img, **aparams) + imgparams = { + 'alt': os.path.basename(img), + 'class_': 'img-responsive', + } + imgparams['src'] = img + imgparams.update(params) + page.img(**imgparams) + page.a.close() + return str(page) + + +def scaffold_plots(plots, nperrow=3): + """Embed a `list` of images in a bootstrap scaffold + + Parameters + ---------- + plot : `list` of `FancyPlot` + the list of image paths to embed + nperrow : `int` + the number of images to place in a row (on a desktop screen) + + Returns + ------- + page : `~glue.markup.page` + the markup object containing the scaffolded HTML + """ + page = markup.page() + x = int(12//nperrow) + # scaffold plots + for i, p in enumerate(plots): + if i % nperrow == 0: + page.div(class_='row', style="width:95%;") + page.div(class_='col-sm-%d' % x) + page.add(fancybox_img(p)) + page.div.close() # col + if i % nperrow == nperrow - 1: + page.div.close() # row + if i % nperrow < nperrow-1: + page.div.close() # row + return page() + + +def write_footer(about=None, date=None): + """Write a