From 495f98196c26540efcd5e6dd8154066747ab38dc Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 26 Mar 2018 15:51:21 +0100 Subject: [PATCH 1/6] Added support for displaying FITS data --- .travis.yml | 2 +- CHANGES.rst | 2 + appveyor.yml | 2 +- pywwt/data_server.py | 88 ++++++++++++++++++++++++ pywwt/nbextension/static/wwt_json_api.js | 4 ++ pywwt/qt.py | 23 ++++++- setup.py | 4 +- 7 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 pywwt/data_server.py diff --git a/.travis.yml b/.travis.yml index 3240ea16..cdbfc574 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: global: - SETUP_XVFB=True - CONDA_CHANNELS="astrofrog/label/dev" - - CONDA_DEPENDENCIES="astropy qtpy traitlets ipywidgets>=7.0 ipyevents widgetsnbextension pyqt pytest requests nomkl matplotlib beautifulsoup4 lxml jupyterlab" + - CONDA_DEPENDENCIES="astropy qtpy traitlets ipywidgets>=7.0 ipyevents widgetsnbextension pyqt pytest requests nomkl matplotlib beautifulsoup4 lxml jupyterlab flask flask-cors" - PIP_DEPENDENCIES="sphinx-automodapi numpydoc sphinx_rtd_theme pytest-faulthandler" matrix: - PYTHON_VERSION=2.7 diff --git a/CHANGES.rst b/CHANGES.rst index dbeb654d..2042aa1f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,8 @@ 0.4.0 (unreleased) ------------------ +- Added ``WWTQtClient.load_fits_data`` method. [#78] + - Fix compatibility with Jupyter Lab. [#63, #65] - Added GUI controls for imagery layers. [#64] diff --git a/appveyor.yml b/appveyor.yml index da66f2db..a064dd08 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,7 +12,7 @@ environment: # to the matrix section. CONDA_CHANNELS: "astrofrog/label/dev conda-forge" - CONDA_DEPENDENCIES: "astropy qtpy traitlets ipywidgets>=7.0 ipyevents widgetsnbextension pyqt pytest requests matplotlib" + CONDA_DEPENDENCIES: "astropy qtpy traitlets ipywidgets>=7.0 ipyevents widgetsnbextension pyqt pytest requests matplotlib flask flask-cors" PIP_DEPENDENCIES: "pytest-faulthandler" matrix: diff --git a/pywwt/data_server.py b/pywwt/data_server.py new file mode 100644 index 00000000..fcfb0017 --- /dev/null +++ b/pywwt/data_server.py @@ -0,0 +1,88 @@ +import os +import time +import socket +import logging +from hashlib import md5 +from threading import Thread + +__all__ = ['get_data_server'] + + +def get_data_server(verbose=True): + """ + This starts up a flask server and returns a handle to a DataServer + object which can be used to register files to serve. + """ + + from flask import Flask + from flask_cors import CORS + + class FlaskWrapper(Flask): + + port = None + host = None + + def run(self, *args, **kwargs): + self.host = kwargs.get('host', None) + self.port = kwargs.get('port', None) + try: + super(FlaskWrapper, self).run(*args, **kwargs) + finally: + self.host = None + self.port = None + + app = FlaskWrapper('DataServer') + CORS(app) + if verbose: + log = logging.getLogger('werkzeug') + log.setLevel(logging.ERROR) + + class DataServer(object): + + def __init__(self): + self._files = {} + self._thread = Thread(target=self.start_app) + self._thread.daemon = True + self._thread.start() + self._app = app + while self._app.host is None and self._app.port is None: + time.sleep(0.1) + + @property + def port(self): + return self._app.port + + @property + def host(self): + return self._app.host + + def start_app(self): + host = socket.gethostbyname('localhost') + for port in range(8000, 9000): + try: + return app.run(host=host, port=port) + except Exception: + pass + raise Exception("Could not start up data server") + + def serve_file(self, filename, real_name=True): + with open(filename, 'rb') as f: + content = f.read() + if real_name: + hash = os.path.basename(filename) + else: + hash = md5(content).hexdigest() + '.fits' + self._files[hash] = os.path.abspath(filename) + return 'http://' + self.host + ':' + str(self.port) + '/data/' + hash + + def get_file_contents(self, hash): + with open(self._files[hash], 'rb') as f: + return f.read() + + ds = DataServer() + + @app.route("/data/") + def data(hash): + return ds.get_file_contents(hash) + + return ds diff --git a/pywwt/nbextension/static/wwt_json_api.js b/pywwt/nbextension/static/wwt_json_api.js index 9fd4648b..8ccd527f 100644 --- a/pywwt/nbextension/static/wwt_json_api.js +++ b/pywwt/nbextension/static/wwt_json_api.js @@ -55,6 +55,10 @@ function wwt_apply_json_message(wwt, msg) { wwt.gotoRaDecZoom(msg['ra'], msg['dec'], msg['fov'], msg['instant']); break; + case 'load_fits': + wwt.loadFits(msg['url']); + break; + case 'setting_set': var name = msg['setting']; wwt.settings["set_" + name](msg['value']); diff --git a/pywwt/qt.py b/pywwt/qt.py index dd789fb1..f2f34539 100644 --- a/pywwt/qt.py +++ b/pywwt/qt.py @@ -13,6 +13,7 @@ from .core import BaseWWTWidget from .logger import logger +from .data_server import get_data_server __all__ = ['WWTQtClient'] @@ -96,7 +97,7 @@ def _check_ready(self): class WWTQtWidget(QtWidgets.QWidget): - def __init__(self, parent=None): + def __init__(self, url=None, parent=None): super(WWTQtWidget, self).__init__(parent=parent) @@ -104,7 +105,7 @@ def __init__(self, parent=None): self.page = WWTQWebEnginePage() self.page.setView(self.web) self.web.setPage(self.page) - self.web.setHtml(WWT_HTML) + self.web.setUrl(QtCore.QUrl(url)) layout = QtWidgets.QVBoxLayout() layout.setContentsMargins(0, 0, 0, 0) @@ -162,7 +163,11 @@ def __init__(self, block_until_ready=False, size=None): if app is None: app = QtWidgets.QApplication(['']) - self.widget = WWTQtWidget() + self._data_server = get_data_server() + self._data_server.serve_file(WWT_JSON_FILE, real_name=True) + wwt_url = self._data_server.serve_file(WWT_HTML_FILE, real_name=True) + + self.widget = WWTQtWidget(url=wwt_url) if size is not None: self.widget.resize(*size) self.widget.show() @@ -186,6 +191,18 @@ def _send_msg(self, async=True, **kwargs): msg = json.dumps(kwargs) return self.widget._run_js("wwt_apply_json_message(wwt, {0});".format(msg), async=async) + def load_fits_data(self, filename): + """ + Load a FITS file. + + Paramters + --------- + filename : str + The filename of the FITS file to display. + """ + url = self._data_server.serve_file(filename) + self._send_msg(event='load_fits', url=url) + def render(self, filename): """ Saves a screenshot of the viewer's current state. diff --git a/setup.py b/setup.py index 7dc28766..c1adb56a 100755 --- a/setup.py +++ b/setup.py @@ -102,7 +102,9 @@ 'ipywidgets>=7.0.0', 'ipyevents', 'traitlets', - 'qtpy' + 'qtpy', + 'flask', + 'flask-cors' ], extras_require = { 'test': [ From befe4211099af339a0a839e6eb593a2185620ecc Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 6 Apr 2018 12:17:05 +0100 Subject: [PATCH 2/6] Initial implementation of load_fits_data for Jupyter --- pywwt/jupyter.py | 14 ++++++++ pywwt/jupyter_server.py | 75 ++++++++++++++++++++++++++++++++++------- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/pywwt/jupyter.py b/pywwt/jupyter.py index 5dcf5111..09bed053 100644 --- a/pywwt/jupyter.py +++ b/pywwt/jupyter.py @@ -13,6 +13,7 @@ from ipyevents import Event as DOMListener from .core import BaseWWTWidget +from .jupyter_server import serve_file __all__ = ['WWTJupyterWidget'] @@ -49,6 +50,19 @@ def _default_layout(self): def _send_msg(self, **kwargs): self.send(kwargs) + def load_fits_data(self, filename): + """ + Load a FITS file. + + Paramters + --------- + filename : str + The filename of the FITS file to display. + """ + url = serve_file(filename, extension='.fits') + print("GET", url) + self._send_msg(event='load_fits', url=url) + @property def layer_controls(self): if self._controls is None: diff --git a/pywwt/jupyter_server.py b/pywwt/jupyter_server.py index 75ae94a6..739ff6a6 100644 --- a/pywwt/jupyter_server.py +++ b/pywwt/jupyter_server.py @@ -1,4 +1,7 @@ import os +import json +from hashlib import md5 +from tornado import web from notebook.utils import url_path_join from notebook.base.handlers import IPythonHandler @@ -6,29 +9,77 @@ STATIC_DIR = os.path.join(os.path.dirname(__file__), 'nbextension', 'static') +CONFIG = os.path.expanduser('~/.pywwt') -class WWTHTMLHandler(IPythonHandler): - def get(self): - with open(os.path.join(STATIC_DIR, 'wwt.html')) as f: - content = f.read() - self.finish(content) +class WWTFileHandler(IPythonHandler): + + def get(self, filename): + + with open('/tmp/test.log', 'w') as f: + f.write(filename) + + filename = os.path.basename(filename) + # First we check if this is a standard file in the static directory + if os.path.exists(os.path.join(STATIC_DIR, filename)): + path = os.path.join(STATIC_DIR, filename) + else: + # If not, we open the config file which should contain a JSON + # dictionary with filenames and paths. + if not os.path.exists(CONFIG): + raise web.HTTPError(404) + with open(CONFIG) as f: + config = json.load(f) + with open('/tmp/test2.log', 'w') as f: + f.write(str(repr(config['paths'])) + " " + str(filename in config['paths'])) + if filename in config['paths']: + path = config['paths'][filename] + else: + raise web.HTTPError(404) -class WWTJSHandler(IPythonHandler): - def get(self): - with open(os.path.join(STATIC_DIR, 'wwt_json_api.js')) as f: + with open(path, 'rb') as f: content = f.read() + self.finish(content) +def serve_file(path, extension=''): + + with open('/tmp/in2.log', 'w') as f: + f.write(str(os.getppid())) + + if not os.path.exists(path): + raise ValueError("Path {0} does not exist".format(path)) + + hash = md5(path.encode('utf-8')).hexdigest() + extension + + with open(CONFIG) as f: + config = json.load(f) + + if hash not in config['paths']: + + config['paths'][hash] = os.path.abspath(path) + + with open(CONFIG, 'w') as f: + json.dump(config, f) + + return url_path_join(config['base_url'], hash) + + def load_jupyter_server_extension(nb_server_app): web_app = nb_server_app.web_app host_pattern = '.*$' - route_pattern = url_path_join(web_app.settings['base_url'], '/wwt.html') - web_app.add_handlers(host_pattern, [(route_pattern, WWTHTMLHandler)]) + # FIXME: the current solution relies on a single file, and won't support + # concurrent connections. + + config = {} + config['paths'] = {} + config['base_url'] = url_path_join(web_app.settings['base_url'], '/wwt') + with open(CONFIG, 'w') as f: + json.dump(config, f) - route_pattern = url_path_join(web_app.settings['base_url'], '/wwt_json_api.js') - web_app.add_handlers(host_pattern, [(route_pattern, WWTJSHandler)]) + route_pattern = url_path_join(web_app.settings['base_url'], '/wwt/(.*)') + web_app.add_handlers(host_pattern, [(route_pattern, WWTFileHandler)]) From ec5d250293e2ff64305f7a0d08cce24eaa2d8fe7 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 6 Apr 2018 12:37:53 +0100 Subject: [PATCH 3/6] Tidy up load_fits_data --- pywwt/core.py | 22 ++++++++++++++++------ pywwt/data_server.py | 4 ++-- pywwt/jupyter.py | 2 +- pywwt/qt.py | 3 ++- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pywwt/core.py b/pywwt/core.py index fcb9fd1e..d224ee2b 100644 --- a/pywwt/core.py +++ b/pywwt/core.py @@ -1,3 +1,4 @@ +import os from traitlets import HasTraits, observe, validate, TraitError from astropy import units as u from astropy.time import Time @@ -266,8 +267,8 @@ def _validate_scale(self, proposal): def track_object(self, obj): """ - Focus the viewer on a particular object while in solar system mode. - Available objects include the Sun, the planets, the Moon, Jupiter's + Focus the viewer on a particular object while in solar system mode. + Available objects include the Sun, the planets, the Moon, Jupiter's Galilean moons, and Pluto. Parameters @@ -308,7 +309,7 @@ def set_view(self, mode): 'panorama'] ss_levels = ['solar_system', 'milky_way', 'universe'] ss_mode = '3D Solar System View' - + if mode in available: self._send_msg(event='set_viewer_mode', mode=mode) if mode == 'sky' or mode == 'panorama': @@ -320,12 +321,12 @@ def set_view(self, mode): self.current_mode = mode else: raise ValueError('the given mode does not exist') - + self.reset_view() def reset_view(self): """ - Reset the current view mode's coordinates and field of view to + Reset the current view mode's coordinates and field of view to their original states. """ if self.current_mode == 'sky': @@ -345,7 +346,7 @@ def reset_view(self): fov=1e14*u.deg, instant=False) if self.current_mode == 'panorama': pass - + def load_image_collection(self, url): """ Load a collection of layers for possible use in the viewer. @@ -489,3 +490,12 @@ def add_collection(self, points, **kwargs): """ collection = CircleCollection(self, points, **kwargs) return collection + + def _validate_fits_data(self, filename): + if not os.path.exists(filename): + raise Exception("File {0} does not exist".format(filename)) + from astropy.wcs import WCS + wcs = WCS(filename) + projection = wcs.celestial.wcs.ctype[0][4:] + if projection != '-TAN': + raise ValueError("Only -TAN FITS files are supported at the moment") diff --git a/pywwt/data_server.py b/pywwt/data_server.py index fcfb0017..a13d0b5a 100644 --- a/pywwt/data_server.py +++ b/pywwt/data_server.py @@ -65,13 +65,13 @@ def start_app(self): pass raise Exception("Could not start up data server") - def serve_file(self, filename, real_name=True): + def serve_file(self, filename, real_name=True, extension=''): with open(filename, 'rb') as f: content = f.read() if real_name: hash = os.path.basename(filename) else: - hash = md5(content).hexdigest() + '.fits' + hash = md5(content).hexdigest() + extension self._files[hash] = os.path.abspath(filename) return 'http://' + self.host + ':' + str(self.port) + '/data/' + hash diff --git a/pywwt/jupyter.py b/pywwt/jupyter.py index 09bed053..fe6d2f80 100644 --- a/pywwt/jupyter.py +++ b/pywwt/jupyter.py @@ -59,8 +59,8 @@ def load_fits_data(self, filename): filename : str The filename of the FITS file to display. """ + self._validate_fits_data(filename) url = serve_file(filename, extension='.fits') - print("GET", url) self._send_msg(event='load_fits', url=url) @property diff --git a/pywwt/qt.py b/pywwt/qt.py index f2f34539..2be58131 100644 --- a/pywwt/qt.py +++ b/pywwt/qt.py @@ -200,7 +200,8 @@ def load_fits_data(self, filename): filename : str The filename of the FITS file to display. """ - url = self._data_server.serve_file(filename) + self._validate_fits_data(filename) + url = self._data_server.serve_file(filename, extension='.fits') self._send_msg(event='load_fits', url=url) def render(self, filename): From 6ebd89dcc318a59362b14b03df5366e5c383af5c Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 6 Apr 2018 12:42:27 +0100 Subject: [PATCH 4/6] Update changelog entry --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2042aa1f..a984db19 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ 0.4.0 (unreleased) ------------------ -- Added ``WWTQtClient.load_fits_data`` method. [#78] +- Added ``load_fits_data`` method for Qt and Jupyter clients. [#78] - Fix compatibility with Jupyter Lab. [#63, #65] From 9a5f66ed641c33d5061611d708747a5fc4f0df35 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 6 Apr 2018 13:45:52 +0100 Subject: [PATCH 5/6] Add base URL for Jupyter server on JS side and remove unused code --- lib/wwt.js | 6 ++++++ pywwt/jupyter_server.py | 22 +++++----------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/wwt.js b/lib/wwt.js index 14680084..0907069a 100644 --- a/lib/wwt.js +++ b/lib/wwt.js @@ -27,9 +27,11 @@ var WWTView = widgets.DOMWidgetView.extend({ try { // This should work for Jupyter notebook nbextensions = requirejs.s.contexts._.config.paths.nbextensions; + this.wwt_base_url = nbextensions.replace('nbextensions', 'wwt'); } catch(e) { // We should get here in Jupyter lab nbextensions = null; + this.wwt_base_url = ""; } if (nbextensions == null) { @@ -122,6 +124,10 @@ var WWTView = widgets.DOMWidgetView.extend({ } } + if (msg['url'] != null && msg['url'].slice(4) == '/wwt') { + msg['url'] = this.wwt_base_url + msg['url']; + } + this.wwt_window.wwt_apply_json_message(this.wwt_window.wwt, msg); } diff --git a/pywwt/jupyter_server.py b/pywwt/jupyter_server.py index 739ff6a6..3722f9c7 100644 --- a/pywwt/jupyter_server.py +++ b/pywwt/jupyter_server.py @@ -16,9 +16,6 @@ class WWTFileHandler(IPythonHandler): def get(self, filename): - with open('/tmp/test.log', 'w') as f: - f.write(filename) - filename = os.path.basename(filename) # First we check if this is a standard file in the static directory @@ -31,8 +28,6 @@ def get(self, filename): raise web.HTTPError(404) with open(CONFIG) as f: config = json.load(f) - with open('/tmp/test2.log', 'w') as f: - f.write(str(repr(config['paths'])) + " " + str(filename in config['paths'])) if filename in config['paths']: path = config['paths'][filename] else: @@ -46,9 +41,6 @@ def get(self, filename): def serve_file(path, extension=''): - with open('/tmp/in2.log', 'w') as f: - f.write(str(os.getppid())) - if not os.path.exists(path): raise ValueError("Path {0} does not exist".format(path)) @@ -64,7 +56,7 @@ def serve_file(path, extension=''): with open(CONFIG, 'w') as f: json.dump(config, f) - return url_path_join(config['base_url'], hash) + return '/wwt/' + hash def load_jupyter_server_extension(nb_server_app): @@ -72,14 +64,10 @@ def load_jupyter_server_extension(nb_server_app): web_app = nb_server_app.web_app host_pattern = '.*$' - # FIXME: the current solution relies on a single file, and won't support - # concurrent connections. - - config = {} - config['paths'] = {} - config['base_url'] = url_path_join(web_app.settings['base_url'], '/wwt') - with open(CONFIG, 'w') as f: - json.dump(config, f) + if not os.path.exists(CONFIG): + config = {'paths': {}} + with open(CONFIG, 'w') as f: + json.dump(config, f) route_pattern = url_path_join(web_app.settings['base_url'], '/wwt/(.*)') web_app.add_handlers(host_pattern, [(route_pattern, WWTFileHandler)]) From bd5073f7bb5ff1b493f5ae8aef91fd477345f21d Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 6 Apr 2018 15:19:51 +0100 Subject: [PATCH 6/6] Simplify logic for getting base URL --- lib/wwt.js | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/lib/wwt.js b/lib/wwt.js index 0907069a..7841d6eb 100644 --- a/lib/wwt.js +++ b/lib/wwt.js @@ -24,21 +24,9 @@ var WWTView = widgets.DOMWidgetView.extend({ // page. We use the same HTML file as for the Qt client. var div = document.createElement("div"); - try { - // This should work for Jupyter notebook - nbextensions = requirejs.s.contexts._.config.paths.nbextensions; - this.wwt_base_url = nbextensions.replace('nbextensions', 'wwt'); - } catch(e) { - // We should get here in Jupyter lab - nbextensions = null; - this.wwt_base_url = ""; - } + this.wwt_base_url = require('@jupyterlab/coreutils').PageConfig.getBaseUrl(); - if (nbextensions == null) { - div.innerHTML = "" - } else { - div.innerHTML = "" - } + div.innerHTML = "" this.el.appendChild(div);