From 80949fc833c4bf950006a61c71c432fdd8b409b8 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Sun, 22 Sep 2019 23:35:49 -0400 Subject: [PATCH 01/15] Refactoring server.py into more intentional files --- py/visdom/__init__.py | 2 + py/visdom/server/__main__.py | 15 + py/visdom/server/app.py | 159 +++ py/visdom/server/build.py | 129 +++ py/visdom/server/defaults.py | 14 + .../handlers/all_handlers.py} | 979 +----------------- py/visdom/server/handlers/base_handlers.py | 103 ++ py/visdom/server/run_server.py | 171 +++ py/visdom/utils/server_utils.py | 414 ++++++++ py/visdom/utils/shared_utils.py | 62 ++ setup.py | 2 +- 11 files changed, 1075 insertions(+), 975 deletions(-) create mode 100644 py/visdom/server/__main__.py create mode 100644 py/visdom/server/app.py create mode 100644 py/visdom/server/build.py create mode 100644 py/visdom/server/defaults.py rename py/visdom/{server.py => server/handlers/all_handlers.py} (52%) create mode 100644 py/visdom/server/handlers/base_handlers.py create mode 100644 py/visdom/server/run_server.py create mode 100644 py/visdom/utils/server_utils.py create mode 100644 py/visdom/utils/shared_utils.py diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index c1f91f4d..4c497f1a 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -6,6 +6,8 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. +from visdom.utils.shared_utils import get_new_window_id +from visdom import server import os.path import requests import traceback diff --git a/py/visdom/server/__main__.py b/py/visdom/server/__main__.py new file mode 100644 index 00000000..aa1195d5 --- /dev/null +++ b/py/visdom/server/__main__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import sys + +assert sys.version_info[0] >= 3, 'To use visdom with python 2, downgrade to v0.1.8.9' + +if __name__ == "__main__": + from visdom.server.run_server import download_scripts_and_run + download_scripts_and_run() diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py new file mode 100644 index 00000000..955ab74f --- /dev/null +++ b/py/visdom/server/app.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +""" +Main application class that pulls handlers together and maintains +all of the required state about the currently running server. +""" + +from visdom.utils.shared_utils import warn_once, ensure_dir_exists, get_visdom_path_to + +from visdom.utils.server_utils import ( + serialize_env, +) + +# TODO replace this next +from visdom.server.handlers.all_handlers import * + +import copy +import hashlib +import logging +import os +import time + +import tornado.web # noqa E402: gotta install ioloop first +import tornado.escape # noqa E402: gotta install ioloop first + +LAYOUT_FILE = 'layouts.json' + +tornado_settings = { + "autoescape": None, + "debug": "/dbg/" in __file__, + "static_path": get_visdom_path_to('static'), + "template_path": get_visdom_path_to('static'), + "compiled_template_cache": False +} + +class Application(tornado.web.Application): + def __init__(self, port=DEFAULT_PORT, base_url='', + env_path=DEFAULT_ENV_PATH, readonly=False, + user_credential=None, use_frontend_client_polling=False): + self.env_path = env_path + self.state = self.load_state() + self.layouts = self.load_layouts() + self.subs = {} + self.sources = {} + self.port = port + self.base_url = base_url + self.readonly = readonly + self.user_credential = user_credential + self.login_enabled = False + self.last_access = time.time() + self.wrap_socket = use_frontend_client_polling + + if user_credential: + self.login_enabled = True + with open(DEFAULT_ENV_PATH + "COOKIE_SECRET", "r") as fn: + tornado_settings["cookie_secret"] = fn.read() + + tornado_settings['static_url_prefix'] = self.base_url + "/static/" + tornado_settings['debug'] = True + handlers = [ + (r"%s/events" % self.base_url, PostHandler, {'app': self}), + (r"%s/update" % self.base_url, UpdateHandler, {'app': self}), + (r"%s/close" % self.base_url, CloseHandler, {'app': self}), + (r"%s/socket" % self.base_url, SocketHandler, {'app': self}), + (r"%s/socket_wrap" % self.base_url, SocketWrap, {'app': self}), + (r"%s/vis_socket" % self.base_url, + VisSocketHandler, {'app': self}), + (r"%s/vis_socket_wrap" % self.base_url, + VisSocketWrap, {'app': self}), + (r"%s/env/(.*)" % self.base_url, EnvHandler, {'app': self}), + (r"%s/compare/(.*)" % self.base_url, + CompareHandler, {'app': self}), + (r"%s/save" % self.base_url, SaveHandler, {'app': self}), + (r"%s/error/(.*)" % self.base_url, ErrorHandler, {'app': self}), + (r"%s/win_exists" % self.base_url, ExistsHandler, {'app': self}), + (r"%s/win_data" % self.base_url, DataHandler, {'app': self}), + (r"%s/delete_env" % self.base_url, + DeleteEnvHandler, {'app': self}), + (r"%s/win_hash" % self.base_url, HashHandler, {'app': self}), + (r"%s/env_state" % self.base_url, EnvStateHandler, {'app': self}), + (r"%s/fork_env" % self.base_url, ForkEnvHandler, {'app': self}), + (r"%s(.*)" % self.base_url, IndexHandler, {'app': self}), + ] + super(Application, self).__init__(handlers, **tornado_settings) + + def get_last_access(self): + if len(self.subs) > 0 or len(self.sources) > 0: + # update the last access time to now, as someone + # is currently connected to the server + self.last_access = time.time() + return self.last_access + + def save_layouts(self): + if self.env_path is None: + warn_once( + 'Saving and loading to disk has no effect when running with ' + 'env_path=None.', + RuntimeWarning + ) + return + layout_filepath = os.path.join(self.env_path, 'view', LAYOUT_FILE) + with open(layout_filepath, 'w') as fn: + fn.write(self.layouts) + + def load_layouts(self): + if self.env_path is None: + warn_once( + 'Saving and loading to disk has no effect when running with ' + 'env_path=None.', + RuntimeWarning + ) + return "" + layout_filepath = os.path.join(self.env_path, 'view', LAYOUT_FILE) + ensure_dir_exists(layout_filepath) + if os.path.isfile(layout_filepath): + with open(layout_filepath, 'r') as fn: + return fn.read() + else: + return "" + + def load_state(self): + state = {} + env_path = self.env_path + if env_path is None: + warn_once( + 'Saving and loading to disk has no effect when running with ' + 'env_path=None.', + RuntimeWarning + ) + return {'main': {'jsons': {}, 'reload': {}}} + ensure_dir_exists(env_path) + env_jsons = [i for i in os.listdir(env_path) if '.json' in i] + + for env_json in env_jsons: + env_path_file = os.path.join(env_path, env_json) + try: + with open(env_path_file, 'r') as fn: + env_data = tornado.escape.json_decode(fn.read()) + except Exception as e: + logging.warn( + "Failed loading environment json: {} - {}".format( + env_path_file, repr(e))) + continue + + eid = env_json.replace('.json', '') + state[eid] = {'jsons': env_data['jsons'], + 'reload': env_data['reload']} + + if 'main' not in state and 'main.json' not in env_jsons: + state['main'] = {'jsons': {}, 'reload': {}} + serialize_env(state, ['main'], env_path=self.env_path) + + return state diff --git a/py/visdom/server/build.py b/py/visdom/server/build.py new file mode 100644 index 00000000..9ca41890 --- /dev/null +++ b/py/visdom/server/build.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import visdom +from visdom.utils.shared_utils import ensure_dir_exists, get_visdom_path +import os +from urllib import request +from urllib.error import HTTPError, URLError + + +def download_scripts(proxies=None, install_dir=None): + """ + Function to download all of the javascript, css, and font dependencies, + and put them in the correct locations to run the server + """ + print("Checking for scripts.") + + # location in which to download stuff: + if install_dir is None: + install_dir = get_visdom_path() + + # all files that need to be downloaded: + b = 'https://unpkg.com/' + bb = '%sbootstrap@3.3.7/dist/' % b + ext_files = { + # - js + '%sjquery@3.1.1/dist/jquery.min.js' % b: 'jquery.min.js', + '%sbootstrap@3.3.7/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js', + '%sreact@16.2.0/umd/react.production.min.js' % b: 'react-react.min.js', + '%sreact-dom@16.2.0/umd/react-dom.production.min.js' % b: + 'react-dom.min.js', + '%sreact-modal@3.1.10/dist/react-modal.min.js' % b: + 'react-modal.min.js', + 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_SVG': # noqa + 'mathjax-MathJax.js', + # here is another url in case the cdn breaks down again. + # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js + 'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js', + # Stanford Javascript Crypto Library for Password Hashing + '%ssjcl@1.0.7/sjcl.js' % b: 'sjcl.js', + + # - css + '%sreact-resizable@1.4.6/css/styles.css' % b: + 'react-resizable-styles.css', + '%sreact-grid-layout@0.16.3/css/styles.css' % b: + 'react-grid-layout-styles.css', + '%scss/bootstrap.min.css' % bb: 'bootstrap.min.css', + + # - fonts + '%sclassnames@2.2.5' % b: 'classnames', + '%slayout-bin-packer@1.4.0/dist/layout-bin-packer.js' % b: + 'layout_bin_packer.js', + '%sfonts/glyphicons-halflings-regular.eot' % bb: + 'glyphicons-halflings-regular.eot', + '%sfonts/glyphicons-halflings-regular.woff2' % bb: + 'glyphicons-halflings-regular.woff2', + '%sfonts/glyphicons-halflings-regular.woff' % bb: + 'glyphicons-halflings-regular.woff', + '%sfonts/glyphicons-halflings-regular.ttf' % bb: + 'glyphicons-halflings-regular.ttf', + '%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: # noqa + 'glyphicons-halflings-regular.svg#glyphicons_halflingsregular', + } + + # make sure all relevant folders exist: + dir_list = [ + '%s' % install_dir, + '%s/static' % install_dir, + '%s/static/js' % install_dir, + '%s/static/css' % install_dir, + '%s/static/fonts' % install_dir, + ] + for directory in dir_list: + if not os.path.exists(directory): + os.makedirs(directory) + + # set up proxy handler: + handler = request.ProxyHandler(proxies) if proxies is not None \ + else request.BaseHandler() + opener = request.build_opener(handler) + request.install_opener(opener) + + built_path = os.path.join(install_dir, 'static/version.built') + is_built = visdom.__version__ == 'no_version_file' + if os.path.exists(built_path): + with open(built_path, 'r') as build_file: + build_version = build_file.read().strip() + if build_version == visdom.__version__: + is_built = True + else: + os.remove(built_path) + if not is_built: + print('Downloading scripts, this may take a little while') + + # download files one-by-one: + for (key, val) in ext_files.items(): + + # set subdirectory: + if val.endswith('.js'): + sub_dir = 'js' + elif val.endswith('.css'): + sub_dir = 'css' + else: + sub_dir = 'fonts' + + # download file: + filename = '%s/static/%s/%s' % (install_dir, sub_dir, val) + if not os.path.exists(filename) or not is_built: + req = request.Request(key, + headers={'User-Agent': 'Chrome/30.0.0.0'}) + try: + data = opener.open(req).read() + with open(filename, 'wb') as fwrite: + fwrite.write(data) + except HTTPError as exc: + logging.error('Error {} while downloading {}'.format( + exc.code, key)) + except URLError as exc: + logging.error('Error {} while downloading {}'.format( + exc.reason, key)) + + if not is_built: + with open(built_path, 'w+') as build_file: + build_file.write(visdom.__version__) diff --git a/py/visdom/server/defaults.py b/py/visdom/server/defaults.py new file mode 100644 index 00000000..99957ef5 --- /dev/null +++ b/py/visdom/server/defaults.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from os.path import expanduser + +DEFAULT_ENV_PATH = '%s/.visdom/' % expanduser("~") +DEFAULT_PORT = 8097 +DEFAULT_HOSTNAME = "localhost" +DEFAULT_BASE_URL = "/" diff --git a/py/visdom/server.py b/py/visdom/server/handlers/all_handlers.py similarity index 52% rename from py/visdom/server.py rename to py/visdom/server/handlers/all_handlers.py index d2b9a6c2..ecf07fde 100644 --- a/py/visdom/server.py +++ b/py/visdom/server/handlers/all_handlers.py @@ -8,23 +8,19 @@ """Server""" -import argparse +# TODO fix these imports +from visdom.utils.shared_utils import * +from visdom.utils.server_utils import * +from visdom.server.handlers.base_handlers import * import copy import getpass import hashlib -import inspect import json import jsonpatch import logging import math import os -import sys import time -import traceback -import uuid -import warnings -import platform -from os.path import expanduser from collections import OrderedDict from collections.abc import Mapping try: @@ -34,352 +30,19 @@ # for python 3.7 and below from collections import Mapping, Sequence -from zmq.eventloop import ioloop -ioloop.install() # Needs to happen before any tornado imports! - import tornado.ioloop # noqa E402: gotta install ioloop first import tornado.web # noqa E402: gotta install ioloop first import tornado.websocket # noqa E402: gotta install ioloop first import tornado.escape # noqa E402: gotta install ioloop first LAYOUT_FILE = 'layouts.json' -DEFAULT_ENV_PATH = '%s/.visdom/' % expanduser("~") -DEFAULT_PORT = 8097 -DEFAULT_HOSTNAME = "localhost" -DEFAULT_BASE_URL = "/" here = os.path.abspath(os.path.dirname(__file__)) COMPACT_SEPARATORS = (',', ':') -_seen_warnings = set() - MAX_SOCKET_WAIT = 15 -assert sys.version_info[0] >= 3, 'To use visdom with python 2, downgrade to v0.1.8.9' - - -def warn_once(msg, warningtype=None): - """ - Raise a warning, but only once. - :param str msg: Message to display - :param Warning warningtype: Type of warning, e.g. DeprecationWarning - """ - global _seen_warnings - if msg not in _seen_warnings: - _seen_warnings.add(msg) - warnings.warn(msg, warningtype, stacklevel=2) - - -def check_auth(f): - def _check_auth(self, *args, **kwargs): - self.last_access = time.time() - if self.login_enabled and not self.current_user: - self.set_status(400) - return - f(self, *args, **kwargs) - return _check_auth - - -def get_rand_id(): - return str(uuid.uuid4()) - - -def ensure_dir_exists(path): - """Make sure the parent dir exists for path so we can write a file.""" - try: - os.makedirs(os.path.dirname(os.path.abspath(path))) - except OSError as e1: - assert e1.errno == 17 # errno.EEXIST - - -def get_path(filename): - """Get the path to an asset.""" - cwd = os.path.dirname( - os.path.abspath(inspect.getfile(inspect.currentframe()))) - return os.path.join(cwd, filename) - - -def escape_eid(eid): - """Replace slashes with underscores, to avoid recognizing them - as directories. - """ - - return eid.replace('/', '_') - - -def extract_eid(args): - """Extract eid from args. If eid does not exist in args, - it returns 'main'.""" - - eid = 'main' if args.get('eid') is None else args.get('eid') - return escape_eid(eid) - - -def set_cookie(value=None): - """Create cookie secret key for authentication""" - if value is not None: - cookie_secret = value - else: - cookie_secret = input("Please input your cookie secret key here: ") - with open(DEFAULT_ENV_PATH + "COOKIE_SECRET", "w") as cookie_file: - cookie_file.write(cookie_secret) - - -def hash_password(password): - """Hashing Password with SHA-256""" - return hashlib.sha256(password.encode("utf-8")).hexdigest() - - - - -class LazyEnvData(Mapping): - def __init__(self, env_path_file): - self._env_path_file = env_path_file - self._raw_dict = None - - def lazy_load_data(self): - if self._raw_dict is not None: - return - - try: - with open(self._env_path_file, 'r') as fn: - env_data = tornado.escape.json_decode(fn.read()) - except Exception as e: - raise ValueError( - "Failed loading environment json: {} - {}".format( - self._env_path_file, repr(e))) - self._raw_dict = { - 'jsons': env_data['jsons'], - 'reload': env_data['reload'] - } - - def __getitem__(self, key): - self.lazy_load_data() - return self._raw_dict.__getitem__(key) - - def __setitem__(self, key, value): - self.lazy_load_data() - return self._raw_dict.__setitem__(key, value) - - def __iter__(self): - self.lazy_load_data() - return iter(self._raw_dict) - - def __len__(self): - self.lazy_load_data() - return len(self._raw_dict) - - -tornado_settings = { - "autoescape": None, - "debug": "/dbg/" in __file__, - "static_path": get_path('static'), - "template_path": get_path('static'), - "compiled_template_cache": False -} - - -def serialize_env(state, eids, env_path=DEFAULT_ENV_PATH): - env_ids = [i for i in eids if i in state] - if env_path is not None: - for env_id in env_ids: - env_path_file = os.path.join(env_path, "{0}.json".format(env_id)) - with open(env_path_file, 'w') as fn: - if isinstance(state[env_id], LazyEnvData): - fn.write(json.dumps(state[env_id]._raw_dict)) - else: - fn.write(json.dumps(state[env_id])) - return env_ids - - -def serialize_all(state, env_path=DEFAULT_ENV_PATH): - serialize_env(state, list(state.keys()), env_path=env_path) - - -class Application(tornado.web.Application): - def __init__(self, port=DEFAULT_PORT, base_url='', - env_path=DEFAULT_ENV_PATH, readonly=False, - user_credential=None, use_frontend_client_polling=False, - eager_data_loading=False): - self.eager_data_loading = eager_data_loading - self.env_path = env_path - self.state = self.load_state() - self.layouts = self.load_layouts() - self.user_settings = self.load_user_settings() - self.subs = {} - self.sources = {} - self.port = port - self.base_url = base_url - self.readonly = readonly - self.user_credential = user_credential - self.login_enabled = False - self.last_access = time.time() - self.wrap_socket = use_frontend_client_polling - - if user_credential: - self.login_enabled = True - with open(DEFAULT_ENV_PATH + "COOKIE_SECRET", "r") as fn: - tornado_settings["cookie_secret"] = fn.read() - - tornado_settings['static_url_prefix'] = self.base_url + "/static/" - tornado_settings['debug'] = True - handlers = [ - (r"%s/events" % self.base_url, PostHandler, {'app': self}), - (r"%s/update" % self.base_url, UpdateHandler, {'app': self}), - (r"%s/close" % self.base_url, CloseHandler, {'app': self}), - (r"%s/socket" % self.base_url, SocketHandler, {'app': self}), - (r"%s/socket_wrap" % self.base_url, SocketWrap, {'app': self}), - (r"%s/vis_socket" % self.base_url, - VisSocketHandler, {'app': self}), - (r"%s/vis_socket_wrap" % self.base_url, - VisSocketWrap, {'app': self}), - (r"%s/env/(.*)" % self.base_url, EnvHandler, {'app': self}), - (r"%s/compare/(.*)" % self.base_url, - CompareHandler, {'app': self}), - (r"%s/save" % self.base_url, SaveHandler, {'app': self}), - (r"%s/error/(.*)" % self.base_url, ErrorHandler, {'app': self}), - (r"%s/win_exists" % self.base_url, ExistsHandler, {'app': self}), - (r"%s/win_data" % self.base_url, DataHandler, {'app': self}), - (r"%s/delete_env" % self.base_url, - DeleteEnvHandler, {'app': self}), - (r"%s/win_hash" % self.base_url, HashHandler, {'app': self}), - (r"%s/env_state" % self.base_url, EnvStateHandler, {'app': self}), - (r"%s/fork_env" % self.base_url, ForkEnvHandler, {'app': self}), - (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {'app': self}), - (r"%s(.*)" % self.base_url, IndexHandler, {'app': self}), - ] - super(Application, self).__init__(handlers, **tornado_settings) - - def get_last_access(self): - if len(self.subs) > 0 or len(self.sources) > 0: - # update the last access time to now, as someone - # is currently connected to the server - self.last_access = time.time() - return self.last_access - - def save_layouts(self): - if self.env_path is None: - warn_once( - 'Saving and loading to disk has no effect when running with ' - 'env_path=None.', - RuntimeWarning - ) - return - layout_filepath = os.path.join(self.env_path, 'view', LAYOUT_FILE) - with open(layout_filepath, 'w') as fn: - fn.write(self.layouts) - - def load_layouts(self): - if self.env_path is None: - warn_once( - 'Saving and loading to disk has no effect when running with ' - 'env_path=None.', - RuntimeWarning - ) - return "" - layout_filepath = os.path.join(self.env_path, 'view', LAYOUT_FILE) - ensure_dir_exists(layout_filepath) - if os.path.isfile(layout_filepath): - with open(layout_filepath, 'r') as fn: - return fn.read() - else: - return "" - - def load_state(self): - state = {} - env_path = self.env_path - if env_path is None: - warn_once( - 'Saving and loading to disk has no effect when running with ' - 'env_path=None.', - RuntimeWarning - ) - return {'main': {'jsons': {}, 'reload': {}}} - ensure_dir_exists(env_path) - env_jsons = [i for i in os.listdir(env_path) if '.json' in i] - for env_json in env_jsons: - eid = env_json.replace('.json', '') - env_path_file = os.path.join(env_path, env_json) - - if self.eager_data_loading: - try: - with open(env_path_file, 'r') as fn: - env_data = tornado.escape.json_decode(fn.read()) - except Exception as e: - logging.warn( - "Failed loading environment json: {} - {}".format( - env_path_file, repr(e))) - continue - - state[eid] = {'jsons': env_data['jsons'], - 'reload': env_data['reload']} - else: - state[eid] = LazyEnvData(env_path_file) - - if 'main' not in state and 'main.json' not in env_jsons: - state['main'] = {'jsons': {}, 'reload': {}} - serialize_env(state, ['main'], env_path=self.env_path) - - return state - - def load_user_settings(self): - settings = {} - - """Determines & uses the platform-specific root directory for user configurations.""" - if platform.system() == "Windows": - base_dir = os.getenv('APPDATA') - elif platform.system() == "Darwin": # osx - base_dir = os.path.expanduser('~/Library/Preferences') - else: - base_dir = os.getenv('XDG_CONFIG_HOME', os.path.expanduser('~/.config')) - config_dir = os.path.join(base_dir, "visdom") - - # initialize user style - user_css = "" - logging.error("initializing") - home_style_path = os.path.join(config_dir, "style.css") - if os.path.exists(home_style_path): - with open(home_style_path, "r") as f: - user_css += "\n" + f.read() - project_style_path = os.path.join(self.env_path, "style.css") - if os.path.exists(project_style_path): - with open(project_style_path, "r") as f: - user_css += "\n" + f.read() - - settings['config_dir'] = config_dir - settings['user_css'] = user_css - - return settings - - - -def broadcast_envs(handler, target_subs=None): - if target_subs is None: - target_subs = handler.subs.values() - for sub in target_subs: - sub.write_message(json.dumps( - {'command': 'env_update', 'data': list(handler.state.keys())} - )) - - -def send_to_sources(handler, msg): - target_sources = handler.sources.values() - for source in target_sources: - source.write_message(json.dumps(msg)) - - -class BaseWebSocketHandler(tornado.websocket.WebSocketHandler): - def get_current_user(self): - """ - This method determines the self.current_user - based the value of cookies that set in POST method - at IndexHandler by self.set_secure_cookie - """ - try: - return self.get_secure_cookie("user_password") - except Exception: # Not using secure cookies - return None - +# TODO Split this file up it's terrible class VisSocketHandler(BaseWebSocketHandler): def initialize(self, app): @@ -746,148 +409,6 @@ def get_messages(self): return to_send -class BaseHandler(tornado.web.RequestHandler): - def __init__(self, *request, **kwargs): - self.include_host = False - super(BaseHandler, self).__init__(*request, **kwargs) - - def get_current_user(self): - """ - This method determines the self.current_user - based the value of cookies that set in POST method - at IndexHandler by self.set_secure_cookie - """ - try: - return self.get_secure_cookie("user_password") - except Exception: # Not using secure cookies - return None - - def write_error(self, status_code, **kwargs): - logging.error("ERROR: %s: %s" % (status_code, kwargs)) - if "exc_info" in kwargs: - logging.info('Traceback: {}'.format( - traceback.format_exception(*kwargs["exc_info"]))) - if self.settings.get("debug") and "exc_info" in kwargs: - logging.error("rendering error page") - exc_info = kwargs["exc_info"] - # exc_info is a tuple consisting of: - # 1. The class of the Exception - # 2. The actual Exception that was thrown - # 3. The traceback opbject - try: - params = { - 'error': exc_info[1], - 'trace_info': traceback.format_exception(*exc_info), - 'request': self.request.__dict__ - } - - self.render("error.html", **params) - logging.error("rendering complete") - except Exception as e: - logging.error(e) - - -def update_window(p, args): - """Adds new args to a window if they exist""" - content = p['content'] - layout_update = args.get('layout', {}) - for layout_name, layout_val in layout_update.items(): - if layout_val is not None: - content['layout'][layout_name] = layout_val - opts = args.get('opts', {}) - for opt_name, opt_val in opts.items(): - if opt_val is not None: - p[opt_name] = opt_val - - if 'legend' in opts: - pdata = p['content']['data'] - for i, d in enumerate(pdata): - d['name'] = opts['legend'][i] - return p - - -def window(args): - """ Build a window dict structure for sending to client """ - uid = args.get('win', 'window_' + get_rand_id()) - if uid is None: - uid = 'window_' + get_rand_id() - opts = args.get('opts', {}) - - ptype = args['data'][0]['type'] - - p = { - 'command': 'window', - 'id': str(uid), - 'title': opts.get('title', ''), - 'inflate': opts.get('inflate', True), - 'width': opts.get('width'), - 'height': opts.get('height'), - 'contentID': get_rand_id(), # to detected updated windows - } - - if ptype == 'image_history': - p.update({ - 'content': [args['data'][0]['content']], - 'selected': 0, - 'type': ptype, - 'show_slider': opts.get('show_slider', True) - }) - elif ptype in ['image', 'text', 'properties']: - p.update({'content': args['data'][0]['content'], 'type': ptype}) - elif ptype == 'network': - p.update({ - 'content': args['data'][0]['content'] , - 'type': ptype, - 'directed': opts.get("directed", False), - 'showEdgeLabels' : opts.get("showEdgeLabels", "hover"), - 'showVertexLabels' : opts.get("showVertexLabels", "hover"), - }) - elif ptype in ['embeddings']: - p.update({ - 'content': args['data'][0]['content'], - 'type': ptype, - 'old_content': [], # Used to cache previous to prevent recompute - }) - p['content']['has_previous'] = False - else: - p['content'] = {'data': args['data'], 'layout': args['layout']} - p['type'] = 'plot' - - return p - - -def broadcast(self, msg, eid): - for s in self.subs: - if isinstance(self.subs[s].eid, dict): - if eid in self.subs[s].eid: - self.subs[s].write_message(msg) - else: - if self.subs[s].eid == eid: - self.subs[s].write_message(msg) - - -def register_window(self, p, eid): - # in case env doesn't exist - is_new_env = False - if eid not in self.state: - is_new_env = True - self.state[eid] = {'jsons': {}, 'reload': {}} - - env = self.state[eid]['jsons'] - - if p['id'] in env: - p['i'] = env[p['id']]['i'] - else: - p['i'] = len(env) - - env[p['id']] = p - - broadcast(self, p, eid) - if is_new_env: - broadcast_envs(self) - self.write(p['id']) - - class PostHandler(BaseHandler): def initialize(self, app): self.state = app.state @@ -949,39 +470,6 @@ def post(self): self.wrap_func(self, args) -def order_by_key(kv): - key, val = kv - return key - - -# Based on json-stable-stringify-python from @haochi with some usecase modifications -def recursive_order(node): - if isinstance(node, Mapping): - ordered_mapping = OrderedDict(sorted(node.items(), key=order_by_key)) - for key, value in ordered_mapping.items(): - ordered_mapping[key] = recursive_order(value) - return ordered_mapping - elif isinstance(node, Sequence): - if isinstance(node, (bytes,)): - return node - elif isinstance(node, (str,)): - return node - else: - return [recursive_order(item) for item in node] - if isinstance(node, float) and node.is_integer(): - return int(node) - return node - - -def stringify(node): - return json.dumps(recursive_order(node), separators=COMPACT_SEPARATORS) - - -def hash_md_window(window_json): - json_string = stringify(window_json).encode("utf-8") - return hashlib.md5(json_string).hexdigest() - - class UpdateHandler(BaseHandler): def initialize(self, app): self.state = app.state @@ -1451,158 +939,6 @@ def post(self): self.wrap_func(self, args) -def load_env(state, eid, socket, env_path=DEFAULT_ENV_PATH): - """ load an environment to a client by socket """ - env = {} - if eid in state: - env = state.get(eid) - elif env_path is not None: - p = os.path.join(env_path, eid.strip(), '.json') - if os.path.exists(p): - with open(p, 'r') as fn: - env = tornado.escape.json_decode(fn.read()) - state[eid] = env - - if 'reload' in env: - socket.write_message( - json.dumps({'command': 'reload', 'data': env['reload']}) - ) - - jsons = list(env.get('jsons', {}).values()) - windows = sorted(jsons, key=lambda k: ('i' not in k, k.get('i', None))) - for v in windows: - socket.write_message(v) - - socket.write_message(json.dumps({'command': 'layout'})) - socket.eid = eid - - -def gather_envs(state, env_path=DEFAULT_ENV_PATH): - if env_path is not None: - items = [i.replace('.json', '') for i in os.listdir(env_path) - if '.json' in i] - else: - items = [] - return sorted(list(set(items + list(state.keys())))) - - -def compare_envs(state, eids, socket, env_path=DEFAULT_ENV_PATH): - logging.info('comparing envs') - eidNums = {e: str(i) for i, e in enumerate(eids)} - env = {} - envs = {} - for eid in eids: - if eid in state: - envs[eid] = state.get(eid) - elif env_path is not None: - p = os.path.join(env_path, eid.strip(), '.json') - if os.path.exists(p): - with open(p, 'r') as fn: - env = tornado.escape.json_decode(fn.read()) - state[eid] = env - envs[eid] = env - - res = copy.deepcopy(envs[list(envs.keys())[0]]) - name2Wid = {res['jsons'][wid].get('title', None): wid + '_compare' - for wid in res.get('jsons', {}) - if 'title' in res['jsons'][wid]} - for wid in list(res['jsons'].keys()): - res['jsons'][wid + '_compare'] = res['jsons'][wid] - res['jsons'][wid] = None - res['jsons'].pop(wid) - - for ix, eid in enumerate(sorted(envs.keys())): - env = envs[eid] - for wid in env.get('jsons', {}).keys(): - win = env['jsons'][wid] - if win.get('type', None) != 'plot': - continue - if 'content' not in win: - continue - if 'title' not in win: - continue - title = win['title'] - if title not in name2Wid or title == '': - continue - - destWid = name2Wid[title] - destWidJson = res['jsons'][destWid] - # Combine plots with the same window title. If plot data source was - # labeled "name" in the legend, rename to "envId_legend" where - # envId is enumeration of the selected environments (not the long - # environment id string). This makes plot lines more readable. - if ix == 0: - if 'name' not in destWidJson['content']['data'][0]: - continue # Skip windows with unnamed data - destWidJson['has_compare'] = False - destWidJson['content']['layout']['showlegend'] = True - destWidJson['contentID'] = get_rand_id() - for dataIdx, data in enumerate(destWidJson['content']['data']): - if 'name' not in data: - break # stop working with this plot, not right format - destWidJson['content']['data'][dataIdx]['name'] = \ - '{}_{}'.format(eidNums[eid], data['name']) - else: - if 'name' not in destWidJson['content']['data'][0]: - continue # Skip windows with unnamed data - # has_compare will be set to True only if the window title is - # shared by at least 2 envs. - destWidJson['has_compare'] = True - for _dataIdx, data in enumerate(win['content']['data']): - data = copy.deepcopy(data) - if 'name' not in data: - destWidJson['has_compare'] = False - break # stop working with this plot, not right format - data['name'] = '{}_{}'.format(eidNums[eid], data['name']) - destWidJson['content']['data'].append(data) - - # Make sure that only plots that are shared by at least two envs are shown. - # Check has_compare flag - for destWid in list(res['jsons'].keys()): - if ('has_compare' not in res['jsons'][destWid]) or \ - (not res['jsons'][destWid]['has_compare']): - del res['jsons'][destWid] - - # create legend mapping environment names to environment numbers so one can - # look it up for the new legend - tableRows = [" {} {} ".format(v, eidNums[v]) - for v in eidNums] - - tbl = """" - {}
""".format(' '.join(tableRows)) - - res['jsons']['window_compare_legend'] = { - "command": "window", - "id": "window_compare_legend", - "title": "compare_legend", - "inflate": True, - "width": None, - "height": None, - "contentID": "compare_legend", - "content": tbl, - "type": "text", - "layout": {"title": "compare_legend"}, - "i": 1, - "has_compare": True, - } - if 'reload' in res: - socket.write_message( - json.dumps({'command': 'reload', 'data': res['reload']}) - ) - - jsons = list(res.get('jsons', {}).values()) - windows = sorted(jsons, key=lambda k: ('i' not in k, k.get('i', None))) - for v in windows: - socket.write_message(v) - - socket.write_message(json.dumps({'command': 'layout'})) - socket.eid = eids - - class EnvHandler(BaseHandler): def initialize(self, app): self.state = app.state @@ -1803,308 +1139,3 @@ class ErrorHandler(BaseHandler): def get(self, text): error_text = text or "test error" raise Exception(error_text) - - -# function that downloads and installs javascript, css, and font dependencies: -def download_scripts(proxies=None, install_dir=None): - import visdom - print("Checking for scripts.") - - # location in which to download stuff: - if install_dir is None: - install_dir = os.path.dirname(visdom.__file__) - - # all files that need to be downloaded: - b = 'https://unpkg.com/' - bb = '%sbootstrap@3.3.7/dist/' % b - ext_files = { - # - js - '%sjquery@3.1.1/dist/jquery.min.js' % b: 'jquery.min.js', - '%sbootstrap@3.3.7/dist/js/bootstrap.min.js' % b: 'bootstrap.min.js', - '%sreact@16.2.0/umd/react.production.min.js' % b: 'react-react.min.js', - '%sreact-dom@16.2.0/umd/react-dom.production.min.js' % b: - 'react-dom.min.js', - '%sreact-modal@3.1.10/dist/react-modal.min.js' % b: - 'react-modal.min.js', - # here is another url in case the cdn breaks down again. - # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js - - ## [shouldsee/visdom/package_version]:latest.min.js not pointing to latest. - ## see https://github.com/plotly/plotly.py/issues/3651 - 'https://cdn.plot.ly/plotly-2.11.1.min.js': 'plotly-plotly.min.js', - - # Stanford Javascript Crypto Library for Password Hashing - '%ssjcl@1.0.7/sjcl.js' % b: 'sjcl.js', - '%slayout-bin-packer@1.4.0/dist/layout-bin-packer.js.map' % b: 'layout-bin-packer.js.map', - # d3 Libraries for plotting d3 graphs! - 'http://d3js.org/d3.v3.min.js' : 'd3.v3.min.js', - 'https://d3js.org/d3-selection-multi.v1.js' : 'd3-selection-multi.v1.js', - # Library to download the svg to png - '%ssave-svg-as-png@1.4.17/lib/saveSvgAsPng.js' % b: 'saveSvgAsPng.js', - - # - css - '%sreact-resizable@1.4.6/css/styles.css' % b: - 'react-resizable-styles.css', - '%sreact-grid-layout@0.16.3/css/styles.css' % b: - 'react-grid-layout-styles.css', - '%scss/bootstrap.min.css' % bb: 'bootstrap.min.css', - - # - fonts - '%sclassnames@2.2.5' % b: 'classnames', - '%slayout-bin-packer@1.4.0/dist/layout-bin-packer.js' % b: - 'layout_bin_packer.js', - '%sfonts/glyphicons-halflings-regular.eot' % bb: - 'glyphicons-halflings-regular.eot', - '%sfonts/glyphicons-halflings-regular.woff2' % bb: - 'glyphicons-halflings-regular.woff2', - '%sfonts/glyphicons-halflings-regular.woff' % bb: - 'glyphicons-halflings-regular.woff', - '%sfonts/glyphicons-halflings-regular.ttf' % bb: - 'glyphicons-halflings-regular.ttf', - '%sfonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular' % bb: # noqa - 'glyphicons-halflings-regular.svg#glyphicons_halflingsregular', - } - - # make sure all relevant folders exist: - dir_list = [ - '%s' % install_dir, - '%s/static' % install_dir, - '%s/static/js' % install_dir, - '%s/static/css' % install_dir, - '%s/static/fonts' % install_dir, - ] - for directory in dir_list: - if not os.path.exists(directory): - os.makedirs(directory) - - # set up proxy handler: - from urllib import request - from urllib.error import HTTPError, URLError - handler = request.ProxyHandler(proxies) if proxies is not None \ - else request.BaseHandler() - opener = request.build_opener(handler) - request.install_opener(opener) - - built_path = os.path.join(here, 'static/version.built') - is_built = visdom.__version__ == 'no_version_file' - if os.path.exists(built_path): - with open(built_path, 'r') as build_file: - build_version = build_file.read().strip() - if build_version == visdom.__version__: - is_built = True - else: - os.remove(built_path) - if not is_built: - print('Downloading scripts, this may take a little while') - - # download files one-by-one: - for (key, val) in ext_files.items(): - - # set subdirectory: - if val.endswith('.js') or val.endswith('.js.map'): - sub_dir = 'js' - elif val.endswith('.css'): - sub_dir = 'css' - else: - sub_dir = 'fonts' - - # download file: - filename = '%s/static/%s/%s' % (install_dir, sub_dir, val) - if not os.path.exists(filename) or not is_built: - req = request.Request(key, - headers={'User-Agent': 'Chrome/30.0.0.0'}) - try: - data = opener.open(req).read() - with open(filename, 'wb') as fwrite: - fwrite.write(data) - except HTTPError as exc: - logging.error('Error {} while downloading {}'.format( - exc.code, key)) - except URLError as exc: - logging.error('Error {} while downloading {}'.format( - exc.reason, key)) - - # Download MathJax Js Files - import requests - cdnjs_url = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/' - mathjax_dir = os.path.join(*cdnjs_url.split('/')[-3:]) - mathjax_path = [ - 'config/Safe.js?V=2.7.5', - 'config/TeX-AMS-MML_HTMLorMML.js?V=2.7.5', - 'extensions/Safe.js?V=2.7.5', - 'jax/output/SVG/fonts/TeX/fontdata.js?V=2.7.5', - 'jax/output/SVG/jax.js?V=2.7.5', - 'jax/output/SVG/fonts/TeX/Size1/Regular/Main.js?V=2.7.5', - 'jax/output/SVG/config.js?V=2.7.5', - 'MathJax.js?config=TeX-AMS-MML_HTMLorMML%2CSafe.js&ver=4.1', - ] - mathjax_dir_path = '%s/static/%s/%s' % (install_dir, 'js', mathjax_dir) - - for path in mathjax_path: - filename = path.split("/")[-1].split("?")[0] - extracted_directory = os.path.join(mathjax_dir_path, *path.split('/')[:-1]) - if not os.path.exists(extracted_directory): - os.makedirs(extracted_directory) - if not os.path.exists(os.path.join(extracted_directory, filename)): - js_file = requests.get(cdnjs_url + path) - with open(os.path.join(extracted_directory, filename), "wb+") as file: - file.write(js_file.content) - - if not is_built: - with open(built_path, 'w+') as build_file: - build_file.write(visdom.__version__) - - -def start_server(port=DEFAULT_PORT, hostname=DEFAULT_HOSTNAME, - base_url=DEFAULT_BASE_URL, env_path=DEFAULT_ENV_PATH, - readonly=False, print_func=None, user_credential=None, - use_frontend_client_polling=False, bind_local=False, - eager_data_loading=False): - print("It's Alive!") - app = Application(port=port, base_url=base_url, env_path=env_path, - readonly=readonly, user_credential=user_credential, - use_frontend_client_polling=use_frontend_client_polling, - eager_data_loading=eager_data_loading) - if bind_local: - app.listen(port, max_buffer_size=1024 ** 3, address='127.0.0.1') - else: - app.listen(port, max_buffer_size=1024 ** 3) - logging.info("Application Started") - - if "HOSTNAME" in os.environ and hostname == DEFAULT_HOSTNAME: - hostname = os.environ["HOSTNAME"] - else: - hostname = hostname - if print_func is None: - print( - "You can navigate to http://%s:%s%s" % (hostname, port, base_url)) - else: - print_func(port) - ioloop.IOLoop.instance().start() - app.subs = [] - app.sources = [] - - -def main(print_func=None): - parser = argparse.ArgumentParser(description='Start the visdom server.') - parser.add_argument('-port', metavar='port', type=int, - default=DEFAULT_PORT, - help='port to run the server on.') - parser.add_argument('--hostname', metavar='hostname', type=str, - default=DEFAULT_HOSTNAME, - help='host to run the server on.') - parser.add_argument('-base_url', metavar='base_url', type=str, - default=DEFAULT_BASE_URL, - help='base url for server (default = /).') - parser.add_argument('-env_path', metavar='env_path', type=str, - default=DEFAULT_ENV_PATH, - help='path to serialized session to reload.') - parser.add_argument('-logging_level', metavar='logger_level', - default='INFO', - help='logging level (default = INFO). Can take ' - 'logging level name or int (example: 20)') - parser.add_argument('-readonly', help='start in readonly mode', - action='store_true') - parser.add_argument('-enable_login', default=False, action='store_true', - help='start the server with authentication') - parser.add_argument('-force_new_cookie', default=False, - action='store_true', - help='start the server with the new cookie, ' - 'available when -enable_login provided') - parser.add_argument('-use_frontend_client_polling', default=False, - action='store_true', - help='Have the frontend communicate via polling ' - 'rather than over websockets.') - parser.add_argument('-bind_local', default=False, - action='store_true', - help='Make server only accessible only from ' - 'localhost.') - parser.add_argument('-eager_data_loading', default=False, - action='store_true', - help='Load data from filesystem when starting server (and not lazily upon first request).') - FLAGS = parser.parse_args() - - # Process base_url - base_url = FLAGS.base_url if FLAGS.base_url != DEFAULT_BASE_URL else "" - assert base_url == '' or base_url.startswith('/'), \ - 'base_url should start with /' - assert base_url == '' or not base_url.endswith('/'), \ - 'base_url should not end with / as it is appended automatically' - - try: - logging_level = int(FLAGS.logging_level) - except ValueError: - try: - logging_level = logging._checkLevel(FLAGS.logging_level) - except ValueError: - raise KeyError( - "Invalid logging level : {0}".format(FLAGS.logging_level) - ) - - logging.getLogger().setLevel(logging_level) - - if FLAGS.enable_login: - enable_env_login = 'VISDOM_USE_ENV_CREDENTIALS' - use_env = os.environ.get(enable_env_login, False) - if use_env: - username_var = 'VISDOM_USERNAME' - password_var = 'VISDOM_PASSWORD' - username = os.environ.get(username_var) - password = os.environ.get(password_var) - if not (username and password): - print( - '*** Warning ***\n' - 'You have set the {0} env variable but probably ' - 'forgot to setup one (or both) {{ {1}, {2} }} ' - 'variables.\nYou should setup these variables with ' - 'proper username and password to enable logging. Try to ' - 'setup the variables, or unset {0} to input credentials ' - 'via command line prompt instead.\n' - .format(enable_env_login, username_var, password_var)) - sys.exit(1) - - else: - username = input("Please input your username: ") - password = getpass.getpass(prompt="Please input your password: ") - - user_credential = { - "username": username, - "password": hash_password(hash_password(password)) - } - - need_to_set_cookie = ( - not os.path.isfile(DEFAULT_ENV_PATH + "COOKIE_SECRET") - or FLAGS.force_new_cookie) - - if need_to_set_cookie: - if use_env: - cookie_var = 'VISDOM_COOKIE' - env_cookie = os.environ.get(cookie_var) - if env_cookie is None: - print( - 'The cookie file is not found. Please setup {0} env ' - 'variable to provide a cookie value, or unset {1} env ' - 'variable to input credentials and cookie via command ' - 'line prompt.'.format(cookie_var, enable_env_login)) - sys.exit(1) - else: - env_cookie = None - set_cookie(env_cookie) - - else: - user_credential = None - - start_server(port=FLAGS.port, hostname=FLAGS.hostname, base_url=base_url, - env_path=FLAGS.env_path, readonly=FLAGS.readonly, - print_func=print_func, user_credential=user_credential, - use_frontend_client_polling=FLAGS.use_frontend_client_polling, - bind_local=FLAGS.bind_local, - eager_data_loading=FLAGS.eager_data_loading) - -def download_scripts_and_run(): - download_scripts() - main() - - -if __name__ == "__main__": - download_scripts_and_run() diff --git a/py/visdom/server/handlers/base_handlers.py b/py/visdom/server/handlers/base_handlers.py new file mode 100644 index 00000000..e6d3f038 --- /dev/null +++ b/py/visdom/server/handlers/base_handlers.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Server""" + +from visdom.utils.shared_utils import ( + warn_once, + get_rand_id, + get_new_window_id, + ensure_dir_exists, +) +import argparse +import copy +import getpass +import hashlib +import inspect +import json +import jsonpatch +import logging +import math +import os +import sys +import time +import traceback +from collections import OrderedDict +try: + # for after python 3.8 + from collections.abc import Mapping, Sequence +except ImportError: + # for python 3.7 and below + from collections import Mapping, Sequence + +# from zmq.eventloop import ioloop +# ioloop.install() # Needs to happen before any tornado imports! + +import tornado.ioloop # noqa E402: gotta install ioloop first +import tornado.web # noqa E402: gotta install ioloop first +import tornado.websocket # noqa E402: gotta install ioloop first +import tornado.escape # noqa E402: gotta install ioloop first + +LAYOUT_FILE = 'layouts.json' + +COMPACT_SEPARATORS = (',', ':') + +MAX_SOCKET_WAIT = 15 + +class BaseWebSocketHandler(tornado.websocket.WebSocketHandler): + def get_current_user(self): + """ + This method determines the self.current_user + based the value of cookies that set in POST method + at IndexHandler by self.set_secure_cookie + """ + try: + return self.get_secure_cookie("user_password") + except Exception: # Not using secure cookies + return None + + +class BaseHandler(tornado.web.RequestHandler): + def __init__(self, *request, **kwargs): + self.include_host = False + super(BaseHandler, self).__init__(*request, **kwargs) + + def get_current_user(self): + """ + This method determines the self.current_user + based the value of cookies that set in POST method + at IndexHandler by self.set_secure_cookie + """ + try: + return self.get_secure_cookie("user_password") + except Exception: # Not using secure cookies + return None + + def write_error(self, status_code, **kwargs): + logging.error("ERROR: %s: %s" % (status_code, kwargs)) + if "exc_info" in kwargs: + logging.info('Traceback: {}'.format( + traceback.format_exception(*kwargs["exc_info"]))) + if self.settings.get("debug") and "exc_info" in kwargs: + logging.error("rendering error page") + exc_info = kwargs["exc_info"] + # exc_info is a tuple consisting of: + # 1. The class of the Exception + # 2. The actual Exception that was thrown + # 3. The traceback opbject + try: + params = { + 'error': exc_info[1], + 'trace_info': traceback.format_exception(*exc_info), + 'request': self.request.__dict__ + } + + self.render("error.html", **params) + logging.error("rendering complete") + except Exception as e: + logging.error(e) diff --git a/py/visdom/server/run_server.py b/py/visdom/server/run_server.py new file mode 100644 index 00000000..0f6e90c3 --- /dev/null +++ b/py/visdom/server/run_server.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from visdom.server.app import Application +from visdom.server.defaults import ( + DEFAULT_BASE_URL, + DEFAULT_ENV_PATH, + DEFAULT_HOSTNAME, + DEFAULT_PORT, +) +from visdom.server.build import download_scripts + +import argparse +import getpass +import logging +import os +import sys + +from tornado import ioloop + + +def start_server(port=DEFAULT_PORT, hostname=DEFAULT_HOSTNAME, + base_url=DEFAULT_BASE_URL, env_path=DEFAULT_ENV_PATH, + readonly=False, print_func=None, user_credential=None, + use_frontend_client_polling=False): + """Run a visdom server with the given arguments""" + logging.info("It's Alive!") + app = Application(port=port, base_url=base_url, env_path=env_path, + readonly=readonly, user_credential=user_credential, + use_frontend_client_polling=use_frontend_client_polling) + app.listen(port, max_buffer_size=1024 ** 3) + logging.info("Application Started") + + if "HOSTNAME" in os.environ and hostname == DEFAULT_HOSTNAME: + hostname = os.environ["HOSTNAME"] + else: + hostname = hostname + if print_func is None: + print( + "You can navigate to http://%s:%s%s" % (hostname, port, base_url)) + else: + print_func(port) + ioloop.IOLoop.instance().start() + app.subs = [] + app.sources = [] + + +def main(print_func=None): + """ + Run a server from the command line, first parsing arguments from the + command line + """ + parser = argparse.ArgumentParser(description='Start the visdom server.') + parser.add_argument('-port', metavar='port', type=int, + default=DEFAULT_PORT, + help='port to run the server on.') + parser.add_argument('--hostname', metavar='hostname', type=str, + default=DEFAULT_HOSTNAME, + help='host to run the server on.') + parser.add_argument('-base_url', metavar='base_url', type=str, + default=DEFAULT_BASE_URL, + help='base url for server (default = /).') + parser.add_argument('-env_path', metavar='env_path', type=str, + default=DEFAULT_ENV_PATH, + help='path to serialized session to reload.') + parser.add_argument('-logging_level', metavar='logger_level', + default='INFO', + help='logging level (default = INFO). Can take ' + 'logging level name or int (example: 20)') + parser.add_argument('-readonly', help='start in readonly mode', + action='store_true') + parser.add_argument('-enable_login', default=False, action='store_true', + help='start the server with authentication') + parser.add_argument('-force_new_cookie', default=False, + action='store_true', + help='start the server with the new cookie, ' + 'available when -enable_login provided') + parser.add_argument('-use_frontend_client_polling', default=False, + action='store_true', + help='Have the frontend communicate via polling ' + 'rather than over websockets.') + FLAGS = parser.parse_args() + + # Process base_url + base_url = FLAGS.base_url if FLAGS.base_url != DEFAULT_BASE_URL else "" + assert base_url == '' or base_url.startswith('/'), \ + 'base_url should start with /' + assert base_url == '' or not base_url.endswith('/'), \ + 'base_url should not end with / as it is appended automatically' + + try: + logging_level = int(FLAGS.logging_level) + except (ValueError,): + try: + logging_level = logging._checkLevel(FLAGS.logging_level) + except ValueError: + raise KeyError( + "Invalid logging level : {0}".format(FLAGS.logging_level) + ) + + logging.getLogger().setLevel(logging_level) + + if FLAGS.enable_login: + enable_env_login = 'VISDOM_USE_ENV_CREDENTIALS' + use_env = os.environ.get(enable_env_login, False) + if use_env: + username_var = 'VISDOM_USERNAME' + password_var = 'VISDOM_PASSWORD' + username = os.environ.get(username_var) + password = os.environ.get(password_var) + if not (username and password): + print( + '*** Warning ***\n' + 'You have set the {0} env variable but probably ' + 'forgot to setup one (or both) {{ {1}, {2} }} ' + 'variables.\nYou should setup these variables with ' + 'proper username and password to enable logging. Try to ' + 'setup the variables, or unset {0} to input credentials ' + 'via command line prompt instead.\n' + .format(enable_env_login, username_var, password_var)) + sys.exit(1) + + else: + username = input("Please input your username: ") + password = getpass.getpass(prompt="Please input your password: ") + + user_credential = { + "username": username, + "password": hash_password(hash_password(password)) + } + + need_to_set_cookie = ( + not os.path.isfile(DEFAULT_ENV_PATH + "COOKIE_SECRET") + or FLAGS.force_new_cookie) + + if need_to_set_cookie: + if use_env: + cookie_var = 'VISDOM_COOKIE' + env_cookie = os.environ.get(cookie_var) + if env_cookie is None: + print( + 'The cookie file is not found. Please setup {0} env ' + 'variable to provide a cookie value, or unset {1} env ' + 'variable to input credentials and cookie via command ' + 'line prompt.'.format(cookie_var, enable_env_login)) + sys.exit(1) + else: + env_cookie = None + set_cookie(env_cookie) + + else: + user_credential = None + + start_server(port=FLAGS.port, hostname=FLAGS.hostname, base_url=base_url, + env_path=FLAGS.env_path, readonly=FLAGS.readonly, + print_func=print_func, user_credential=user_credential, + use_frontend_client_polling=FLAGS.use_frontend_client_polling) + + +def download_scripts_and_run(): + download_scripts() + main() + + +if __name__ == "__main__": + download_scripts_and_run() diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py new file mode 100644 index 00000000..bd16d6a6 --- /dev/null +++ b/py/visdom/utils/server_utils.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +""" +Utilities for the server architecture that don't really have +a more appropriate place. + +At the moment, this just inherited all of the floating functions +in the previous server.py class. +""" + +from visdom.server.defaults import ( + DEFAULT_BASE_URL, + DEFAULT_ENV_PATH, + DEFAULT_HOSTNAME, + DEFAULT_PORT, +) +from visdom.utils.shared_utils import get_new_window_id +from visdom.utils.shared_utils import ( + warn_once, + get_rand_id, + get_new_window_id, + ensure_dir_exists, +) +import copy +import hashlib +import json +import logging +import os +import time +from collections import OrderedDict +try: + # for after python 3.8 + from collections.abc import Mapping, Sequence +except ImportError: + # for python 3.7 and below + from collections import Mapping, Sequence + +from zmq.eventloop import ioloop +ioloop.install() # Needs to happen before any tornado imports! + +import tornado.escape # noqa E402: gotta install ioloop first + +LAYOUT_FILE = 'layouts.json' + +here = os.path.abspath(os.path.dirname(__file__)) +COMPACT_SEPARATORS = (',', ':') + +MAX_SOCKET_WAIT = 15 + +# ---- Vaguely server-security related functions ---- # + +def check_auth(f): + """ + Wrapper for server access methods to ensure that the access + is authorized. + """ + def _check_auth(app, *args, **kwargs): + app.last_access = time.time() + if app.login_enabled and not app.current_user: + app.set_status(400) + return + f(app, *args, **kwargs) + return _check_auth + +def set_cookie(value=None): + """Create cookie secret key for authentication""" + if value is not None: + cookie_secret = value + else: + cookie_secret = input("Please input your cookie secret key here: ") + with open(DEFAULT_ENV_PATH + "COOKIE_SECRET", "w") as cookie_file: + cookie_file.write(cookie_secret) + +def hash_password(password): + """Hashing Password with SHA-256""" + return hashlib.sha256(password.encode("utf-8")).hexdigest() + + +# ------- File management helprs ----- # + +def serialize_env(state, eids, env_path=DEFAULT_ENV_PATH): + env_ids = [i for i in eids if i in state] + if env_path is not None: + for env_id in env_ids: + env_path_file = os.path.join(env_path, "{0}.json".format(env_id)) + with open(env_path_file, 'w') as fn: + fn.write(json.dumps(state[env_id])) + return env_ids + + +def serialize_all(state, env_path=DEFAULT_ENV_PATH): + serialize_env(state, list(state.keys()), env_path=env_path) + + +# ------- Environment management helpers ----- # + + +def escape_eid(eid): + """Replace slashes with underscores, to avoid recognizing them + as directories. + """ + return eid.replace('/', '_') + + +def extract_eid(args): + """Extract eid from args. If eid does not exist in args, + it returns 'main'.""" + eid = 'main' if args.get('eid') is None else args.get('eid') + return escape_eid(eid) + + +def update_window(p, args): + """Adds new args to a window if they exist""" + content = p['content'] + layout_update = args.get('layout', {}) + for layout_name, layout_val in layout_update.items(): + if layout_val is not None: + content['layout'][layout_name] = layout_val + opts = args.get('opts', {}) + for opt_name, opt_val in opts.items(): + if opt_val is not None: + p[opt_name] = opt_val + + if 'legend' in opts: + pdata = p['content']['data'] + for i, d in enumerate(pdata): + d['name'] = opts['legend'][i] + return p + + +def window(args): + """ Build a window dict structure for sending to client """ + uid = args.get('win', get_new_window_id()) + if uid is None: + uid = get_new_window_id() + opts = args.get('opts', {}) + + ptype = args['data'][0]['type'] + + p = { + 'command': 'window', + 'id': str(uid), + 'title': opts.get('title', ''), + 'inflate': opts.get('inflate', True), + 'width': opts.get('width'), + 'height': opts.get('height'), + 'contentID': get_rand_id(), # to detected updated windows + } + + if ptype == 'image_history': + p.update({ + 'content': [args['data'][0]['content']], + 'selected': 0, + 'type': ptype, + 'show_slider': opts.get('show_slider', True) + }) + elif ptype in ['image', 'text', 'properties']: + p.update({'content': args['data'][0]['content'], 'type': ptype}) + elif ptype in ['embeddings']: + p.update({ + 'content': args['data'][0]['content'], + 'type': ptype, + 'old_content': [], # Used to cache previous to prevent recompute + }) + p['content']['has_previous'] = False + else: + p['content'] = {'data': args['data'], 'layout': args['layout']} + p['type'] = 'plot' + + return p + + +def gather_envs(state, env_path=DEFAULT_ENV_PATH): + if env_path is not None: + items = [i.replace('.json', '') for i in os.listdir(env_path) + if '.json' in i] + else: + items = [] + return sorted(list(set(items + list(state.keys())))) + + +def compare_envs(state, eids, socket, env_path=DEFAULT_ENV_PATH): + logging.info('comparing envs') + eidNums = {e: str(i) for i, e in enumerate(eids)} + env = {} + envs = {} + for eid in eids: + if eid in state: + envs[eid] = state.get(eid) + elif env_path is not None: + p = os.path.join(env_path, eid.strip(), '.json') + if os.path.exists(p): + with open(p, 'r') as fn: + env = tornado.escape.json_decode(fn.read()) + state[eid] = env + envs[eid] = env + + res = copy.deepcopy(envs[list(envs.keys())[0]]) + name2Wid = {res['jsons'][wid].get('title', None): wid + '_compare' + for wid in res.get('jsons', {}) + if 'title' in res['jsons'][wid]} + for wid in list(res['jsons'].keys()): + res['jsons'][wid + '_compare'] = res['jsons'][wid] + res['jsons'][wid] = None + res['jsons'].pop(wid) + + for ix, eid in enumerate(envs.keys()): + env = envs[eid] + for wid in env.get('jsons', {}).keys(): + win = env['jsons'][wid] + if win.get('type', None) != 'plot': + continue + if 'content' not in win: + continue + if 'title' not in win: + continue + title = win['title'] + if title not in name2Wid or title == '': + continue + + destWid = name2Wid[title] + destWidJson = res['jsons'][destWid] + # Combine plots with the same window title. If plot data source was + # labeled "name" in the legend, rename to "envId_legend" where + # envId is enumeration of the selected environments (not the long + # environment id string). This makes plot lines more readable. + if ix == 0: + if 'name' not in destWidJson['content']['data'][0]: + continue # Skip windows with unnamed data + destWidJson['has_compare'] = False + destWidJson['content']['layout']['showlegend'] = True + destWidJson['contentID'] = get_rand_id() + for dataIdx, data in enumerate(destWidJson['content']['data']): + if 'name' not in data: + break # stop working with this plot, not right format + destWidJson['content']['data'][dataIdx]['name'] = \ + '{}_{}'.format(eidNums[eid], data['name']) + else: + if 'name' not in destWidJson['content']['data'][0]: + continue # Skip windows with unnamed data + # has_compare will be set to True only if the window title is + # shared by at least 2 envs. + destWidJson['has_compare'] = True + for _dataIdx, data in enumerate(win['content']['data']): + data = copy.deepcopy(data) + if 'name' not in data: + destWidJson['has_compare'] = False + break # stop working with this plot, not right format + data['name'] = '{}_{}'.format(eidNums[eid], data['name']) + destWidJson['content']['data'].append(data) + + # Make sure that only plots that are shared by at least two envs are shown. + # Check has_compare flag + for destWid in list(res['jsons'].keys()): + if ('has_compare' not in res['jsons'][destWid]) or \ + (not res['jsons'][destWid]['has_compare']): + del res['jsons'][destWid] + + # create legend mapping environment names to environment numbers so one can + # look it up for the new legend + tableRows = [" {} {} ".format(v, eidNums[v]) + for v in eidNums] + + tbl = """" + {}
""".format(' '.join(tableRows)) + + res['jsons']['window_compare_legend'] = { + "command": "window", + "id": "window_compare_legend", + "title": "compare_legend", + "inflate": True, + "width": None, + "height": None, + "contentID": "compare_legend", + "content": tbl, + "type": "text", + "layout": {"title": "compare_legend"}, + "i": 1, + "has_compare": True, + } + if 'reload' in res: + socket.write_message( + json.dumps({'command': 'reload', 'data': res['reload']}) + ) + + jsons = list(res.get('jsons', {}).values()) + windows = sorted(jsons, key=lambda k: ('i' not in k, k.get('i', None))) + for v in windows: + socket.write_message(v) + + socket.write_message(json.dumps({'command': 'layout'})) + socket.eid = eids + + + +# ------- Broadcasting functions ---------- # + +def broadcast_envs(handler, target_subs=None): + if target_subs is None: + target_subs = handler.subs.values() + for sub in target_subs: + sub.write_message(json.dumps( + {'command': 'env_update', 'data': list(handler.state.keys())} + )) + + +def send_to_sources(handler, msg): + target_sources = handler.sources.values() + for source in target_sources: + source.write_message(json.dumps(msg)) + + +def load_env(state, eid, socket, env_path=DEFAULT_ENV_PATH): + """ load an environment to a client by socket """ + env = {} + if eid in state: + env = state.get(eid) + elif env_path is not None: + p = os.path.join(env_path, eid.strip(), '.json') + if os.path.exists(p): + with open(p, 'r') as fn: + env = tornado.escape.json_decode(fn.read()) + state[eid] = env + + if 'reload' in env: + socket.write_message( + json.dumps({'command': 'reload', 'data': env['reload']}) + ) + + jsons = list(env.get('jsons', {}).values()) + windows = sorted(jsons, key=lambda k: ('i' not in k, k.get('i', None))) + for v in windows: + socket.write_message(v) + + socket.write_message(json.dumps({'command': 'layout'})) + socket.eid = eid + + +def broadcast(self, msg, eid): + for s in self.subs: + if type(self.subs[s].eid) is list: + if eid in self.subs[s].eid: + self.subs[s].write_message(msg) + else: + if self.subs[s].eid == eid: + self.subs[s].write_message(msg) + + +def register_window(self, p, eid): + # in case env doesn't exist + is_new_env = False + if eid not in self.state: + is_new_env = True + self.state[eid] = {'jsons': {}, 'reload': {}} + + env = self.state[eid]['jsons'] + + if p['id'] in env: + p['i'] = env[p['id']]['i'] + else: + p['i'] = len(env) + + env[p['id']] = p + + broadcast(self, p, eid) + if is_new_env: + broadcast_envs(self) + self.write(p['id']) + + +# ----- Json patch helpers ---------- # + + +def order_by_key(kv): + key, val = kv + return key + + +# Based on json-stable-stringify-python from @haochi with some usecase modifications +def recursive_order(node): + if isinstance(node, Mapping): + ordered_mapping = OrderedDict(sorted(node.items(), key=order_by_key)) + for key, value in ordered_mapping.items(): + ordered_mapping[key] = recursive_order(value) + return ordered_mapping + elif isinstance(node, Sequence): + if isinstance(node, (bytes,)): + return node + elif isinstance(node, (str,)): + return node + else: + return [recursive_order(item) for item in node] + if isinstance(node, float) and node.is_integer(): + return int(node) + return node + + +def stringify(node): + return json.dumps(recursive_order(node), separators=COMPACT_SEPARATORS) + + +def hash_md_window(window_json): + json_string = stringify(window_json).encode("utf-8") + return hashlib.md5(json_string).hexdigest() diff --git a/py/visdom/utils/shared_utils.py b/py/visdom/utils/shared_utils.py new file mode 100644 index 00000000..23c164d4 --- /dev/null +++ b/py/visdom/utils/shared_utils.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +""" +Utilities that could be potentially useful in various different +parts of the visdom stack. Not to be used for particularly specific +helper functions. +""" + +import inspect +import uuid +import warnings +import os + +_seen_warnings = set() + + +def warn_once(msg, warningtype=None): + """ + Raise a warning, but only once. + :param str msg: Message to display + :param Warning warningtype: Type of warning, e.g. DeprecationWarning + """ + global _seen_warnings + if msg not in _seen_warnings: + _seen_warnings.add(msg) + warnings.warn(msg, warningtype, stacklevel=2) + + +def get_rand_id(): + """Returns a random id string""" + return str(uuid.uuid4()) + + +def get_new_window_id(): + """Return a string to be used for a new window""" + return f'win_{get_rand_id()}' + + +def ensure_dir_exists(path): + """Make sure the parent dir exists for path so we can write a file.""" + try: + os.makedirs(os.path.dirname(path)) + except OSError as e1: + assert e1.errno == 17 # errno.EEXIST + + +def get_visdom_path(): + """Get the path to the visdom/py/visdom directory.""" + cwd = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe()))) + return os.path.dirname(cwd) + + +def get_visdom_path_to(filename): + """Get the path to a file in the visdom/py/visdom directory.""" + return os.path.join(get_visdom_path(), filename) diff --git a/setup.py b/setup.py index f321fc75..17852bc1 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def get_dist(pkgname): 'scipy', 'requests', 'tornado', - 'pyzmq', + # 'pyzmq', 'six', 'jsonpatch', 'websocket-client', From 802c6125865e994505adde8c8503d954e2d56f2e Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Mon, 23 Sep 2019 00:02:11 -0400 Subject: [PATCH 02/15] Splitting all handlers into socket and base handlers --- py/visdom/server/app.py | 3 +- py/visdom/server/handlers/base_handlers.py | 54 +- py/visdom/server/handlers/socket_handlers.py | 515 ++++++++++++++++++ .../{all_handlers.py => web_handlers.py} | 495 +---------------- py/visdom/server/run_server.py | 4 + py/visdom/utils/server_utils.py | 11 +- 6 files changed, 558 insertions(+), 524 deletions(-) create mode 100644 py/visdom/server/handlers/socket_handlers.py rename py/visdom/server/handlers/{all_handlers.py => web_handlers.py} (55%) diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 955ab74f..7f64fef3 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -18,7 +18,8 @@ ) # TODO replace this next -from visdom.server.handlers.all_handlers import * +from visdom.server.handlers.socket_handlers import * +from visdom.server.handlers.web_handlers import * import copy import hashlib diff --git a/py/visdom/server/handlers/base_handlers.py b/py/visdom/server/handlers/base_handlers.py index e6d3f038..21607999 100644 --- a/py/visdom/server/handlers/base_handlers.py +++ b/py/visdom/server/handlers/base_handlers.py @@ -6,50 +6,24 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -"""Server""" +""" +Contain the basic web request handlers that all other handlers derive from +""" -from visdom.utils.shared_utils import ( - warn_once, - get_rand_id, - get_new_window_id, - ensure_dir_exists, -) -import argparse -import copy -import getpass -import hashlib -import inspect -import json -import jsonpatch import logging -import math -import os -import sys -import time import traceback -from collections import OrderedDict -try: - # for after python 3.8 - from collections.abc import Mapping, Sequence -except ImportError: - # for python 3.7 and below - from collections import Mapping, Sequence -# from zmq.eventloop import ioloop -# ioloop.install() # Needs to happen before any tornado imports! +import tornado.web +import tornado.websocket -import tornado.ioloop # noqa E402: gotta install ioloop first -import tornado.web # noqa E402: gotta install ioloop first -import tornado.websocket # noqa E402: gotta install ioloop first -import tornado.escape # noqa E402: gotta install ioloop first - -LAYOUT_FILE = 'layouts.json' - -COMPACT_SEPARATORS = (',', ':') - -MAX_SOCKET_WAIT = 15 class BaseWebSocketHandler(tornado.websocket.WebSocketHandler): + """ + Implements any required overriden functionality from the basic tornado + websocket handler. Also contains some shared logic for all WebSocketHandler + classes. + """ + def get_current_user(self): """ This method determines the self.current_user @@ -63,6 +37,11 @@ def get_current_user(self): class BaseHandler(tornado.web.RequestHandler): + """ + Implements any required overriden functionality from the basic tornado + request handlers, and contains any convenient shared logic helpers. + """ + def __init__(self, *request, **kwargs): self.include_host = False super(BaseHandler, self).__init__(*request, **kwargs) @@ -97,6 +76,7 @@ def write_error(self, status_code, **kwargs): 'request': self.request.__dict__ } + # TODO make an error.html page self.render("error.html", **params) logging.error("rendering complete") except Exception as e: diff --git a/py/visdom/server/handlers/socket_handlers.py b/py/visdom/server/handlers/socket_handlers.py new file mode 100644 index 00000000..d3d1c891 --- /dev/null +++ b/py/visdom/server/handlers/socket_handlers.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +""" +Handlers for the different types of socket events. Mostly handles parsing and +processing the web events themselves and interfacing with the server as +necessary, but defers underlying manipulations of the server's data to +the data_model itself. +""" + +# TODO fix these imports +from visdom.utils.shared_utils import * +from visdom.utils.server_utils import * +from visdom.server.handlers.base_handlers import * +import copy +import getpass +import hashlib +import json +import jsonpatch +import logging +import math +import os +import time +from collections import OrderedDict +try: + # for after python 3.8 + from collections.abc import Mapping, Sequence +except ImportError: + # for python 3.7 and below + from collections import Mapping, Sequence + +import tornado.ioloop +import tornado.escape + +MAX_SOCKET_WAIT = 15 + +# TODO move the logic that actually parses environments and layouts to +# new classes in the data_model folder. +# TODO move generalized initialization logic from these handlers into the +# basehandler +# TODO abstract out any direct references to the app where possible from +# all handlers. Can instead provide accessor functions on the state? +# TODO abstract socket interaction logic such that both the regular +# sockets and the poll-based wrappers are using as much shared code as +# possible. Try to standardize the code between the client-server and +# visdom-server socket edges. +class VisSocketHandler(BaseWebSocketHandler): + def initialize(self, app): + self.state = app.state + self.subs = app.subs + self.sources = app.sources + self.port = app.port + self.env_path = app.env_path + self.login_enabled = app.login_enabled + + def check_origin(self, origin): + return True + + def open(self): + if self.login_enabled and not self.current_user: + self.close() + return + self.sid = str(hex(int(time.time() * 10000000))[2:]) + if self not in list(self.sources.values()): + self.eid = 'main' + self.sources[self.sid] = self + logging.info('Opened visdom socket from ip: {}'.format( + self.request.remote_ip)) + + self.write_message( + json.dumps({'command': 'alive', 'data': 'vis_alive'})) + + def on_message(self, message): + logging.info('from visdom client: {}'.format(message)) + msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) + + cmd = msg.get('cmd') + if cmd == 'echo': + for sub in self.sources.values(): + sub.write_message(json.dumps(msg)) + + def on_close(self): + if self in list(self.sources.values()): + self.sources.pop(self.sid, None) + + +class VisSocketWrapper(): + def __init__(self, app): + self.state = app.state + self.subs = app.subs + self.sources = app.sources + self.port = app.port + self.env_path = app.env_path + self.login_enabled = app.login_enabled + self.app = app + self.messages = [] + self.last_read_time = time.time() + self.open() + try: + if not self.app.socket_wrap_monitor.is_running(): + self.app.socket_wrap_monitor.start() + except AttributeError: + self.app.socket_wrap_monitor = tornado.ioloop.PeriodicCallback( + self.socket_wrap_monitor_thread, 15000 + ) + self.app.socket_wrap_monitor.start() + + # TODO refactor the two socket wrappers into a wrapper class + def socket_wrap_monitor_thread(self): + if len(self.subs) > 0 or len(self.sources) > 0: + for sub in list(self.subs.values()): + if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: + sub.close() + for sub in list(self.sources.values()): + if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: + sub.close() + else: + self.app.socket_wrap_monitor.stop() + + def open(self): + if self.login_enabled and not self.current_user: + print("AUTH Failed in SocketHandler") + self.close() + return + self.sid = get_rand_id() + if self not in list(self.sources.values()): + self.eid = 'main' + self.sources[self.sid] = self + logging.info('Mocking visdom socket: {}'.format(self.sid)) + + self.write_message( + json.dumps({'command': 'alive', 'data': 'vis_alive'})) + + def on_message(self, message): + logging.info('from visdom client: {}'.format(message)) + msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) + + cmd = msg.get('cmd') + if cmd == 'echo': + for sub in self.sources.values(): + sub.write_message(json.dumps(msg)) + + def close(self): + if self in list(self.sources.values()): + self.sources.pop(self.sid, None) + + def write_message(self, msg): + self.messages.append(msg) + + def get_messages(self): + to_send = [] + while len(self.messages) > 0: + message = self.messages.pop() + if type(message) is dict: + # Not all messages are being formatted the same way (JSON) + # TODO investigate + message = json.dumps(message) + to_send.append(message) + self.last_read_time = time.time() + return to_send + + +class SocketHandler(BaseWebSocketHandler): + def initialize(self, app): + self.port = app.port + self.env_path = app.env_path + self.app = app + self.state = app.state + self.subs = app.subs + self.sources = app.sources + self.broadcast_layouts() + self.readonly = app.readonly + self.login_enabled = app.login_enabled + + def check_origin(self, origin): + return True + + def broadcast_layouts(self, target_subs=None): + if target_subs is None: + target_subs = self.subs.values() + for sub in target_subs: + sub.write_message(json.dumps( + {'command': 'layout_update', 'data': self.app.layouts} + )) + + def open(self): + if self.login_enabled and not self.current_user: + print("AUTH Failed in SocketHandler") + self.close() + return + self.sid = get_rand_id() + if self not in list(self.subs.values()): + self.eid = 'main' + self.subs[self.sid] = self + logging.info( + 'Opened new socket from ip: {}'.format(self.request.remote_ip)) + + self.write_message( + json.dumps({'command': 'register', 'data': self.sid, + 'readonly': self.readonly})) + self.broadcast_layouts([self]) + broadcast_envs(self, [self]) + + def on_message(self, message): + logging.info('from web client: {}'.format(message)) + msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) + + cmd = msg.get('cmd') + + if self.readonly: + return + + if cmd == 'close': + if 'data' in msg and 'eid' in msg: + logging.info('closing window {}'.format(msg['data'])) + p_data = self.state[msg['eid']]['jsons'].pop(msg['data'], None) + event = { + 'event_type': 'close', + 'target': msg['data'], + 'eid': msg['eid'], + 'pane_data': p_data, + } + send_to_sources(self, event) + elif cmd == 'save': + # save localStorage window metadata + if 'data' in msg and 'eid' in msg: + msg['eid'] = escape_eid(msg['eid']) + self.state[msg['eid']] = \ + copy.deepcopy(self.state[msg['prev_eid']]) + self.state[msg['eid']]['reload'] = msg['data'] + self.eid = msg['eid'] + serialize_env(self.state, [self.eid], env_path=self.env_path) + elif cmd == 'delete_env': + if 'eid' in msg: + logging.info('closing environment {}'.format(msg['eid'])) + del self.state[msg['eid']] + if self.env_path is not None: + p = os.path.join( + self.env_path, + "{0}.json".format(msg['eid']) + ) + os.remove(p) + broadcast_envs(self) + elif cmd == 'save_layouts': + if 'data' in msg: + self.app.layouts = msg.get('data') + self.app.save_layouts() + self.broadcast_layouts() + elif cmd == 'forward_to_vis': + packet = msg.get('data') + environment = self.state[packet['eid']] + if packet.get('pane_data') is not False: + packet['pane_data'] = environment['jsons'][packet['target']] + send_to_sources(self, msg.get('data')) + elif cmd == 'layout_item_update': + eid = msg.get('eid') + win = msg.get('win') + self.state[eid]['reload'][win] = msg.get('data') + elif cmd == 'pop_embeddings_pane': + packet = msg.get('data') + eid = packet['eid'] + win = packet['target'] + p = self.state[eid]['jsons'][win] + p['content']['selected'] = None + p['content']['data'] = p['old_content'].pop() + if len(p['old_content']) == 0: + p['content']['has_previous'] = False + p['contentID'] = get_rand_id() + broadcast(self, p, eid) + + def on_close(self): + if self in list(self.subs.values()): + self.subs.pop(self.sid, None) + + +# TODO condense some of the functionality between this class and the +# original SocketHandler class +class ClientSocketWrapper(): + """ + Wraps all of the socket actions in regular request handling, thus + allowing all of the same information to be sent via a polling interface + """ + def __init__(self, app): + self.port = app.port + self.env_path = app.env_path + self.app = app + self.state = app.state + self.subs = app.subs + self.sources = app.sources + self.readonly = app.readonly + self.login_enabled = app.login_enabled + self.messages = [] + self.last_read_time = time.time() + self.open() + try: + if not self.app.socket_wrap_monitor.is_running(): + self.app.socket_wrap_monitor.start() + except AttributeError: + self.app.socket_wrap_monitor = tornado.ioloop.PeriodicCallback( + self.socket_wrap_monitor_thread, 15000 + ) + self.app.socket_wrap_monitor.start() + + def socket_wrap_monitor_thread(self): + # TODO mark wrapped subs and sources separately + if len(self.subs) > 0 or len(self.sources) > 0: + for sub in list(self.subs.values()): + if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: + sub.close() + for sub in list(self.sources.values()): + if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: + sub.close() + else: + self.app.socket_wrap_monitor.stop() + + def broadcast_layouts(self, target_subs=None): + if target_subs is None: + target_subs = self.subs.values() + for sub in target_subs: + sub.write_message(json.dumps( + {'command': 'layout_update', 'data': self.app.layouts} + )) + + def open(self): + if self.login_enabled and not self.current_user: + print("AUTH Failed in SocketHandler") + self.close() + return + self.sid = get_rand_id() + if self not in list(self.subs.values()): + self.eid = 'main' + self.subs[self.sid] = self + logging.info('Mocking new socket: {}'.format(self.sid)) + + self.write_message( + json.dumps({'command': 'register', 'data': self.sid, + 'readonly': self.readonly})) + self.broadcast_layouts([self]) + broadcast_envs(self, [self]) + + def on_message(self, message): + logging.info('from web client: {}'.format(message)) + msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) + + cmd = msg.get('cmd') + + if self.readonly: + return + + if cmd == 'close': + if 'data' in msg and 'eid' in msg: + logging.info('closing window {}'.format(msg['data'])) + p_data = self.state[msg['eid']]['jsons'].pop(msg['data'], None) + event = { + 'event_type': 'close', + 'target': msg['data'], + 'eid': msg['eid'], + 'pane_data': p_data, + } + send_to_sources(self, event) + elif cmd == 'save': + # save localStorage window metadata + if 'data' in msg and 'eid' in msg: + msg['eid'] = escape_eid(msg['eid']) + self.state[msg['eid']] = \ + copy.deepcopy(self.state[msg['prev_eid']]) + self.state[msg['eid']]['reload'] = msg['data'] + self.eid = msg['eid'] + serialize_env(self.state, [self.eid], env_path=self.env_path) + elif cmd == 'delete_env': + if 'eid' in msg: + logging.info('closing environment {}'.format(msg['eid'])) + del self.state[msg['eid']] + if self.env_path is not None: + p = os.path.join( + self.env_path, + "{0}.json".format(msg['eid']) + ) + os.remove(p) + broadcast_envs(self) + elif cmd == 'save_layouts': + if 'data' in msg: + self.app.layouts = msg.get('data') + self.app.save_layouts() + self.broadcast_layouts() + elif cmd == 'forward_to_vis': + packet = msg.get('data') + environment = self.state[packet['eid']] + packet['pane_data'] = environment['jsons'][packet['target']] + send_to_sources(self, msg.get('data')) + elif cmd == 'layout_item_update': + eid = msg.get('eid') + win = msg.get('win') + self.state[eid]['reload'][win] = msg.get('data') + + def close(self): + if self in list(self.subs.values()): + self.subs.pop(self.sid, None) + + def write_message(self, msg): + self.messages.append(msg) + + def get_messages(self): + to_send = [] + while len(self.messages) > 0: + message = self.messages.pop() + if type(message) is dict: + # Not all messages are being formatted the same way (JSON) + # TODO investigate + message = json.dumps(message) + to_send.append(message) + self.last_read_time = time.time() + return to_send + + +class SocketWrap(BaseHandler): + def initialize(self, app): + self.state = app.state + self.subs = app.subs + self.sources = app.sources + self.port = app.port + self.env_path = app.env_path + self.login_enabled = app.login_enabled + self.app = app + + @check_auth + def post(self): + """Either write a message to the socket, or query what's there""" + # TODO formalize failure reasons + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + type = args.get('message_type') + sid = args.get('sid') + socket_wrap = self.subs.get(sid) + # ensure a wrapper still exists for this connection + if socket_wrap is None: + self.write(json.dumps({'success': False, 'reason': 'closed'})) + return + + # handle the requests + if type == 'query': + messages = socket_wrap.get_messages() + self.write(json.dumps({ + 'success': True, 'messages': messages + })) + elif type == 'send': + msg = args.get('message') + if msg is None: + self.write(json.dumps({'success': False, 'reason': 'no msg'})) + else: + socket_wrap.on_message(msg) + self.write(json.dumps({'success': True})) + else: + self.write(json.dumps({'success': False, 'reason': 'invalid'})) + + @check_auth + def get(self): + """Create a new socket wrapper for this requester, return the id""" + new_sub = ClientSocketWrapper(self.app) + self.write(json.dumps({'success': True, 'sid': new_sub.sid})) + + +# TODO refactor socket wrappers to one class +class VisSocketWrap(BaseHandler): + def initialize(self, app): + self.state = app.state + self.subs = app.subs + self.sources = app.sources + self.port = app.port + self.env_path = app.env_path + self.login_enabled = app.login_enabled + self.app = app + + @check_auth + def post(self): + """Either write a message to the socket, or query what's there""" + # TODO formalize failure reasons + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + type = args.get('message_type') + sid = args.get('sid') + + if sid is None: + new_sub = VisSocketWrapper(self.app) + self.write(json.dumps({'success': True, 'sid': new_sub.sid})) + return + + socket_wrap = self.sources.get(sid) + # ensure a wrapper still exists for this connection + if socket_wrap is None: + self.write(json.dumps({'success': False, 'reason': 'closed'})) + return + + # handle the requests + if type == 'query': + messages = socket_wrap.get_messages() + self.write(json.dumps({ + 'success': True, 'messages': messages + })) + elif type == 'send': + msg = args.get('message') + if msg is None: + self.write(json.dumps({'success': False, 'reason': 'no msg'})) + else: + socket_wrap.on_message(msg) + self.write(json.dumps({'success': True})) + else: + self.write(json.dumps({'success': False, 'reason': 'invalid'})) diff --git a/py/visdom/server/handlers/all_handlers.py b/py/visdom/server/handlers/web_handlers.py similarity index 55% rename from py/visdom/server/handlers/all_handlers.py rename to py/visdom/server/handlers/web_handlers.py index ecf07fde..464a2d42 100644 --- a/py/visdom/server/handlers/all_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -6,12 +6,17 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -"""Server""" +""" +Handlers for the different types of web request events. Mostly handles parsing +and processing the web events themselves and interfacing with the server as +necessary, but defers underlying manipulations of the server's data to +the data_model itself. +""" # TODO fix these imports from visdom.utils.shared_utils import * from visdom.utils.server_utils import * -from visdom.server.handlers.base_handlers import * +from visdom.server.handlers.base_handlers import BaseHandler import copy import getpass import hashlib @@ -30,385 +35,17 @@ # for python 3.7 and below from collections import Mapping, Sequence -import tornado.ioloop # noqa E402: gotta install ioloop first -import tornado.web # noqa E402: gotta install ioloop first -import tornado.websocket # noqa E402: gotta install ioloop first -import tornado.escape # noqa E402: gotta install ioloop first - -LAYOUT_FILE = 'layouts.json' - -here = os.path.abspath(os.path.dirname(__file__)) -COMPACT_SEPARATORS = (',', ':') +import tornado.escape MAX_SOCKET_WAIT = 15 -# TODO Split this file up it's terrible - -class VisSocketHandler(BaseWebSocketHandler): - def initialize(self, app): - self.state = app.state - self.subs = app.subs - self.sources = app.sources - self.port = app.port - self.env_path = app.env_path - self.login_enabled = app.login_enabled - - def check_origin(self, origin): - return True - - def open(self): - if self.login_enabled and not self.current_user: - self.close() - return - self.sid = str(hex(int(time.time() * 10000000))[2:]) - if self not in list(self.sources.values()): - self.eid = 'main' - self.sources[self.sid] = self - logging.info('Opened visdom socket from ip: {}'.format( - self.request.remote_ip)) - - self.write_message( - json.dumps({'command': 'alive', 'data': 'vis_alive'})) - - def on_message(self, message): - logging.info('from visdom client: {}'.format(message)) - msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) - - cmd = msg.get('cmd') - if cmd == 'echo': - for sub in self.sources.values(): - sub.write_message(json.dumps(msg)) - - def on_close(self): - if self in list(self.sources.values()): - self.sources.pop(self.sid, None) - - -class VisSocketWrapper(): - def __init__(self, app): - self.state = app.state - self.subs = app.subs - self.sources = app.sources - self.port = app.port - self.env_path = app.env_path - self.login_enabled = app.login_enabled - self.app = app - self.messages = [] - self.last_read_time = time.time() - self.open() - try: - if not self.app.socket_wrap_monitor.is_running(): - self.app.socket_wrap_monitor.start() - except AttributeError: - self.app.socket_wrap_monitor = tornado.ioloop.PeriodicCallback( - self.socket_wrap_monitor_thread, 15000 - ) - self.app.socket_wrap_monitor.start() - - # TODO refactor the two socket wrappers into a wrapper class - def socket_wrap_monitor_thread(self): - if len(self.subs) > 0 or len(self.sources) > 0: - for sub in list(self.subs.values()): - if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: - sub.close() - for sub in list(self.sources.values()): - if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: - sub.close() - else: - self.app.socket_wrap_monitor.stop() - - def open(self): - if self.login_enabled and not self.current_user: - print("AUTH Failed in SocketHandler") - self.close() - return - self.sid = get_rand_id() - if self not in list(self.sources.values()): - self.eid = 'main' - self.sources[self.sid] = self - logging.info('Mocking visdom socket: {}'.format(self.sid)) - - self.write_message( - json.dumps({'command': 'alive', 'data': 'vis_alive'})) - - def on_message(self, message): - logging.info('from visdom client: {}'.format(message)) - msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) - - cmd = msg.get('cmd') - if cmd == 'echo': - for sub in self.sources.values(): - sub.write_message(json.dumps(msg)) - - def close(self): - if self in list(self.sources.values()): - self.sources.pop(self.sid, None) - - def write_message(self, msg): - self.messages.append(msg) - - def get_messages(self): - to_send = [] - while len(self.messages) > 0: - message = self.messages.pop() - if isinstance(message, dict): - # Not all messages are being formatted the same way (JSON) - # TODO investigate - message = json.dumps(message) - to_send.append(message) - self.last_read_time = time.time() - return to_send - - -class SocketHandler(BaseWebSocketHandler): - def initialize(self, app): - self.port = app.port - self.env_path = app.env_path - self.app = app - self.state = app.state - self.subs = app.subs - self.sources = app.sources - self.broadcast_layouts() - self.readonly = app.readonly - self.login_enabled = app.login_enabled - - def check_origin(self, origin): - return True - - def broadcast_layouts(self, target_subs=None): - if target_subs is None: - target_subs = self.subs.values() - for sub in target_subs: - sub.write_message(json.dumps( - {'command': 'layout_update', 'data': self.app.layouts} - )) - - def open(self): - if self.login_enabled and not self.current_user: - print("AUTH Failed in SocketHandler") - self.close() - return - self.sid = get_rand_id() - if self not in list(self.subs.values()): - self.eid = 'main' - self.subs[self.sid] = self - logging.info( - 'Opened new socket from ip: {}'.format(self.request.remote_ip)) - - self.write_message( - json.dumps({'command': 'register', 'data': self.sid, - 'readonly': self.readonly})) - self.broadcast_layouts([self]) - broadcast_envs(self, [self]) - - def on_message(self, message): - logging.info('from web client: {}'.format(message)) - msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) - - cmd = msg.get('cmd') - - if self.readonly: - return - - if cmd == 'close': - if 'data' in msg and 'eid' in msg: - logging.info('closing window {}'.format(msg['data'])) - p_data = self.state[msg['eid']]['jsons'].pop(msg['data'], None) - event = { - 'event_type': 'close', - 'target': msg['data'], - 'eid': msg['eid'], - 'pane_data': p_data, - } - send_to_sources(self, event) - elif cmd == 'save': - # save localStorage window metadata - if 'data' in msg and 'eid' in msg: - msg['eid'] = escape_eid(msg['eid']) - self.state[msg['eid']] = \ - copy.deepcopy(self.state[msg['prev_eid']]) - self.state[msg['eid']]['reload'] = msg['data'] - self.eid = msg['eid'] - serialize_env(self.state, [self.eid], env_path=self.env_path) - elif cmd == 'delete_env': - if 'eid' in msg: - logging.info('closing environment {}'.format(msg['eid'])) - del self.state[msg['eid']] - if self.env_path is not None: - p = os.path.join( - self.env_path, - "{0}.json".format(msg['eid']) - ) - os.remove(p) - broadcast_envs(self) - elif cmd == 'save_layouts': - if 'data' in msg: - self.app.layouts = msg.get('data') - self.app.save_layouts() - self.broadcast_layouts() - elif cmd == 'forward_to_vis': - packet = msg.get('data') - environment = self.state[packet['eid']] - if packet.get('pane_data') is not False: - packet['pane_data'] = environment['jsons'][packet['target']] - send_to_sources(self, msg.get('data')) - elif cmd == 'layout_item_update': - eid = msg.get('eid') - win = msg.get('win') - self.state[eid]['reload'][win] = msg.get('data') - elif cmd == 'pop_embeddings_pane': - packet = msg.get('data') - eid = packet['eid'] - win = packet['target'] - p = self.state[eid]['jsons'][win] - p['content']['selected'] = None - p['content']['data'] = p['old_content'].pop() - if len(p['old_content']) == 0: - p['content']['has_previous'] = False - p['contentID'] = get_rand_id() - broadcast(self, p, eid) - - def on_close(self): - if self in list(self.subs.values()): - self.subs.pop(self.sid, None) - - -# TODO condense some of the functionality between this class and the -# original SocketHandler class -class ClientSocketWrapper(): - """ - Wraps all of the socket actions in regular request handling, thus - allowing all of the same information to be sent via a polling interface - """ - def __init__(self, app): - self.port = app.port - self.env_path = app.env_path - self.app = app - self.state = app.state - self.subs = app.subs - self.sources = app.sources - self.readonly = app.readonly - self.login_enabled = app.login_enabled - self.messages = [] - self.last_read_time = time.time() - self.open() - try: - if not self.app.socket_wrap_monitor.is_running(): - self.app.socket_wrap_monitor.start() - except AttributeError: - self.app.socket_wrap_monitor = tornado.ioloop.PeriodicCallback( - self.socket_wrap_monitor_thread, 15000 - ) - self.app.socket_wrap_monitor.start() - - def socket_wrap_monitor_thread(self): - # TODO mark wrapped subs and sources separately - if len(self.subs) > 0 or len(self.sources) > 0: - for sub in list(self.subs.values()): - if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: - sub.close() - for sub in list(self.sources.values()): - if time.time() - sub.last_read_time > MAX_SOCKET_WAIT: - sub.close() - else: - self.app.socket_wrap_monitor.stop() - - def broadcast_layouts(self, target_subs=None): - if target_subs is None: - target_subs = self.subs.values() - for sub in target_subs: - sub.write_message(json.dumps( - {'command': 'layout_update', 'data': self.app.layouts} - )) - - def open(self): - self.sid = get_rand_id() - if self not in list(self.subs.values()): - self.eid = 'main' - self.subs[self.sid] = self - logging.info('Mocking new socket: {}'.format(self.sid)) - - self.write_message( - json.dumps({'command': 'register', 'data': self.sid, - 'readonly': self.readonly})) - self.broadcast_layouts([self]) - broadcast_envs(self, [self]) - - def on_message(self, message): - logging.info('from web client: {}'.format(message)) - msg = tornado.escape.json_decode(tornado.escape.to_basestring(message)) - - cmd = msg.get('cmd') - - if self.readonly: - return - - if cmd == 'close': - if 'data' in msg and 'eid' in msg: - logging.info('closing window {}'.format(msg['data'])) - p_data = self.state[msg['eid']]['jsons'].pop(msg['data'], None) - event = { - 'event_type': 'close', - 'target': msg['data'], - 'eid': msg['eid'], - 'pane_data': p_data, - } - send_to_sources(self, event) - elif cmd == 'save': - # save localStorage window metadata - if 'data' in msg and 'eid' in msg: - msg['eid'] = escape_eid(msg['eid']) - self.state[msg['eid']] = \ - copy.deepcopy(self.state[msg['prev_eid']]) - self.state[msg['eid']]['reload'] = msg['data'] - self.eid = msg['eid'] - serialize_env(self.state, [self.eid], env_path=self.env_path) - elif cmd == 'delete_env': - if 'eid' in msg: - logging.info('closing environment {}'.format(msg['eid'])) - del self.state[msg['eid']] - if self.env_path is not None: - p = os.path.join( - self.env_path, - "{0}.json".format(msg['eid']) - ) - os.remove(p) - broadcast_envs(self) - elif cmd == 'save_layouts': - if 'data' in msg: - self.app.layouts = msg.get('data') - self.app.save_layouts() - self.broadcast_layouts() - elif cmd == 'forward_to_vis': - packet = msg.get('data') - environment = self.state[packet['eid']] - packet['pane_data'] = environment['jsons'][packet['target']] - send_to_sources(self, msg.get('data')) - elif cmd == 'layout_item_update': - eid = msg.get('eid') - win = msg.get('win') - self.state[eid]['reload'][win] = msg.get('data') - - def close(self): - if self in list(self.subs.values()): - self.subs.pop(self.sid, None) - - def write_message(self, msg): - self.messages.append(msg) - - def get_messages(self): - to_send = [] - while len(self.messages) > 0: - message = self.messages.pop() - if isinstance(message, dict): - # Not all messages are being formatted the same way (JSON) - # TODO investigate - message = json.dumps(message) - to_send.append(message) - self.last_read_time = time.time() - return to_send - +# TODO move the logic that actually parses environments and layouts to +# new classes in the data_model folder. +# TODO move generalized initialization logic from these handlers into the +# basehandler +# TODO abstract out any direct references to the app where possible from +# all handlers. Can instead provide accessor functions on the state? class PostHandler(BaseHandler): def initialize(self, app): self.state = app.state @@ -417,13 +54,6 @@ def initialize(self, app): self.port = app.port self.env_path = app.env_path self.login_enabled = app.login_enabled - self.handlers = { - 'update': UpdateHandler, - 'save': SaveHandler, - 'close': CloseHandler, - 'win_exists': ExistsHandler, - 'delete_env': DeleteEnvHandler, - } @check_auth def post(self): @@ -743,103 +373,6 @@ def post(self): self.wrap_func(self, args) -class SocketWrap(BaseHandler): - def initialize(self, app): - self.state = app.state - self.subs = app.subs - self.sources = app.sources - self.port = app.port - self.env_path = app.env_path - self.login_enabled = app.login_enabled - self.app = app - - @check_auth - def post(self): - """Either write a message to the socket, or query what's there""" - # TODO formalize failure reasons - args = tornado.escape.json_decode( - tornado.escape.to_basestring(self.request.body) - ) - type = args.get('message_type') - sid = args.get('sid') - socket_wrap = self.subs.get(sid) - # ensure a wrapper still exists for this connection - if socket_wrap is None: - self.write(json.dumps({'success': False, 'reason': 'closed'})) - return - - # handle the requests - if type == 'query': - messages = socket_wrap.get_messages() - self.write(json.dumps({ - 'success': True, 'messages': messages - })) - elif type == 'send': - msg = args.get('message') - if msg is None: - self.write(json.dumps({'success': False, 'reason': 'no msg'})) - else: - socket_wrap.on_message(msg) - self.write(json.dumps({'success': True})) - else: - self.write(json.dumps({'success': False, 'reason': 'invalid'})) - - @check_auth - def get(self): - """Create a new socket wrapper for this requester, return the id""" - new_sub = ClientSocketWrapper(self.app) - self.write(json.dumps({'success': True, 'sid': new_sub.sid})) - - -# TODO refactor socket wrappers to one class -class VisSocketWrap(BaseHandler): - def initialize(self, app): - self.state = app.state - self.subs = app.subs - self.sources = app.sources - self.port = app.port - self.env_path = app.env_path - self.login_enabled = app.login_enabled - self.app = app - - @check_auth - def post(self): - """Either write a message to the socket, or query what's there""" - # TODO formalize failure reasons - args = tornado.escape.json_decode( - tornado.escape.to_basestring(self.request.body) - ) - type = args.get('message_type') - sid = args.get('sid') - - if sid is None: - new_sub = VisSocketWrapper(self.app) - self.write(json.dumps({'success': True, 'sid': new_sub.sid})) - return - - socket_wrap = self.sources.get(sid) - # ensure a wrapper still exists for this connection - if socket_wrap is None: - self.write(json.dumps({'success': False, 'reason': 'closed'})) - return - - # handle the requests - if type == 'query': - messages = socket_wrap.get_messages() - self.write(json.dumps({ - 'success': True, 'messages': messages - })) - elif type == 'send': - msg = args.get('message') - if msg is None: - self.write(json.dumps({'success': False, 'reason': 'no msg'})) - else: - socket_wrap.on_message(msg) - self.write(json.dumps({'success': True})) - else: - self.write(json.dumps({'success': False, 'reason': 'invalid'})) - - class DeleteEnvHandler(BaseHandler): def initialize(self, app): self.state = app.state diff --git a/py/visdom/server/run_server.py b/py/visdom/server/run_server.py index 0f6e90c3..91296ad4 100644 --- a/py/visdom/server/run_server.py +++ b/py/visdom/server/run_server.py @@ -6,6 +6,10 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. +""" +Provides simple entrypoints to set up and run the main visdom server. +""" + from visdom.server.app import Application from visdom.server.defaults import ( DEFAULT_BASE_URL, diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index bd16d6a6..ee33de48 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -60,12 +60,13 @@ def check_auth(f): Wrapper for server access methods to ensure that the access is authorized. """ - def _check_auth(app, *args, **kwargs): - app.last_access = time.time() - if app.login_enabled and not app.current_user: - app.set_status(400) + def _check_auth(handler, *args, **kwargs): + # TODO this should call a shared method of the handler + handler.last_access = time.time() + if handler.login_enabled and not handler.current_user: + handler.set_status(400) return - f(app, *args, **kwargs) + f(handler, *args, **kwargs) return _check_auth def set_cookie(value=None): From fb3541f062759a1eebf441eb53eab4a1cd01b634 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Mon, 30 May 2022 13:58:48 +0200 Subject: [PATCH 03/15] reapply all changes since server refactor --- py/visdom/server/__main__.py | 2 +- py/visdom/server/app.py | 67 ++++++++++++++++---- py/visdom/server/build.py | 44 +++++++++++-- py/visdom/server/defaults.py | 2 +- py/visdom/server/handlers/base_handlers.py | 2 +- py/visdom/server/handlers/socket_handlers.py | 10 +-- py/visdom/server/handlers/web_handlers.py | 4 -- py/visdom/server/run_server.py | 29 ++++++--- py/visdom/utils/server_utils.py | 59 +++++++++++++++-- py/visdom/utils/shared_utils.py | 14 ++-- 10 files changed, 180 insertions(+), 53 deletions(-) diff --git a/py/visdom/server/__main__.py b/py/visdom/server/__main__.py index aa1195d5..e68acbd7 100644 --- a/py/visdom/server/__main__.py +++ b/py/visdom/server/__main__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 7f64fef3..afaea82c 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the @@ -21,6 +21,7 @@ from visdom.server.handlers.socket_handlers import * from visdom.server.handlers.web_handlers import * +import platform import copy import hashlib import logging @@ -43,10 +44,13 @@ class Application(tornado.web.Application): def __init__(self, port=DEFAULT_PORT, base_url='', env_path=DEFAULT_ENV_PATH, readonly=False, - user_credential=None, use_frontend_client_polling=False): + user_credential=None, use_frontend_client_polling=False, + eager_data_loading=False): + self.eager_data_loading = eager_data_loading self.env_path = env_path self.state = self.load_state() self.layouts = self.load_layouts() + self.user_settings = self.load_user_settings() self.subs = {} self.sources = {} self.port = port @@ -86,6 +90,7 @@ def __init__(self, port=DEFAULT_PORT, base_url='', (r"%s/win_hash" % self.base_url, HashHandler, {'app': self}), (r"%s/env_state" % self.base_url, EnvStateHandler, {'app': self}), (r"%s/fork_env" % self.base_url, ForkEnvHandler, {'app': self}), + (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {'app': self}), (r"%s(.*)" % self.base_url, IndexHandler, {'app': self}), ] super(Application, self).__init__(handlers, **tornado_settings) @@ -137,24 +142,58 @@ def load_state(self): return {'main': {'jsons': {}, 'reload': {}}} ensure_dir_exists(env_path) env_jsons = [i for i in os.listdir(env_path) if '.json' in i] - for env_json in env_jsons: + eid = env_json.replace('.json', '') env_path_file = os.path.join(env_path, env_json) - try: - with open(env_path_file, 'r') as fn: - env_data = tornado.escape.json_decode(fn.read()) - except Exception as e: - logging.warn( - "Failed loading environment json: {} - {}".format( - env_path_file, repr(e))) - continue - eid = env_json.replace('.json', '') - state[eid] = {'jsons': env_data['jsons'], - 'reload': env_data['reload']} + if self.eager_data_loading: + try: + with open(env_path_file, 'r') as fn: + env_data = tornado.escape.json_decode(fn.read()) + except Exception as e: + logging.warn( + "Failed loading environment json: {} - {}".format( + env_path_file, repr(e))) + continue + + state[eid] = {'jsons': env_data['jsons'], + 'reload': env_data['reload']} + else: + state[eid] = LazyEnvData(env_path_file) if 'main' not in state and 'main.json' not in env_jsons: state['main'] = {'jsons': {}, 'reload': {}} serialize_env(state, ['main'], env_path=self.env_path) return state + + def load_user_settings(self): + settings = {} + + """Determines & uses the platform-specific root directory for user configurations.""" + if platform.system() == "Windows": + base_dir = os.getenv('APPDATA') + elif platform.system() == "Darwin": # osx + base_dir = os.path.expanduser('~/Library/Preferences') + else: + base_dir = os.getenv('XDG_CONFIG_HOME', os.path.expanduser('~/.config')) + config_dir = os.path.join(base_dir, "visdom") + + # initialize user style + user_css = "" + logging.error("initializing") + home_style_path = os.path.join(config_dir, "style.css") + if os.path.exists(home_style_path): + with open(home_style_path, "r") as f: + user_css += "\n" + f.read() + project_style_path = os.path.join(self.env_path, "style.css") + if os.path.exists(project_style_path): + with open(project_style_path, "r") as f: + user_css += "\n" + f.read() + + settings['config_dir'] = config_dir + settings['user_css'] = user_css + + return settings + + diff --git a/py/visdom/server/build.py b/py/visdom/server/build.py index 9ca41890..78cc6b83 100644 --- a/py/visdom/server/build.py +++ b/py/visdom/server/build.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. +import logging import visdom from visdom.utils.shared_utils import ensure_dir_exists, get_visdom_path import os @@ -36,13 +37,20 @@ def download_scripts(proxies=None, install_dir=None): 'react-dom.min.js', '%sreact-modal@3.1.10/dist/react-modal.min.js' % b: 'react-modal.min.js', - 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_SVG': # noqa - 'mathjax-MathJax.js', # here is another url in case the cdn breaks down again. # https://raw.githubusercontent.com/plotly/plotly.js/master/dist/plotly.min.js - 'https://cdn.plot.ly/plotly-latest.min.js': 'plotly-plotly.min.js', + ## [shouldsee/visdom/package_version]:latest.min.js not pointing to latest. + ## see https://github.com/plotly/plotly.py/issues/3651 + 'https://cdn.plot.ly/plotly-2.11.1.min.js': 'plotly-plotly.min.js', + # Stanford Javascript Crypto Library for Password Hashing '%ssjcl@1.0.7/sjcl.js' % b: 'sjcl.js', + '%slayout-bin-packer@1.4.0/dist/layout-bin-packer.js.map' % b: 'layout-bin-packer.js.map', + # d3 Libraries for plotting d3 graphs! + 'http://d3js.org/d3.v3.min.js' : 'd3.v3.min.js', + 'https://d3js.org/d3-selection-multi.v1.js' : 'd3-selection-multi.v1.js', + # Library to download the svg to png + '%ssave-svg-as-png@1.4.17/lib/saveSvgAsPng.js' % b: 'saveSvgAsPng.js', # - css '%sreact-resizable@1.4.6/css/styles.css' % b: @@ -101,7 +109,7 @@ def download_scripts(proxies=None, install_dir=None): for (key, val) in ext_files.items(): # set subdirectory: - if val.endswith('.js'): + if val.endswith('.js') or val.endswith('.js.map'): sub_dir = 'js' elif val.endswith('.css'): sub_dir = 'css' @@ -124,6 +132,32 @@ def download_scripts(proxies=None, install_dir=None): logging.error('Error {} while downloading {}'.format( exc.reason, key)) + # Download MathJax Js Files + import requests + cdnjs_url = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/' + mathjax_dir = os.path.join(*cdnjs_url.split('/')[-3:]) + mathjax_path = [ + 'config/Safe.js?V=2.7.5', + 'config/TeX-AMS-MML_HTMLorMML.js?V=2.7.5', + 'extensions/Safe.js?V=2.7.5', + 'jax/output/SVG/fonts/TeX/fontdata.js?V=2.7.5', + 'jax/output/SVG/jax.js?V=2.7.5', + 'jax/output/SVG/fonts/TeX/Size1/Regular/Main.js?V=2.7.5', + 'jax/output/SVG/config.js?V=2.7.5', + 'MathJax.js?config=TeX-AMS-MML_HTMLorMML%2CSafe.js&ver=4.1', + ] + mathjax_dir_path = '%s/static/%s/%s' % (install_dir, 'js', mathjax_dir) + + for path in mathjax_path: + filename = path.split("/")[-1].split("?")[0] + extracted_directory = os.path.join(mathjax_dir_path, *path.split('/')[:-1]) + if not os.path.exists(extracted_directory): + os.makedirs(extracted_directory) + if not os.path.exists(os.path.join(extracted_directory, filename)): + js_file = requests.get(cdnjs_url + path) + with open(os.path.join(extracted_directory, filename), "wb+") as file: + file.write(js_file.content) + if not is_built: with open(built_path, 'w+') as build_file: build_file.write(visdom.__version__) diff --git a/py/visdom/server/defaults.py b/py/visdom/server/defaults.py index 99957ef5..d3f95852 100644 --- a/py/visdom/server/defaults.py +++ b/py/visdom/server/defaults.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the diff --git a/py/visdom/server/handlers/base_handlers.py b/py/visdom/server/handlers/base_handlers.py index 21607999..14e6c5d6 100644 --- a/py/visdom/server/handlers/base_handlers.py +++ b/py/visdom/server/handlers/base_handlers.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the diff --git a/py/visdom/server/handlers/socket_handlers.py b/py/visdom/server/handlers/socket_handlers.py index d3d1c891..6e5f5281 100644 --- a/py/visdom/server/handlers/socket_handlers.py +++ b/py/visdom/server/handlers/socket_handlers.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the @@ -156,7 +156,7 @@ def get_messages(self): to_send = [] while len(self.messages) > 0: message = self.messages.pop() - if type(message) is dict: + if isinstance(message, dict): # Not all messages are being formatted the same way (JSON) # TODO investigate message = json.dumps(message) @@ -327,10 +327,6 @@ def broadcast_layouts(self, target_subs=None): )) def open(self): - if self.login_enabled and not self.current_user: - print("AUTH Failed in SocketHandler") - self.close() - return self.sid = get_rand_id() if self not in list(self.subs.values()): self.eid = 'main' @@ -409,7 +405,7 @@ def get_messages(self): to_send = [] while len(self.messages) > 0: message = self.messages.pop() - if type(message) is dict: + if isinstance(message, dict): # Not all messages are being formatted the same way (JSON) # TODO investigate message = json.dumps(message) diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index 464a2d42..e49a6efa 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -19,15 +19,12 @@ from visdom.server.handlers.base_handlers import BaseHandler import copy import getpass -import hashlib import json import jsonpatch import logging import math import os -import time from collections import OrderedDict -from collections.abc import Mapping try: # for after python 3.8 from collections.abc import Mapping, Sequence @@ -37,7 +34,6 @@ import tornado.escape -MAX_SOCKET_WAIT = 15 # TODO move the logic that actually parses environments and layouts to diff --git a/py/visdom/server/run_server.py b/py/visdom/server/run_server.py index 91296ad4..3b09a474 100644 --- a/py/visdom/server/run_server.py +++ b/py/visdom/server/run_server.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the @@ -31,13 +31,17 @@ def start_server(port=DEFAULT_PORT, hostname=DEFAULT_HOSTNAME, base_url=DEFAULT_BASE_URL, env_path=DEFAULT_ENV_PATH, readonly=False, print_func=None, user_credential=None, - use_frontend_client_polling=False): - """Run a visdom server with the given arguments""" - logging.info("It's Alive!") + use_frontend_client_polling=False, bind_local=False, + eager_data_loading=False): + print("It's Alive!") app = Application(port=port, base_url=base_url, env_path=env_path, readonly=readonly, user_credential=user_credential, - use_frontend_client_polling=use_frontend_client_polling) - app.listen(port, max_buffer_size=1024 ** 3) + use_frontend_client_polling=use_frontend_client_polling, + eager_data_loading=eager_data_loading) + if bind_local: + app.listen(port, max_buffer_size=1024 ** 3, address='127.0.0.1') + else: + app.listen(port, max_buffer_size=1024 ** 3) logging.info("Application Started") if "HOSTNAME" in os.environ and hostname == DEFAULT_HOSTNAME: @@ -88,6 +92,13 @@ def main(print_func=None): action='store_true', help='Have the frontend communicate via polling ' 'rather than over websockets.') + parser.add_argument('-bind_local', default=False, + action='store_true', + help='Make server only accessible only from ' + 'localhost.') + parser.add_argument('-eager_data_loading', default=False, + action='store_true', + help='Load data from filesystem when starting server (and not lazily upon first request).') FLAGS = parser.parse_args() # Process base_url @@ -99,7 +110,7 @@ def main(print_func=None): try: logging_level = int(FLAGS.logging_level) - except (ValueError,): + except ValueError: try: logging_level = logging._checkLevel(FLAGS.logging_level) except ValueError: @@ -163,7 +174,9 @@ def main(print_func=None): start_server(port=FLAGS.port, hostname=FLAGS.hostname, base_url=base_url, env_path=FLAGS.env_path, readonly=FLAGS.readonly, print_func=print_func, user_credential=user_credential, - use_frontend_client_polling=FLAGS.use_frontend_client_polling) + use_frontend_client_polling=FLAGS.use_frontend_client_polling, + bind_local=FLAGS.bind_local, + eager_data_loading=FLAGS.eager_data_loading) def download_scripts_and_run(): diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index ee33de48..627b3b84 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the @@ -20,7 +20,6 @@ DEFAULT_HOSTNAME, DEFAULT_PORT, ) -from visdom.utils.shared_utils import get_new_window_id from visdom.utils.shared_utils import ( warn_once, get_rand_id, @@ -48,7 +47,6 @@ LAYOUT_FILE = 'layouts.json' -here = os.path.abspath(os.path.dirname(__file__)) COMPACT_SEPARATORS = (',', ':') MAX_SOCKET_WAIT = 15 @@ -85,13 +83,54 @@ def hash_password(password): # ------- File management helprs ----- # +class LazyEnvData(Mapping): + def __init__(self, env_path_file): + self._env_path_file = env_path_file + self._raw_dict = None + + def lazy_load_data(self): + if self._raw_dict is not None: + return + + try: + with open(self._env_path_file, 'r') as fn: + env_data = tornado.escape.json_decode(fn.read()) + except Exception as e: + raise ValueError( + "Failed loading environment json: {} - {}".format( + self._env_path_file, repr(e))) + self._raw_dict = { + 'jsons': env_data['jsons'], + 'reload': env_data['reload'] + } + + def __getitem__(self, key): + self.lazy_load_data() + return self._raw_dict.__getitem__(key) + + def __setitem__(self, key, value): + self.lazy_load_data() + return self._raw_dict.__setitem__(key, value) + + def __iter__(self): + self.lazy_load_data() + return iter(self._raw_dict) + + def __len__(self): + self.lazy_load_data() + return len(self._raw_dict) + + def serialize_env(state, eids, env_path=DEFAULT_ENV_PATH): env_ids = [i for i in eids if i in state] if env_path is not None: for env_id in env_ids: env_path_file = os.path.join(env_path, "{0}.json".format(env_id)) with open(env_path_file, 'w') as fn: - fn.write(json.dumps(state[env_id])) + if isinstance(state[env_id], LazyEnvData): + fn.write(json.dumps(state[env_id]._raw_dict)) + else: + fn.write(json.dumps(state[env_id])) return env_ids @@ -163,6 +202,14 @@ def window(args): }) elif ptype in ['image', 'text', 'properties']: p.update({'content': args['data'][0]['content'], 'type': ptype}) + elif ptype == 'network': + p.update({ + 'content': args['data'][0]['content'] , + 'type': ptype, + 'directed': opts.get("directed", False), + 'showEdgeLabels' : opts.get("showEdgeLabels", "hover"), + 'showVertexLabels' : opts.get("showVertexLabels", "hover"), + }) elif ptype in ['embeddings']: p.update({ 'content': args['data'][0]['content'], @@ -211,7 +258,7 @@ def compare_envs(state, eids, socket, env_path=DEFAULT_ENV_PATH): res['jsons'][wid] = None res['jsons'].pop(wid) - for ix, eid in enumerate(envs.keys()): + for ix, eid in enumerate(sorted(envs.keys())): env = envs[eid] for wid in env.get('jsons', {}).keys(): win = env['jsons'][wid] @@ -349,7 +396,7 @@ def load_env(state, eid, socket, env_path=DEFAULT_ENV_PATH): def broadcast(self, msg, eid): for s in self.subs: - if type(self.subs[s].eid) is list: + if isinstance(self.subs[s].eid, dict): if eid in self.subs[s].eid: self.subs[s].write_message(msg) else: diff --git a/py/visdom/utils/shared_utils.py b/py/visdom/utils/shared_utils.py index 23c164d4..90a28d1f 100644 --- a/py/visdom/utils/shared_utils.py +++ b/py/visdom/utils/shared_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2017-present, Facebook, Inc. +# Copyright 2017-present, The Visdom Authors # All rights reserved. # # This source code is licensed under the license found in the @@ -39,22 +39,24 @@ def get_rand_id(): def get_new_window_id(): """Return a string to be used for a new window""" - return f'win_{get_rand_id()}' + return f'window_{get_rand_id()}' + def ensure_dir_exists(path): """Make sure the parent dir exists for path so we can write a file.""" try: - os.makedirs(os.path.dirname(path)) + os.makedirs(os.path.dirname(os.path.abspath(path))) except OSError as e1: assert e1.errno == 17 # errno.EEXIST -def get_visdom_path(): - """Get the path to the visdom/py/visdom directory.""" +def get_visdom_path(filename): + """Get the path to an asset.""" cwd = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) - return os.path.dirname(cwd) + return os.path.join(cwd, filename) + def get_visdom_path_to(filename): From 75c63e3733b00b1fb7a337f2931e41f98fee4dd6 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Mon, 30 May 2022 15:15:18 +0200 Subject: [PATCH 04/15] server: simplify & fix get_visdom_path/_to --- py/visdom/server/app.py | 6 +++--- py/visdom/utils/shared_utils.py | 15 +++++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index afaea82c..474835c2 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -11,7 +11,7 @@ all of the required state about the currently running server. """ -from visdom.utils.shared_utils import warn_once, ensure_dir_exists, get_visdom_path_to +from visdom.utils.shared_utils import warn_once, ensure_dir_exists, get_visdom_path from visdom.utils.server_utils import ( serialize_env, @@ -36,8 +36,8 @@ tornado_settings = { "autoescape": None, "debug": "/dbg/" in __file__, - "static_path": get_visdom_path_to('static'), - "template_path": get_visdom_path_to('static'), + "static_path": get_visdom_path('static'), + "template_path": get_visdom_path('static'), "compiled_template_cache": False } diff --git a/py/visdom/utils/shared_utils.py b/py/visdom/utils/shared_utils.py index 90a28d1f..833095ac 100644 --- a/py/visdom/utils/shared_utils.py +++ b/py/visdom/utils/shared_utils.py @@ -12,7 +12,7 @@ helper functions. """ -import inspect +import importlib import uuid import warnings import os @@ -51,14 +51,9 @@ def ensure_dir_exists(path): assert e1.errno == 17 # errno.EEXIST -def get_visdom_path(filename): +def get_visdom_path(filename=None): """Get the path to an asset.""" - cwd = os.path.dirname( - os.path.abspath(inspect.getfile(inspect.currentframe()))) + cwd = os.path.dirname(importlib.util.find_spec("visdom").origin) + if filename is None: + return cwd return os.path.join(cwd, filename) - - - -def get_visdom_path_to(filename): - """Get the path to a file in the visdom/py/visdom directory.""" - return os.path.join(get_visdom_path(), filename) From 26a9ef1fc2cdc54bc0687565f64a5846ae4e68f3 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Mon, 30 May 2022 15:16:43 +0200 Subject: [PATCH 05/15] server: move LAYOUT_FILE to defaults.py --- py/visdom/server/app.py | 11 ++++++++++- py/visdom/server/defaults.py | 1 + py/visdom/utils/server_utils.py | 3 +-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 474835c2..c0c5499e 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -31,7 +31,16 @@ import tornado.web # noqa E402: gotta install ioloop first import tornado.escape # noqa E402: gotta install ioloop first -LAYOUT_FILE = 'layouts.json' + +from visdom.server.defaults import ( + LAYOUT_FILE, + DEFAULT_BASE_URL, + DEFAULT_ENV_PATH, + DEFAULT_HOSTNAME, + DEFAULT_PORT, +) + + tornado_settings = { "autoescape": None, diff --git a/py/visdom/server/defaults.py b/py/visdom/server/defaults.py index d3f95852..05384480 100644 --- a/py/visdom/server/defaults.py +++ b/py/visdom/server/defaults.py @@ -8,6 +8,7 @@ from os.path import expanduser +LAYOUT_FILE = 'layouts.json' DEFAULT_ENV_PATH = '%s/.visdom/' % expanduser("~") DEFAULT_PORT = 8097 DEFAULT_HOSTNAME = "localhost" diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index 627b3b84..d57239b7 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -15,6 +15,7 @@ """ from visdom.server.defaults import ( + LAYOUT_FILE, DEFAULT_BASE_URL, DEFAULT_ENV_PATH, DEFAULT_HOSTNAME, @@ -45,8 +46,6 @@ import tornado.escape # noqa E402: gotta install ioloop first -LAYOUT_FILE = 'layouts.json' - COMPACT_SEPARATORS = (',', ':') MAX_SOCKET_WAIT = 15 From 8bdb9c1fd9ba00b04cb3188161dfb2ecb6ddb72d Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Mon, 30 May 2022 15:17:00 +0200 Subject: [PATCH 06/15] server: cleanup of imports --- py/visdom/server/app.py | 41 +++++++++++++------- py/visdom/server/build.py | 4 +- py/visdom/server/handlers/socket_handlers.py | 8 ++-- py/visdom/server/handlers/web_handlers.py | 7 ++-- py/visdom/server/run_server.py | 18 +++++---- 5 files changed, 46 insertions(+), 32 deletions(-) diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index c0c5499e..65ea6c36 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -11,37 +11,50 @@ all of the required state about the currently running server. """ -from visdom.utils.shared_utils import warn_once, ensure_dir_exists, get_visdom_path - -from visdom.utils.server_utils import ( - serialize_env, -) - -# TODO replace this next -from visdom.server.handlers.socket_handlers import * -from visdom.server.handlers.web_handlers import * - -import platform import copy import hashlib import logging import os +import platform import time import tornado.web # noqa E402: gotta install ioloop first import tornado.escape # noqa E402: gotta install ioloop first - +from visdom.utils.shared_utils import warn_once, ensure_dir_exists, get_visdom_path +from visdom.utils.server_utils import serialize_env, LazyEnvData +from visdom.server.handlers.socket_handlers import ( + SocketHandler, + SocketWrap, + VisSocketHandler, + VisSocketWrap, +) +from visdom.server.handlers.web_handlers import ( + CloseHandler, + CompareHandler, + DataHandler, + DeleteEnvHandler, + EnvHandler, + EnvStateHandler, + ErrorHandler, + ExistsHandler, + ForkEnvHandler, + HashHandler, + IndexHandler, + PostHandler, + SaveHandler, + UpdateHandler, + UserSettingsHandler +) from visdom.server.defaults import ( - LAYOUT_FILE, DEFAULT_BASE_URL, DEFAULT_ENV_PATH, DEFAULT_HOSTNAME, DEFAULT_PORT, + LAYOUT_FILE ) - tornado_settings = { "autoescape": None, "debug": "/dbg/" in __file__, diff --git a/py/visdom/server/build.py b/py/visdom/server/build.py index 78cc6b83..c6d7c148 100644 --- a/py/visdom/server/build.py +++ b/py/visdom/server/build.py @@ -7,11 +7,11 @@ # LICENSE file in the root directory of this source tree. import logging -import visdom -from visdom.utils.shared_utils import ensure_dir_exists, get_visdom_path import os +import visdom from urllib import request from urllib.error import HTTPError, URLError +from visdom.utils.shared_utils import ensure_dir_exists, get_visdom_path def download_scripts(proxies=None, install_dir=None): diff --git a/py/visdom/server/handlers/socket_handlers.py b/py/visdom/server/handlers/socket_handlers.py index 6e5f5281..cec69dcb 100644 --- a/py/visdom/server/handlers/socket_handlers.py +++ b/py/visdom/server/handlers/socket_handlers.py @@ -13,10 +13,6 @@ the data_model itself. """ -# TODO fix these imports -from visdom.utils.shared_utils import * -from visdom.utils.server_utils import * -from visdom.server.handlers.base_handlers import * import copy import getpass import hashlib @@ -36,6 +32,10 @@ import tornado.ioloop import tornado.escape +from visdom.server.handlers.base_handlers import BaseWebSocketHandler, BaseHandler +from visdom.utils.shared_utils import get_rand_id +from visdom.utils.server_utils import check_auth, broadcast_envs, serialize_env, send_to_sources, broadcast, escape_eid + MAX_SOCKET_WAIT = 15 diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index e49a6efa..371989ed 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -13,10 +13,6 @@ the data_model itself. """ -# TODO fix these imports -from visdom.utils.shared_utils import * -from visdom.utils.server_utils import * -from visdom.server.handlers.base_handlers import BaseHandler import copy import getpass import json @@ -33,6 +29,9 @@ from collections import Mapping, Sequence import tornado.escape +from visdom.utils.shared_utils import get_rand_id +from visdom.utils.server_utils import check_auth, extract_eid, window, register_window, gather_envs, broadcast_envs, serialize_env, escape_eid, compare_envs, load_env, hash_md_window, broadcast, update_window, hash_password, stringify +from visdom.server.handlers.base_handlers import BaseHandler diff --git a/py/visdom/server/run_server.py b/py/visdom/server/run_server.py index 3b09a474..ca739796 100644 --- a/py/visdom/server/run_server.py +++ b/py/visdom/server/run_server.py @@ -10,6 +10,12 @@ Provides simple entrypoints to set up and run the main visdom server. """ +import argparse +import getpass +import logging +import os +import sys +from tornado import ioloop from visdom.server.app import Application from visdom.server.defaults import ( DEFAULT_BASE_URL, @@ -18,14 +24,10 @@ DEFAULT_PORT, ) from visdom.server.build import download_scripts - -import argparse -import getpass -import logging -import os -import sys - -from tornado import ioloop +from visdom.utils.server_utils import ( + hash_password, + set_cookie +) def start_server(port=DEFAULT_PORT, hostname=DEFAULT_HOSTNAME, From 277bcbf7e2437210c52174026097b72010c5664e Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 20:02:26 +0200 Subject: [PATCH 07/15] setup: adapt console_script to new file structure --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17852bc1..8fea70a2 100644 --- a/setup.py +++ b/setup.py @@ -70,5 +70,5 @@ def get_dist(pkgname): include_package_data=True, zip_safe=False, install_requires=requirements, - entry_points={'console_scripts': ['visdom=visdom.server:download_scripts_and_run']} + entry_points={'console_scripts': ['visdom=visdom.server.run_server:download_scripts_and_run']} ) From d3ce2d3633ae27141286651e6b4de198c34484ad Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 20:03:07 +0200 Subject: [PATCH 08/15] server: remove zmq (as it has not been used & zmq.ioloop is deprecated) --- py/visdom/utils/server_utils.py | 4 ---- setup.py | 1 - 2 files changed, 5 deletions(-) diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index d57239b7..68eaa5fe 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -41,13 +41,9 @@ # for python 3.7 and below from collections import Mapping, Sequence -from zmq.eventloop import ioloop -ioloop.install() # Needs to happen before any tornado imports! - import tornado.escape # noqa E402: gotta install ioloop first COMPACT_SEPARATORS = (',', ':') - MAX_SOCKET_WAIT = 15 # ---- Vaguely server-security related functions ---- # diff --git a/setup.py b/setup.py index 8fea70a2..8b0c570f 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ def get_dist(pkgname): 'scipy', 'requests', 'tornado', - # 'pyzmq', 'six', 'jsonpatch', 'websocket-client', From b3d838a853f524d5a0da6e59fac14d677eb10588 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 20:09:44 +0200 Subject: [PATCH 09/15] server: (minor) changed order of imports --- py/visdom/utils/server_utils.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index 68eaa5fe..b6fcfc47 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -14,25 +14,14 @@ in the previous server.py class. """ -from visdom.server.defaults import ( - LAYOUT_FILE, - DEFAULT_BASE_URL, - DEFAULT_ENV_PATH, - DEFAULT_HOSTNAME, - DEFAULT_PORT, -) -from visdom.utils.shared_utils import ( - warn_once, - get_rand_id, - get_new_window_id, - ensure_dir_exists, -) + import copy import hashlib import json import logging import os import time +import tornado.escape from collections import OrderedDict try: # for after python 3.8 @@ -40,10 +29,20 @@ except ImportError: # for python 3.7 and below from collections import Mapping, Sequence +from visdom.server.defaults import ( + LAYOUT_FILE, + DEFAULT_BASE_URL, + DEFAULT_ENV_PATH, + DEFAULT_HOSTNAME, + DEFAULT_PORT, +) +from visdom.utils.shared_utils import ( + warn_once, + get_rand_id, + get_new_window_id, + ensure_dir_exists, +) -import tornado.escape # noqa E402: gotta install ioloop first - -COMPACT_SEPARATORS = (',', ':') MAX_SOCKET_WAIT = 15 # ---- Vaguely server-security related functions ---- # @@ -449,7 +448,7 @@ def recursive_order(node): def stringify(node): - return json.dumps(recursive_order(node), separators=COMPACT_SEPARATORS) + return json.dumps(recursive_order(node), separators=(',', ':')) def hash_md_window(window_json): From f597e9e0700871dfb15a58e9aad700662b06e52a Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 20:09:14 +0200 Subject: [PATCH 10/15] server: move MAX_SOCKET_WAIT to defaults.py --- py/visdom/server/defaults.py | 1 + py/visdom/server/handlers/socket_handlers.py | 3 +-- py/visdom/utils/server_utils.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/py/visdom/server/defaults.py b/py/visdom/server/defaults.py index 05384480..458a230e 100644 --- a/py/visdom/server/defaults.py +++ b/py/visdom/server/defaults.py @@ -13,3 +13,4 @@ DEFAULT_PORT = 8097 DEFAULT_HOSTNAME = "localhost" DEFAULT_BASE_URL = "/" +MAX_SOCKET_WAIT = 15 diff --git a/py/visdom/server/handlers/socket_handlers.py b/py/visdom/server/handlers/socket_handlers.py index cec69dcb..f52702b3 100644 --- a/py/visdom/server/handlers/socket_handlers.py +++ b/py/visdom/server/handlers/socket_handlers.py @@ -35,10 +35,9 @@ from visdom.server.handlers.base_handlers import BaseWebSocketHandler, BaseHandler from visdom.utils.shared_utils import get_rand_id from visdom.utils.server_utils import check_auth, broadcast_envs, serialize_env, send_to_sources, broadcast, escape_eid +from visdom.server.defaults import MAX_SOCKET_WAIT -MAX_SOCKET_WAIT = 15 - # TODO move the logic that actually parses environments and layouts to # new classes in the data_model folder. # TODO move generalized initialization logic from these handlers into the diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index b6fcfc47..57bcf728 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -43,7 +43,6 @@ ensure_dir_exists, ) -MAX_SOCKET_WAIT = 15 # ---- Vaguely server-security related functions ---- # From 2bbe4c72d22b9a9d22fb8154835b73703e3b1723 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 20:41:03 +0200 Subject: [PATCH 11/15] server: added implementation comments about Sockets/Handlers --- py/visdom/server/handlers/socket_handlers.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/py/visdom/server/handlers/socket_handlers.py b/py/visdom/server/handlers/socket_handlers.py index f52702b3..254d97a2 100644 --- a/py/visdom/server/handlers/socket_handlers.py +++ b/py/visdom/server/handlers/socket_handlers.py @@ -48,6 +48,24 @@ # sockets and the poll-based wrappers are using as much shared code as # possible. Try to standardize the code between the client-server and # visdom-server socket edges. + + +# ============== # +# About & Naming # +# ============== # + +# 1. *Handler- & *Wrap-classes are intended to have the same functionality +# - *Handler (e.g. VisSocketHandler) use WebSockets +# - *Wrap (e.g. VisSocketWrap) use polling-based connections instead +# - *Wrapper (e.g. VisSocketWrapper) is just a helper class for the respective *Wrap-class +# to process the current state (instead of the state at the time of polling) +# 2. VisSocket* classes (VisSocketHandler, VisSocketWrap & VisSocketWrapper) +# Their goal is to register clients with write access of actual data. +# 3. Socket* classes (SocketHandler, SocketWrap & SocketWrapper) +# Their goal is to register clients with read access of data. +# Write access is limited to data and view organization (i.e. layout settings, env removal and env saving) + + class VisSocketHandler(BaseWebSocketHandler): def initialize(self, app): self.state = app.state From 77d593107ed587dc4fad5b28318ad426522d8fa1 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 20:41:26 +0200 Subject: [PATCH 12/15] server: renamed ClientSocketWrapper -> SocketWrapper to unify naming scheme --- py/visdom/server/handlers/socket_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/visdom/server/handlers/socket_handlers.py b/py/visdom/server/handlers/socket_handlers.py index 254d97a2..d0bcbc41 100644 --- a/py/visdom/server/handlers/socket_handlers.py +++ b/py/visdom/server/handlers/socket_handlers.py @@ -297,7 +297,7 @@ def on_close(self): # TODO condense some of the functionality between this class and the # original SocketHandler class -class ClientSocketWrapper(): +class SocketWrapper(): """ Wraps all of the socket actions in regular request handling, thus allowing all of the same information to be sent via a polling interface @@ -475,7 +475,7 @@ def post(self): @check_auth def get(self): """Create a new socket wrapper for this requester, return the id""" - new_sub = ClientSocketWrapper(self.app) + new_sub = SocketWrapper(self.app) self.write(json.dumps({'success': True, 'sid': new_sub.sid})) From fd70fcd5b47e911c2c718a564c7a9353d5224aa1 Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 21:00:25 +0200 Subject: [PATCH 13/15] github: adapted workflows to new filestructur in py/visdom/server --- .github/workflows/process-changes.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/process-changes.yml b/.github/workflows/process-changes.yml index 03c42884..36cabb16 100644 --- a/.github/workflows/process-changes.yml +++ b/.github/workflows/process-changes.yml @@ -69,7 +69,7 @@ jobs: uses: cypress-io/github-action@v2 with: install: false - start: python3 py/visdom/server.py -port 8098 -env_path /tmp + start: python3 py/visdom/server -port 8098 -env_path /tmp wait-on: 'http://localhost:8098' spec: cypress/integration/*.init.js @@ -96,7 +96,7 @@ jobs: uses: cypress-io/github-action@v2 with: install: false - start: python3 py/visdom/server.py -port 8098 -env_path /tmp + start: python3 py/visdom/server -port 8098 -env_path /tmp wait-on: 'http://localhost:8098' spec: cypress/integration/screenshots.js @@ -118,7 +118,7 @@ jobs: uses: cypress-io/github-action@v2 with: install: false - start: python3 py/visdom/server.py -port 8098 -env_path /tmp + start: python3 py/visdom/server -port 8098 -env_path /tmp wait-on: 'http://localhost:8098' config: ignoreTestFiles=screenshots.* From c573028dca580d2b4e68583774705f2406e3db2e Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 29 Jul 2022 21:12:14 +0200 Subject: [PATCH 14/15] github: (may be reverted after merge) ensures that cypress tests on master work in PR --- .github/workflows/process-changes.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/process-changes.yml b/.github/workflows/process-changes.yml index 36cabb16..27b2a0d8 100644 --- a/.github/workflows/process-changes.yml +++ b/.github/workflows/process-changes.yml @@ -65,8 +65,26 @@ jobs: run: | git checkout PR-HEAD -- ./cypress + - name: Check file existence + id: check_files + run: | + if test "py/visdom/server.py"; then + echo '::set-output name=file_exists::true' + else + echo '::set-output name=file_exists::false' + fi + + - name: Cypress test:init + if: steps.check_files.outputs.file_exists == 'true' + uses: cypress-io/github-action@v2 + with: + install: false + start: python3 py/visdom/server.py -port 8098 -env_path /tmp + wait-on: 'http://localhost:8098' + spec: cypress/integration/*.init.js - name: Cypress test:init uses: cypress-io/github-action@v2 + if: steps.check_files.outputs.file_exists != 'true' with: install: false start: python3 py/visdom/server -port 8098 -env_path /tmp From f54852fa9b10b4d4daa572f76c36e0d2a021fdfd Mon Sep 17 00:00:00 2001 From: David Hartmann Date: Fri, 7 Oct 2022 00:52:19 +0200 Subject: [PATCH 15/15] release version 0.2.2 --- py/visdom/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/visdom/VERSION b/py/visdom/VERSION index 0c62199f..ee1372d3 100644 --- a/py/visdom/VERSION +++ b/py/visdom/VERSION @@ -1 +1 @@ -0.2.1 +0.2.2