From fc5f3031a9dfb3078231798b2e0d1ba26d3166da Mon Sep 17 00:00:00 2001 From: Geert Jan Bex Date: Thu, 14 May 2020 15:54:09 +0200 Subject: [PATCH 1/6] Add direnv config file to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index da8463d..2504c30 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,6 @@ venv.bak/ *.swp *.bak ~$python_for_data_science.pptx + +# direnv file +.envrc From 390f789b5f05308678331cae416f30e4ea523436 Mon Sep 17 00:00:00 2001 From: Geert Jan Bex Date: Thu, 14 May 2020 22:44:46 +0200 Subject: [PATCH 2/6] Add data generator --- source-code/dash/generate_data.py | 148 ++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100755 source-code/dash/generate_data.py diff --git a/source-code/dash/generate_data.py b/source-code/dash/generate_data.py new file mode 100755 index 0000000..56aa858 --- /dev/null +++ b/source-code/dash/generate_data.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser +import csv +from datetime import datetime, timedelta +import random + +class Job: + + def __init__(self, job_id, nr_nodes, walltime): + self._job_id = job_id + self._nr_nodes = nr_nodes + self._nodes = list() + self._walltime = timedelta(seconds=walltime) + self._remaining = None + + @property + def job_id(self): + return f'{self._job_id:05d}' + + @property + def nr_nodes(self): + return self._nr_nodes + + @property + def nodes(self): + return self._nodes.copy() + + @property + def walltime(self): + return self._walltime + + @property + def remaining(self): + return self._remaining + + def start(self, nodes): + self._nodes = nodes.copy() + self._remaining = self._walltime + + def update(self, delta): + self._remaining -= delta + + @property + def is_done(self): + return self._remaining <= timedelta(seconds=0) + + def __repr__(self): + return f'{self.job_id} {self.nr_nodes} {self.walltime}' + + +class ResoureManager: + + def __init__(self, nodes): + self._free_nodes = list(nodes) + self._busy_nodes = set() + self._queue = list() + self._running = set() + self._completed = set() + self._next_id = 1 + + @property + def nr_nodes(self): + return len(self._free_nodes) + len(self._busy_nodes) + + @property + def nr_free_nodes(self): + return len(self._free_nodes) + + @property + def running_jobs(self): + return self._running + + def qstat(self): + for job in self._queue: + print(job, 'Q') + for job in self._running: + print(job, job.remaining, 'R') + + def submit(self): + self._queue.append(Job(self._next_id, + random.randrange(1, self.nr_nodes), + random.randint(5, 50))) + self._next_id += 1 + + def cycle(self, delta): + # check running jobs to see whether they are done + for job in self._running: + job.update(delta) + if job.is_done: + nodes = job.nodes + for node in nodes: + self._busy_nodes.remove(node) + self._free_nodes.extend(nodes) + self._completed.add(job) + for job in self._completed: + if job in self._running: + self._running.remove(job) + # start all jobs that can be started + for job in self._queue: + if not self._free_nodes: + break + if job.nr_nodes <= self.nr_free_nodes: + nodes = [self._free_nodes.pop(0) for _ in range(job.nr_nodes)] + for node in nodes: + self._busy_nodes.add(node) + job.start(nodes) + self._running.add(job) + for job in self._running: + if job in self._queue: + self._queue.remove(job) + + +if __name__ == '__main__': + arg_parser = ArgumentParser(description='generate data') + arg_parser.add_argument('--nr-nodes', type=int, default=1, + help='number of nodes to generate data for') + arg_parser.add_argument('--nr-timesteps', type=int, default=10, + help='number of timesteps') + arg_parser.add_argument('--nr-jobs', type=int, default=5, + help='number of jobs to submit') + arg_parser.add_argument('file', help='prefix name of the file to write to') + options = arg_parser.parse_args() + time = datetime.now() + delta = timedelta(seconds=5) + nodes = [f'node_{i + 1:03d}' for i in range(options.nr_nodes)] + resource_manager = ResoureManager(nodes) + for _ in range(options.nr_jobs): + resource_manager.submit() + with open(f'{options.file}_load.csv', 'w', newline='') as csv_load_file: + csv_load_writer = csv.writer(csv_load_file) + csv_load_writer.writerow(['time', 'node', 'cpu_load', 'mem_load']) + with open(f'{options.file}_jobs.csv', 'w', newline='') as csv_jobs_file: + csv_jobs_writer = csv.writer(csv_jobs_file) + csv_jobs_writer.writerow(['time', 'job_id', 'node']) + for _append in range(options.nr_timesteps): + print('-'*20) + print(f'---- {time} ----') + resource_manager.qstat() + for node in nodes: + cpu_load = f'{100*random.random():.2f}' + mem_load = f'{100*random.random():.2f}' + csv_load_writer.writerow([node, time, cpu_load, mem_load]) + resource_manager.cycle(delta) + for job in resource_manager.running_jobs: + for node in job.nodes: + csv_jobs_writer.writerow([time, job.job_id, node]) + time += delta From d91b79e64f1a9c9e8680ba6a5a3753df6fe84b8c Mon Sep 17 00:00:00 2001 From: Geert Jan Bex Date: Fri, 15 May 2020 14:26:49 +0200 Subject: [PATCH 3/6] Fix column order for load --- source-code/dash/generate_data.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/source-code/dash/generate_data.py b/source-code/dash/generate_data.py index 56aa858..9976676 100755 --- a/source-code/dash/generate_data.py +++ b/source-code/dash/generate_data.py @@ -71,6 +71,12 @@ def nr_free_nodes(self): def running_jobs(self): return self._running + def is_free(self, node): + return node in self._free_nodes + + def is_busy(self, node): + return node in self._busy_nodes + def qstat(self): for job in self._queue: print(job, 'Q') @@ -133,16 +139,21 @@ def cycle(self, delta): with open(f'{options.file}_jobs.csv', 'w', newline='') as csv_jobs_file: csv_jobs_writer = csv.writer(csv_jobs_file) csv_jobs_writer.writerow(['time', 'job_id', 'node']) - for _append in range(options.nr_timesteps): - print('-'*20) + for _ in range(options.nr_timesteps): + print('-'*36) print(f'---- {time} ----') + time_str = datetime.strftime(time, '%Y-%m-%d %H:%M:%S') resource_manager.qstat() for node in nodes: - cpu_load = f'{100*random.random():.2f}' - mem_load = f'{100*random.random():.2f}' - csv_load_writer.writerow([node, time, cpu_load, mem_load]) + if resource_manager.is_busy(node): + cpu_load = f'{50 + 50*random.random():.2f}' + mem_load = f'{10 + 90*random.random():.2f}' + else: + cpu_load = f'{10*random.random():.2f}' + mem_load = f'{10*random.random():.2f}' + csv_load_writer.writerow([time_str, node, cpu_load, mem_load]) resource_manager.cycle(delta) for job in resource_manager.running_jobs: for node in job.nodes: - csv_jobs_writer.writerow([time, job.job_id, node]) + csv_jobs_writer.writerow([time_str, job.job_id, node]) time += delta From 096ce43e1af93e7aeafb05df48299cd3003d9d5f Mon Sep 17 00:00:00 2001 From: Geert Jan Bex Date: Mon, 18 May 2020 09:07:41 +0200 Subject: [PATCH 4/6] Update environment description for dash --- environment.yml | 189 +++++++++++++++++- ...n_for_data_science_linux64_conda_specs.txt | 127 +++++++----- 2 files changed, 251 insertions(+), 65 deletions(-) diff --git a/environment.yml b/environment.yml index 7340a8c..1ae7696 100644 --- a/environment.yml +++ b/environment.yml @@ -1,23 +1,190 @@ name: python_for_data_science channels: - - conda-forge - defaults dependencies: + - _libgcc_mutex + - blas + - ca-certificates + - intel-openmp + - libgfortran-ng + - libstdcxx-ng + - poppler-data + - libgcc-ng + - mkl + - bzip2 + - expat + - freexl + - geos + - giflib + - gmp + - icu + - jpeg + - json-c + - libffi + - libsodium + - libspatialindex + - libuuid + - libxcb + - lz4-c + - ncurses + - openssl + - pcre + - pixman + - tbb + - xz + - yaml + - zlib + - glib + - hdf4 + - hdf5 + - libboost + - libedit + - libpng + - libssh2 + - libxml2 + - pandoc + - readline + - tk + - xerces-c + - zeromq + - zstd + - dbus + - freetype + - gstreamer + - kealib + - krb5 + - libkml + - libtiff + - sqlite + - fontconfig + - gst-plugins-base + - libcurl + - libpq + - openjpeg + - proj + - python=3.7 + - asn1crypto + - attrs + - backcall + - cairo + - certifi + - cfitsio + - chardet + - click + - curl + - decorator + - defusedxml + - entrypoints + - geotiff + - idna + - ipython_genutils + - json5 + - kiwisolver + - libspatialite + - markupsafe + - mistune + - more-itertools + - olefile + - pandocfilters + - param + - parso + - pickleshare + - postgresql + - prometheus_client + - ptyprocess + - pycparser + - pyparsing + - pyproj + - pysocks + - pytz + - pyyaml + - pyzmq + - qt + - rtree + - send2trash + - sip + - six + - soupsieve + - sqlalchemy + - testpath + - toolz + - tornado + - wcwidth + - webencodings + - xlrd - beautifulsoup4 - - folium - - geopandas - - holoviews + - cffi + - click-plugins + - cligj + - cycler + - jedi + - libdap4 + - libnetcdf + - mkl-service + - packaging + - pexpect + - pillow + - poppler + - pyqt + - pyrsistent + - python-dateutil + - pyviz_comms + - setuptools + - terminado + - tiledb + - traitlets + - zipp + - bleach + - cryptography + - importlib_metadata + - jinja2 + - jupyter_core + - libgdal + - munch + - networkx + - numpy-base + - pygments + - wheel + - branca + - jsonschema + - jupyter_client + - pip + - prompt_toolkit + - pyopenssl + - ipython + - nbformat + - urllib3 + - ipykernel + - nbconvert + - requests + - jupyter_console + - notebook + - qtconsole + - jupyterlab_server + - widgetsnbextension - ipywidgets - - jupyter - jupyterlab + - jupyter + - vincent + - bokeh + - gdal - matplotlib - - networkx - - nodejs + - holoviews + - mkl_fft + - mkl_random + - numpy - pandas - - python=3.7 - scipy - - seaborn - shapely - - sqlalchemy + - altair + - fiona + - patsy + - folium + - geopandas + - statsmodels + - seaborn + - nodejs - xarray - - xlrd + - dash +prefix: /home/gjb/miniconda3/envs/python_for_data_science + diff --git a/python_for_data_science_linux64_conda_specs.txt b/python_for_data_science_linux64_conda_specs.txt index a083671..e06d7b6 100644 --- a/python_for_data_science_linux64_conda_specs.txt +++ b/python_for_data_science_linux64_conda_specs.txt @@ -5,20 +5,20 @@ https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-mkl.conda https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2020.1.1-0.conda -https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2020.0-166.conda +https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2020.1-217.conda https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.33.1-h53a641e_7.conda https://repo.anaconda.com/pkgs/main/linux-64/libgfortran-ng-7.3.0-hdf63c60_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-9.1.0-hdf63c60_0.conda https://repo.anaconda.com/pkgs/main/linux-64/poppler-data-0.4.9-0.conda https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-9.1.0-hdf63c60_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/mkl-2020.0-166.conda +https://repo.anaconda.com/pkgs/main/linux-64/mkl-2020.1-217.conda https://repo.anaconda.com/pkgs/main/linux-64/bzip2-1.0.8-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/expat-2.2.6-he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/freexl-1.0.5-h14c3975_0.conda https://repo.anaconda.com/pkgs/main/linux-64/geos-3.8.0-he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/giflib-5.1.4-h14c3975_1.conda https://repo.anaconda.com/pkgs/main/linux-64/gmp-6.1.2-h6c8ec71_1.conda -https://repo.anaconda.com/pkgs/main/linux-64/icu-58.2-h9c2bf20_1.conda +https://repo.anaconda.com/pkgs/main/linux-64/icu-58.2-he6710b0_3.conda https://repo.anaconda.com/pkgs/main/linux-64/jpeg-9b-h024ee3a_2.conda https://repo.anaconda.com/pkgs/main/linux-64/json-c-0.13.1-h1bed415_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.2.1-hd88cf55_4.conda @@ -27,13 +27,13 @@ https://repo.anaconda.com/pkgs/main/linux-64/libspatialindex-1.9.3-he6710b0_0.ta https://repo.anaconda.com/pkgs/main/linux-64/libuuid-1.0.3-h1bed415_2.conda https://repo.anaconda.com/pkgs/main/linux-64/libxcb-1.13-h1bed415_1.conda https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.8.1.2-h14c3975_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.2-he6710b0_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.2-he6710b0_1.conda https://repo.anaconda.com/pkgs/main/linux-64/nodejs-10.13.0-he6710b0_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1d-h7b6447c_4.conda +https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1g-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pcre-8.43-he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pixman-0.38.0-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/tbb-2018.0.5-h6bb024c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.4-h14c3975_4.conda +https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.5-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/yaml-0.1.7-had09818_2.conda https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.11-h7b6447c_3.conda https://repo.anaconda.com/pkgs/main/linux-64/glib-2.63.1-h5a9c865_0.tar.bz2 @@ -42,7 +42,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/hdf5-1.10.4-hb1b8bf9_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libboost-1.67.0-h46d08c1_4.conda https://repo.anaconda.com/pkgs/main/linux-64/libedit-3.1.20181209-hc058e9b_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libpng-1.6.37-hbc83047_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/libssh2-1.8.2-h1ba5d50_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/libssh2-1.9.0-h1ba5d50_1.conda https://repo.anaconda.com/pkgs/main/linux-64/libxml2-2.9.9-hea5a465_1.conda https://repo.anaconda.com/pkgs/main/linux-64/pandoc-2.2.3.2-0.conda https://repo.anaconda.com/pkgs/main/linux-64/readline-7.0-h7b6447c_5.conda @@ -50,38 +50,41 @@ https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.8-hbc83047_0.conda https://repo.anaconda.com/pkgs/main/linux-64/xerces-c-3.2.2-h780794e_0.conda https://repo.anaconda.com/pkgs/main/linux-64/zeromq-4.3.1-he6710b0_3.conda https://repo.anaconda.com/pkgs/main/linux-64/zstd-1.3.7-h0b5b093_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/dbus-1.13.12-h746ee38_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/dbus-1.13.14-hb2f20db_0.conda https://repo.anaconda.com/pkgs/main/linux-64/freetype-2.9.1-h8a8886c_1.conda https://repo.anaconda.com/pkgs/main/linux-64/gstreamer-1.14.0-hb453b48_1.conda https://repo.anaconda.com/pkgs/main/linux-64/kealib-1.4.7-hd0c454d_6.conda https://repo.anaconda.com/pkgs/main/linux-64/krb5-1.16.4-h173b8e3_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libkml-1.3.0-h590aaf7_4.conda https://repo.anaconda.com/pkgs/main/linux-64/libtiff-4.1.0-h2733197_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.31.1-h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.31.1-h62c20be_1.conda https://repo.anaconda.com/pkgs/main/linux-64/fontconfig-2.13.0-h9420a91_0.conda https://repo.anaconda.com/pkgs/main/linux-64/gst-plugins-base-1.14.0-hbbd80ab_1.conda https://repo.anaconda.com/pkgs/main/linux-64/libcurl-7.67.0-h20c2e04_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/libpq-11.2-h20c2e04_0.conda https://repo.anaconda.com/pkgs/main/linux-64/openjpeg-2.3.0-h05c96fa_1.conda https://repo.anaconda.com/pkgs/main/linux-64/proj-6.2.1-haa6030c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/python-3.7.6-h0371630_2.conda +https://repo.anaconda.com/pkgs/main/linux-64/python-3.7.7-h191fe78_0_cpython.conda https://repo.anaconda.com/pkgs/main/linux-64/asn1crypto-1.3.0-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/attrs-19.3.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/backcall-0.1.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cairo-1.14.12-h8948797_3.conda -https://repo.anaconda.com/pkgs/main/linux-64/certifi-2019.11.28-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/certifi-2020.4.5.1-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cfitsio-3.470-hb7c8383_2.conda https://repo.anaconda.com/pkgs/main/linux-64/chardet-3.0.4-py37_1003.conda -https://repo.anaconda.com/pkgs/main/linux-64/click-7.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/click-7.1.2-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/curl-7.67.0-hbc83047_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/decorator-4.4.1-py_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/dash-renderer-1.1.2-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/decorator-4.4.2-py_0.conda https://repo.anaconda.com/pkgs/main/noarch/defusedxml-0.6.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/entrypoints-0.3-py37_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/future-0.18.2-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/geotiff-1.5.1-h21e8280_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/idna-2.8-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/idna-2.9-py_1.conda https://repo.anaconda.com/pkgs/main/linux-64/ipython_genutils-0.2.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/json5-0.9.1-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/kiwisolver-1.1.0-py37he6710b0_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/itsdangerous-1.1.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/json5-0.9.4-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/kiwisolver-1.2.0-py37hfd86e86_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libspatialite-4.3.0a-h793db0d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/markupsafe-1.1.1-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/mistune-0.8.4-py37h7b6447c_0.conda @@ -89,101 +92,117 @@ https://repo.anaconda.com/pkgs/main/noarch/more-itertools-8.2.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/olefile-0.46-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pandocfilters-1.4.2-py37_1.conda https://repo.anaconda.com/pkgs/main/noarch/param-1.9.3-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/parso-0.6.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/parso-0.7.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pickleshare-0.7.5-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/postgresql-11.2-h20c2e04_0.conda https://repo.anaconda.com/pkgs/main/noarch/prometheus_client-0.7.1-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/ptyprocess-0.6.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pycparser-2.19-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.6-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pyproj-2.4.1-py37h2ea87da_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pycparser-2.20-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.7-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pyproj-2.6.0-py37hd003209_1.conda https://repo.anaconda.com/pkgs/main/linux-64/pysocks-1.7.1-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pytz-2019.3-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pyyaml-5.3-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pytz-2020.1-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pyyaml-5.3.1-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pyzmq-18.1.1-py37he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/qt-5.9.7-h5867ecd_1.conda -https://repo.anaconda.com/pkgs/main/linux-64/rtree-0.9.3-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/qtpy-1.9.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/rtree-0.9.4-py37_1.conda https://repo.anaconda.com/pkgs/main/linux-64/send2trash-1.5.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/sip-4.19.8-py37hf484d3e_0.conda https://repo.anaconda.com/pkgs/main/linux-64/six-1.14.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/soupsieve-1.9.5-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/sqlalchemy-1.3.13-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/soupsieve-2.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/sqlalchemy-1.3.16-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/noarch/testpath-0.4.4-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/toolz-0.10.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/tornado-6.0.3-py37h7b6447c_3.conda -https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.1.8-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/tornado-6.0.4-py37h7b6447c_1.conda +https://repo.anaconda.com/pkgs/main/noarch/tqdm-4.46.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/typing_extensions-3.7.4.1-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.1.9-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/webencodings-0.5.1-py37_1.conda +https://repo.anaconda.com/pkgs/main/noarch/werkzeug-1.0.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/xlrd-1.2.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/zipp-2.2.0-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/beautifulsoup4-4.8.2-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/zipp-3.1.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/beautifulsoup4-4.9.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cffi-1.14.0-py37h2e261b9_0.conda https://repo.anaconda.com/pkgs/main/noarch/click-plugins-1.1.1-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/cligj-0.5.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cycler-0.10.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/importlib_metadata-1.5.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/jedi-0.16.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/jedi-0.17.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libdap4-3.19.1-h6ec2957_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libnetcdf-4.6.1-h11d0813_2.conda https://repo.anaconda.com/pkgs/main/linux-64/mkl-service-2.3.0-py37he904b0f_0.conda -https://repo.anaconda.com/pkgs/main/noarch/packaging-20.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/packaging-20.3-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pexpect-4.8.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pillow-7.0.0-py37hb39fc2d_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pillow-7.1.2-py37hb39fc2d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/poppler-0.65.0-h581218d_1.conda https://repo.anaconda.com/pkgs/main/linux-64/pyqt-5.9.2-py37h05f1152_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/pyrsistent-0.15.7-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pyrsistent-0.16.0-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pyviz_comms-0.7.3-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/setuptools-45.2.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pyviz_comms-0.7.4-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/retrying-1.3.3-py37_2.conda +https://repo.anaconda.com/pkgs/main/linux-64/setuptools-46.2.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/terminado-0.8.3-py37_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/tiledb-1.6.3-h1fb8f14_0.conda https://repo.anaconda.com/pkgs/main/linux-64/traitlets-4.3.3-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/bleach-3.1.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/cryptography-2.8-py37h1ba5d50_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jinja2-2.11.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/bleach-3.1.4-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/cryptography-2.9.2-py37h1ba5d50_0.conda +https://repo.anaconda.com/pkgs/main/noarch/jinja2-2.11.2-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/jsonschema-3.2.0-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/jupyter_core-4.6.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/jupyter_core-4.6.3-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libgdal-3.0.2-h27ab9cc_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/markdown-3.1.1-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/munch-2.5.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/networkx-2.4-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.18.1-py37hde5b4d6_1.conda -https://repo.anaconda.com/pkgs/main/noarch/pygments-2.5.2-py_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/plotly-4.6.0-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pygments-2.6.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/wheel-0.34.2-py37_0.conda https://conda.anaconda.org/conda-forge/noarch/branca-0.3.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/jupyter_client-5.3.4-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/nbformat-5.0.4-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pip-20.0.2-py37_1.conda -https://repo.anaconda.com/pkgs/main/noarch/prompt_toolkit-3.0.3-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/flask-1.1.2-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/jupyter_client-6.1.3-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/nbformat-5.0.6-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pip-20.0.2-py37_3.conda +https://repo.anaconda.com/pkgs/main/noarch/prompt-toolkit-3.0.4-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pyopenssl-19.1.0-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/ipython-7.12.0-py37h5ca1d4c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/flask-compress-1.4.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/nbconvert-5.6.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/prompt_toolkit-3.0.4-0.conda https://repo.anaconda.com/pkgs/main/linux-64/urllib3-1.25.8-py37_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/ipython-7.13.0-py37h5ca1d4c_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/requests-2.23.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/ipykernel-5.1.4-py37h39e3cac_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/requests-2.22.0-py37_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/pyct-0.4.6-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/jupyter_console-6.1.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/notebook-6.0.3-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/qtconsole-4.6.0-py_1.conda -https://repo.anaconda.com/pkgs/main/noarch/jupyterlab_server-1.0.6-py_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/qtconsole-4.7.4-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/jupyterlab_server-1.1.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/widgetsnbextension-3.5.1-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/ipywidgets-7.5.1-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/jupyterlab-1.2.6-pyhf63ae98_0.conda https://repo.anaconda.com/pkgs/main/linux-64/jupyter-1.0.0-py37_7.conda https://conda.anaconda.org/conda-forge/noarch/vincent-0.4.4-py_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/bokeh-1.4.0-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/bokeh-2.0.2-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/panel-0.9.5-py_1.conda https://repo.anaconda.com/pkgs/main/linux-64/gdal-3.0.2-py37hbb2a789_0.conda https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-3.1.3-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-base-3.1.3-py37hef1b27d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/mkl_fft-1.0.15-py37ha843d7b_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/mkl_random-1.1.0-py37hd6b4f25_0.conda https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.18.1-py37h4f9e942_0.conda -https://repo.anaconda.com/pkgs/main/noarch/holoviews-1.12.7-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.0.1-py37h0573a6f_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.0.3-py37h0573a6f_0.conda https://repo.anaconda.com/pkgs/main/linux-64/scipy-1.4.1-py37h0b6359f_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/shapely-1.6.4-py37hc5e8c75_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/shapely-1.7.0-py37h98ec03d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/altair-3.2.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/fiona-1.8.11-py37h41e4f33_0.conda +https://repo.anaconda.com/pkgs/main/noarch/holoviews-1.13.2-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/patsy-0.5.1-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/seaborn-0.10.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/xarray-0.15.0-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/seaborn-0.10.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/xarray-0.15.1-py_0.conda https://conda.anaconda.org/conda-forge/noarch/folium-0.10.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/geopandas-0.6.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/statsmodels-0.11.0-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-core-components-1.3.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-html-components-1.0.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-1.4.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-table-4.4.1-py_0.conda From 3bbe83029ef596ab9c49526c7421771d282c3d4d Mon Sep 17 00:00:00 2001 From: Geert Jan Bex Date: Mon, 18 May 2020 09:36:56 +0200 Subject: [PATCH 5/6] Add dash example --- source-code/README.md | 2 + source-code/dash/.gitignore | 2 + source-code/dash/README.md | 10 +++ source-code/dash/jobs_overview.py | 107 ++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 source-code/dash/.gitignore create mode 100644 source-code/dash/README.md create mode 100755 source-code/dash/jobs_overview.py diff --git a/source-code/README.md b/source-code/README.md index a1768bf..e5c2b9f 100644 --- a/source-code/README.md +++ b/source-code/README.md @@ -21,6 +21,7 @@ to create it. There is some material not covered in the presentation as well. * jupyter * folium (with conda, use `-c conda-forge`) * xarray + * dash ## What is it? * [`db-access`](db-access): illustration of accessing SQLite databases and using @@ -38,3 +39,4 @@ to create it. There is some material not covered in the presentation as well. and graph representation using networkx. * [`xarray`](xarray): illustrates the xarray library for pandas-like operations on multi-dimensional arrays. +* [`dash`](dash): illustration of creating a simple dashboard using dash. diff --git a/source-code/dash/.gitignore b/source-code/dash/.gitignore new file mode 100644 index 0000000..45f6095 --- /dev/null +++ b/source-code/dash/.gitignore @@ -0,0 +1,2 @@ +# data files are generated +*.csv diff --git a/source-code/dash/README.md b/source-code/dash/README.md new file mode 100644 index 0000000..54993d6 --- /dev/null +++ b/source-code/dash/README.md @@ -0,0 +1,10 @@ +# dash + +The dash framework lets you make dashboards in a convenient way. The HTML, CSS and +Javascript components are generated automatically. + +## What is it? + +1. `generate_data.py`: script to generate data. It simulates a scheduler. +1. `jobs_overview.py`: script that generatees a dashboard that provides a dynamic + view on the jobs' performance. diff --git a/source-code/dash/jobs_overview.py b/source-code/dash/jobs_overview.py new file mode 100755 index 0000000..be0a0a8 --- /dev/null +++ b/source-code/dash/jobs_overview.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser +import dash +import dash_core_components as dcc +import dash_html_components as html +from dash.dependencies import Input, Output +from datetime import datetime +import pandas as pd + + +def generate_overview_table(jobs): + jobs_summary = jobs[['job_id', 'node']]\ + .drop_duplicates()\ + .groupby('job_id')\ + .count() + jobs_summary = jobs_summary.merge(jobs.groupby('job_id')[['time']].min(), + on='job_id') + jobs_summary = jobs_summary.merge(jobs.groupby('job_id')[['time']].max(), + on='job_id') + jobs_summary.columns = ['nodes', 'start time', 'end time'] + jobs_summary.insert(0, 'job_id', jobs_summary.index) + return html.Table([ + html.Thead( + html.Tr([html.Th(column) for column in jobs_summary.columns]) + ), + html.Tbody([ + html.Tr([ + html.Td(jobs_summary.loc[job_id][column]) + for column in jobs_summary.columns]) + for job_id in jobs_summary.index] + )] + ) + +def generate_job_menu(jobs): + job_list = [{'label': str(job_id), 'value': str(job_id)} + for job_id in jobs.job_id.unique()] + if len(job_list): + return dcc.Dropdown( + id='job_menu', + options=job_list, + value=job_list[0]['value'], + placeholder='Select a job...', + ) + + + +if __name__ == '__main__': + arg_parsr = ArgumentParser(description='visualize job performance') + arg_parsr.add_argument('--job-file', required=True, + help='file with job/node information') + arg_parsr.add_argument('--load-file', required=True, + help='file with node/load information') + options = arg_parsr.parse_args() + jobs = pd.read_csv(options.job_file, parse_dates=True, + date_parser=pd.to_datetime) + loads = pd.read_csv(options.load_file, parse_dates=True, + date_parser=pd.to_datetime) + external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] + app = dash.Dash(__name__, external_stylesheets=external_stylesheets) + app.layout = html.Div(children=[ + html.H1(children='Job overview'), + html.Div(children=[ + html.Div(children=[generate_overview_table(jobs)], + style={'width': '70%', 'display': 'inline-block', + 'margin': '20pt'}), + html.Div(children=[ + html.Label('Jobs'), + generate_job_menu(jobs), + ], style={'width': '10%', 'float': 'left', 'display': 'inline-block', + 'margin': '20pt'}) + ]), + html.Div(children=[ + html.Div(children=[ dcc.Graph(id='cpu_load_graph')], + style={'width': '45%', 'display': 'inline-block'}), + html.Div(children=[dcc.Graph(id='mem_load_graph')], + style={'widht': '45%', 'float': 'right', 'display': 'inline-block'}) + ]) + ]) + + @app.callback( + [Output('cpu_load_graph', 'figure'), Output('mem_load_graph', 'figure')], + [Input('job_menu', 'value')]) + def update(job_id): + data = loads.merge(jobs.query(f'job_id == {job_id}'), on=['time', 'node'], + how='right') + figures = list() + for quantity in ['cpu', 'mem']: + nodes_data = list() + for node_df in data.groupby('node'): + node_data = dict() + node_data['name'] = node_df[0] + node_data['x'] = node_df[1].time + node_data['y'] = node_df[1][f'{quantity}_load'] + node_data['mode'] = ['markers', 'line'], + nodes_data.append(node_data) + figures.append({ + 'data': nodes_data, + 'layout': dict( + xaxis={'title': 'time'}, + yaxis={'title': f'{quantity} load'}, + hovermode='closest'), + }) + return figures + + + app.run_server(debug=True) From 63d9b5c587e13df632e525fa55c3ca1dfe833210 Mon Sep 17 00:00:00 2001 From: Geert Jan Bex Date: Thu, 14 May 2020 22:44:46 +0200 Subject: [PATCH 6/6] Add dash example * add data generator * update conda environment * add dashboard implemented in dash --- environment.yml | 189 +++++++++++++++++- ...n_for_data_science_linux64_conda_specs.txt | 127 +++++++----- source-code/README.md | 2 + source-code/dash/.gitignore | 2 + source-code/dash/README.md | 10 + source-code/dash/generate_data.py | 159 +++++++++++++++ source-code/dash/jobs_overview.py | 107 ++++++++++ 7 files changed, 531 insertions(+), 65 deletions(-) create mode 100644 source-code/dash/.gitignore create mode 100644 source-code/dash/README.md create mode 100755 source-code/dash/generate_data.py create mode 100755 source-code/dash/jobs_overview.py diff --git a/environment.yml b/environment.yml index 7340a8c..1ae7696 100644 --- a/environment.yml +++ b/environment.yml @@ -1,23 +1,190 @@ name: python_for_data_science channels: - - conda-forge - defaults dependencies: + - _libgcc_mutex + - blas + - ca-certificates + - intel-openmp + - libgfortran-ng + - libstdcxx-ng + - poppler-data + - libgcc-ng + - mkl + - bzip2 + - expat + - freexl + - geos + - giflib + - gmp + - icu + - jpeg + - json-c + - libffi + - libsodium + - libspatialindex + - libuuid + - libxcb + - lz4-c + - ncurses + - openssl + - pcre + - pixman + - tbb + - xz + - yaml + - zlib + - glib + - hdf4 + - hdf5 + - libboost + - libedit + - libpng + - libssh2 + - libxml2 + - pandoc + - readline + - tk + - xerces-c + - zeromq + - zstd + - dbus + - freetype + - gstreamer + - kealib + - krb5 + - libkml + - libtiff + - sqlite + - fontconfig + - gst-plugins-base + - libcurl + - libpq + - openjpeg + - proj + - python=3.7 + - asn1crypto + - attrs + - backcall + - cairo + - certifi + - cfitsio + - chardet + - click + - curl + - decorator + - defusedxml + - entrypoints + - geotiff + - idna + - ipython_genutils + - json5 + - kiwisolver + - libspatialite + - markupsafe + - mistune + - more-itertools + - olefile + - pandocfilters + - param + - parso + - pickleshare + - postgresql + - prometheus_client + - ptyprocess + - pycparser + - pyparsing + - pyproj + - pysocks + - pytz + - pyyaml + - pyzmq + - qt + - rtree + - send2trash + - sip + - six + - soupsieve + - sqlalchemy + - testpath + - toolz + - tornado + - wcwidth + - webencodings + - xlrd - beautifulsoup4 - - folium - - geopandas - - holoviews + - cffi + - click-plugins + - cligj + - cycler + - jedi + - libdap4 + - libnetcdf + - mkl-service + - packaging + - pexpect + - pillow + - poppler + - pyqt + - pyrsistent + - python-dateutil + - pyviz_comms + - setuptools + - terminado + - tiledb + - traitlets + - zipp + - bleach + - cryptography + - importlib_metadata + - jinja2 + - jupyter_core + - libgdal + - munch + - networkx + - numpy-base + - pygments + - wheel + - branca + - jsonschema + - jupyter_client + - pip + - prompt_toolkit + - pyopenssl + - ipython + - nbformat + - urllib3 + - ipykernel + - nbconvert + - requests + - jupyter_console + - notebook + - qtconsole + - jupyterlab_server + - widgetsnbextension - ipywidgets - - jupyter - jupyterlab + - jupyter + - vincent + - bokeh + - gdal - matplotlib - - networkx - - nodejs + - holoviews + - mkl_fft + - mkl_random + - numpy - pandas - - python=3.7 - scipy - - seaborn - shapely - - sqlalchemy + - altair + - fiona + - patsy + - folium + - geopandas + - statsmodels + - seaborn + - nodejs - xarray - - xlrd + - dash +prefix: /home/gjb/miniconda3/envs/python_for_data_science + diff --git a/python_for_data_science_linux64_conda_specs.txt b/python_for_data_science_linux64_conda_specs.txt index a083671..e06d7b6 100644 --- a/python_for_data_science_linux64_conda_specs.txt +++ b/python_for_data_science_linux64_conda_specs.txt @@ -5,20 +5,20 @@ https://repo.anaconda.com/pkgs/main/linux-64/_libgcc_mutex-0.1-main.conda https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-mkl.conda https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2020.1.1-0.conda -https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2020.0-166.conda +https://repo.anaconda.com/pkgs/main/linux-64/intel-openmp-2020.1-217.conda https://repo.anaconda.com/pkgs/main/linux-64/ld_impl_linux-64-2.33.1-h53a641e_7.conda https://repo.anaconda.com/pkgs/main/linux-64/libgfortran-ng-7.3.0-hdf63c60_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-9.1.0-hdf63c60_0.conda https://repo.anaconda.com/pkgs/main/linux-64/poppler-data-0.4.9-0.conda https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-9.1.0-hdf63c60_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/mkl-2020.0-166.conda +https://repo.anaconda.com/pkgs/main/linux-64/mkl-2020.1-217.conda https://repo.anaconda.com/pkgs/main/linux-64/bzip2-1.0.8-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/expat-2.2.6-he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/freexl-1.0.5-h14c3975_0.conda https://repo.anaconda.com/pkgs/main/linux-64/geos-3.8.0-he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/giflib-5.1.4-h14c3975_1.conda https://repo.anaconda.com/pkgs/main/linux-64/gmp-6.1.2-h6c8ec71_1.conda -https://repo.anaconda.com/pkgs/main/linux-64/icu-58.2-h9c2bf20_1.conda +https://repo.anaconda.com/pkgs/main/linux-64/icu-58.2-he6710b0_3.conda https://repo.anaconda.com/pkgs/main/linux-64/jpeg-9b-h024ee3a_2.conda https://repo.anaconda.com/pkgs/main/linux-64/json-c-0.13.1-h1bed415_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.2.1-hd88cf55_4.conda @@ -27,13 +27,13 @@ https://repo.anaconda.com/pkgs/main/linux-64/libspatialindex-1.9.3-he6710b0_0.ta https://repo.anaconda.com/pkgs/main/linux-64/libuuid-1.0.3-h1bed415_2.conda https://repo.anaconda.com/pkgs/main/linux-64/libxcb-1.13-h1bed415_1.conda https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.8.1.2-h14c3975_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.2-he6710b0_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.2-he6710b0_1.conda https://repo.anaconda.com/pkgs/main/linux-64/nodejs-10.13.0-he6710b0_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1d-h7b6447c_4.conda +https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1g-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pcre-8.43-he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pixman-0.38.0-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/tbb-2018.0.5-h6bb024c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.4-h14c3975_4.conda +https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.5-h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/yaml-0.1.7-had09818_2.conda https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.11-h7b6447c_3.conda https://repo.anaconda.com/pkgs/main/linux-64/glib-2.63.1-h5a9c865_0.tar.bz2 @@ -42,7 +42,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/hdf5-1.10.4-hb1b8bf9_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libboost-1.67.0-h46d08c1_4.conda https://repo.anaconda.com/pkgs/main/linux-64/libedit-3.1.20181209-hc058e9b_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libpng-1.6.37-hbc83047_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/libssh2-1.8.2-h1ba5d50_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/libssh2-1.9.0-h1ba5d50_1.conda https://repo.anaconda.com/pkgs/main/linux-64/libxml2-2.9.9-hea5a465_1.conda https://repo.anaconda.com/pkgs/main/linux-64/pandoc-2.2.3.2-0.conda https://repo.anaconda.com/pkgs/main/linux-64/readline-7.0-h7b6447c_5.conda @@ -50,38 +50,41 @@ https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.8-hbc83047_0.conda https://repo.anaconda.com/pkgs/main/linux-64/xerces-c-3.2.2-h780794e_0.conda https://repo.anaconda.com/pkgs/main/linux-64/zeromq-4.3.1-he6710b0_3.conda https://repo.anaconda.com/pkgs/main/linux-64/zstd-1.3.7-h0b5b093_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/dbus-1.13.12-h746ee38_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/dbus-1.13.14-hb2f20db_0.conda https://repo.anaconda.com/pkgs/main/linux-64/freetype-2.9.1-h8a8886c_1.conda https://repo.anaconda.com/pkgs/main/linux-64/gstreamer-1.14.0-hb453b48_1.conda https://repo.anaconda.com/pkgs/main/linux-64/kealib-1.4.7-hd0c454d_6.conda https://repo.anaconda.com/pkgs/main/linux-64/krb5-1.16.4-h173b8e3_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libkml-1.3.0-h590aaf7_4.conda https://repo.anaconda.com/pkgs/main/linux-64/libtiff-4.1.0-h2733197_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.31.1-h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.31.1-h62c20be_1.conda https://repo.anaconda.com/pkgs/main/linux-64/fontconfig-2.13.0-h9420a91_0.conda https://repo.anaconda.com/pkgs/main/linux-64/gst-plugins-base-1.14.0-hbbd80ab_1.conda https://repo.anaconda.com/pkgs/main/linux-64/libcurl-7.67.0-h20c2e04_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/libpq-11.2-h20c2e04_0.conda https://repo.anaconda.com/pkgs/main/linux-64/openjpeg-2.3.0-h05c96fa_1.conda https://repo.anaconda.com/pkgs/main/linux-64/proj-6.2.1-haa6030c_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/python-3.7.6-h0371630_2.conda +https://repo.anaconda.com/pkgs/main/linux-64/python-3.7.7-h191fe78_0_cpython.conda https://repo.anaconda.com/pkgs/main/linux-64/asn1crypto-1.3.0-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/attrs-19.3.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/backcall-0.1.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cairo-1.14.12-h8948797_3.conda -https://repo.anaconda.com/pkgs/main/linux-64/certifi-2019.11.28-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/certifi-2020.4.5.1-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cfitsio-3.470-hb7c8383_2.conda https://repo.anaconda.com/pkgs/main/linux-64/chardet-3.0.4-py37_1003.conda -https://repo.anaconda.com/pkgs/main/linux-64/click-7.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/click-7.1.2-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/curl-7.67.0-hbc83047_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/decorator-4.4.1-py_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/dash-renderer-1.1.2-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/decorator-4.4.2-py_0.conda https://repo.anaconda.com/pkgs/main/noarch/defusedxml-0.6.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/entrypoints-0.3-py37_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/future-0.18.2-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/geotiff-1.5.1-h21e8280_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/idna-2.8-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/idna-2.9-py_1.conda https://repo.anaconda.com/pkgs/main/linux-64/ipython_genutils-0.2.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/json5-0.9.1-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/kiwisolver-1.1.0-py37he6710b0_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/itsdangerous-1.1.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/json5-0.9.4-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/kiwisolver-1.2.0-py37hfd86e86_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libspatialite-4.3.0a-h793db0d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/markupsafe-1.1.1-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/mistune-0.8.4-py37h7b6447c_0.conda @@ -89,101 +92,117 @@ https://repo.anaconda.com/pkgs/main/noarch/more-itertools-8.2.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/olefile-0.46-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pandocfilters-1.4.2-py37_1.conda https://repo.anaconda.com/pkgs/main/noarch/param-1.9.3-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/parso-0.6.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/parso-0.7.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pickleshare-0.7.5-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/postgresql-11.2-h20c2e04_0.conda https://repo.anaconda.com/pkgs/main/noarch/prometheus_client-0.7.1-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/ptyprocess-0.6.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pycparser-2.19-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.6-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pyproj-2.4.1-py37h2ea87da_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pycparser-2.20-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.7-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pyproj-2.6.0-py37hd003209_1.conda https://repo.anaconda.com/pkgs/main/linux-64/pysocks-1.7.1-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pytz-2019.3-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pyyaml-5.3-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pytz-2020.1-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pyyaml-5.3.1-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pyzmq-18.1.1-py37he6710b0_0.conda https://repo.anaconda.com/pkgs/main/linux-64/qt-5.9.7-h5867ecd_1.conda -https://repo.anaconda.com/pkgs/main/linux-64/rtree-0.9.3-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/qtpy-1.9.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/rtree-0.9.4-py37_1.conda https://repo.anaconda.com/pkgs/main/linux-64/send2trash-1.5.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/sip-4.19.8-py37hf484d3e_0.conda https://repo.anaconda.com/pkgs/main/linux-64/six-1.14.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/soupsieve-1.9.5-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/sqlalchemy-1.3.13-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/soupsieve-2.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/sqlalchemy-1.3.16-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/noarch/testpath-0.4.4-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/toolz-0.10.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/tornado-6.0.3-py37h7b6447c_3.conda -https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.1.8-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/tornado-6.0.4-py37h7b6447c_1.conda +https://repo.anaconda.com/pkgs/main/noarch/tqdm-4.46.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/typing_extensions-3.7.4.1-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.1.9-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/webencodings-0.5.1-py37_1.conda +https://repo.anaconda.com/pkgs/main/noarch/werkzeug-1.0.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/xlrd-1.2.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/zipp-2.2.0-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/beautifulsoup4-4.8.2-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/zipp-3.1.0-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/beautifulsoup4-4.9.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cffi-1.14.0-py37h2e261b9_0.conda https://repo.anaconda.com/pkgs/main/noarch/click-plugins-1.1.1-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/cligj-0.5.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/cycler-0.10.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/importlib_metadata-1.5.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/jedi-0.16.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/jedi-0.17.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libdap4-3.19.1-h6ec2957_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libnetcdf-4.6.1-h11d0813_2.conda https://repo.anaconda.com/pkgs/main/linux-64/mkl-service-2.3.0-py37he904b0f_0.conda -https://repo.anaconda.com/pkgs/main/noarch/packaging-20.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/packaging-20.3-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pexpect-4.8.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pillow-7.0.0-py37hb39fc2d_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pillow-7.1.2-py37hb39fc2d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/poppler-0.65.0-h581218d_1.conda https://repo.anaconda.com/pkgs/main/linux-64/pyqt-5.9.2-py37h05f1152_2.conda -https://repo.anaconda.com/pkgs/main/linux-64/pyrsistent-0.15.7-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pyrsistent-0.16.0-py37h7b6447c_0.conda https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pyviz_comms-0.7.3-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/setuptools-45.2.0-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pyviz_comms-0.7.4-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/retrying-1.3.3-py37_2.conda +https://repo.anaconda.com/pkgs/main/linux-64/setuptools-46.2.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/terminado-0.8.3-py37_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/tiledb-1.6.3-h1fb8f14_0.conda https://repo.anaconda.com/pkgs/main/linux-64/traitlets-4.3.3-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/bleach-3.1.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/cryptography-2.8-py37h1ba5d50_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jinja2-2.11.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/bleach-3.1.4-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/cryptography-2.9.2-py37h1ba5d50_0.conda +https://repo.anaconda.com/pkgs/main/noarch/jinja2-2.11.2-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/jsonschema-3.2.0-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/jupyter_core-4.6.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/jupyter_core-4.6.3-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/libgdal-3.0.2-h27ab9cc_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/markdown-3.1.1-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/munch-2.5.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/networkx-2.4-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/numpy-base-1.18.1-py37hde5b4d6_1.conda -https://repo.anaconda.com/pkgs/main/noarch/pygments-2.5.2-py_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/plotly-4.6.0-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pygments-2.6.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/wheel-0.34.2-py37_0.conda https://conda.anaconda.org/conda-forge/noarch/branca-0.3.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/jupyter_client-5.3.4-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/nbformat-5.0.4-py_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/pip-20.0.2-py37_1.conda -https://repo.anaconda.com/pkgs/main/noarch/prompt_toolkit-3.0.3-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/flask-1.1.2-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/jupyter_client-6.1.3-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/nbformat-5.0.6-py_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pip-20.0.2-py37_3.conda +https://repo.anaconda.com/pkgs/main/noarch/prompt-toolkit-3.0.4-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/pyopenssl-19.1.0-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/ipython-7.12.0-py37h5ca1d4c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/flask-compress-1.4.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/nbconvert-5.6.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/prompt_toolkit-3.0.4-0.conda https://repo.anaconda.com/pkgs/main/linux-64/urllib3-1.25.8-py37_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/ipython-7.13.0-py37h5ca1d4c_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/requests-2.23.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/ipykernel-5.1.4-py37h39e3cac_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/requests-2.22.0-py37_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/pyct-0.4.6-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/jupyter_console-6.1.0-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/notebook-6.0.3-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/qtconsole-4.6.0-py_1.conda -https://repo.anaconda.com/pkgs/main/noarch/jupyterlab_server-1.0.6-py_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/qtconsole-4.7.4-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/jupyterlab_server-1.1.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/widgetsnbextension-3.5.1-py37_0.conda https://repo.anaconda.com/pkgs/main/noarch/ipywidgets-7.5.1-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/jupyterlab-1.2.6-pyhf63ae98_0.conda https://repo.anaconda.com/pkgs/main/linux-64/jupyter-1.0.0-py37_7.conda https://conda.anaconda.org/conda-forge/noarch/vincent-0.4.4-py_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/bokeh-1.4.0-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/bokeh-2.0.2-py37_0.conda +https://repo.anaconda.com/pkgs/main/noarch/panel-0.9.5-py_1.conda https://repo.anaconda.com/pkgs/main/linux-64/gdal-3.0.2-py37hbb2a789_0.conda https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-3.1.3-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/matplotlib-base-3.1.3-py37hef1b27d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/mkl_fft-1.0.15-py37ha843d7b_0.tar.bz2 https://repo.anaconda.com/pkgs/main/linux-64/mkl_random-1.1.0-py37hd6b4f25_0.conda https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.18.1-py37h4f9e942_0.conda -https://repo.anaconda.com/pkgs/main/noarch/holoviews-1.12.7-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.0.1-py37h0573a6f_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/pandas-1.0.3-py37h0573a6f_0.conda https://repo.anaconda.com/pkgs/main/linux-64/scipy-1.4.1-py37h0b6359f_0.conda -https://repo.anaconda.com/pkgs/main/linux-64/shapely-1.6.4-py37hc5e8c75_0.conda +https://repo.anaconda.com/pkgs/main/linux-64/shapely-1.7.0-py37h98ec03d_0.conda https://repo.anaconda.com/pkgs/main/linux-64/altair-3.2.0-py37_0.conda https://repo.anaconda.com/pkgs/main/linux-64/fiona-1.8.11-py37h41e4f33_0.conda +https://repo.anaconda.com/pkgs/main/noarch/holoviews-1.13.2-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/patsy-0.5.1-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/seaborn-0.10.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/xarray-0.15.0-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/seaborn-0.10.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/xarray-0.15.1-py_0.conda https://conda.anaconda.org/conda-forge/noarch/folium-0.10.0-py_0.tar.bz2 https://repo.anaconda.com/pkgs/main/noarch/geopandas-0.6.1-py_0.conda https://repo.anaconda.com/pkgs/main/linux-64/statsmodels-0.11.0-py37h7b6447c_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-core-components-1.3.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-html-components-1.0.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-1.4.1-py_0.conda +https://repo.anaconda.com/pkgs/main/noarch/dash-table-4.4.1-py_0.conda diff --git a/source-code/README.md b/source-code/README.md index a1768bf..e5c2b9f 100644 --- a/source-code/README.md +++ b/source-code/README.md @@ -21,6 +21,7 @@ to create it. There is some material not covered in the presentation as well. * jupyter * folium (with conda, use `-c conda-forge`) * xarray + * dash ## What is it? * [`db-access`](db-access): illustration of accessing SQLite databases and using @@ -38,3 +39,4 @@ to create it. There is some material not covered in the presentation as well. and graph representation using networkx. * [`xarray`](xarray): illustrates the xarray library for pandas-like operations on multi-dimensional arrays. +* [`dash`](dash): illustration of creating a simple dashboard using dash. diff --git a/source-code/dash/.gitignore b/source-code/dash/.gitignore new file mode 100644 index 0000000..45f6095 --- /dev/null +++ b/source-code/dash/.gitignore @@ -0,0 +1,2 @@ +# data files are generated +*.csv diff --git a/source-code/dash/README.md b/source-code/dash/README.md new file mode 100644 index 0000000..54993d6 --- /dev/null +++ b/source-code/dash/README.md @@ -0,0 +1,10 @@ +# dash + +The dash framework lets you make dashboards in a convenient way. The HTML, CSS and +Javascript components are generated automatically. + +## What is it? + +1. `generate_data.py`: script to generate data. It simulates a scheduler. +1. `jobs_overview.py`: script that generatees a dashboard that provides a dynamic + view on the jobs' performance. diff --git a/source-code/dash/generate_data.py b/source-code/dash/generate_data.py new file mode 100755 index 0000000..9976676 --- /dev/null +++ b/source-code/dash/generate_data.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser +import csv +from datetime import datetime, timedelta +import random + +class Job: + + def __init__(self, job_id, nr_nodes, walltime): + self._job_id = job_id + self._nr_nodes = nr_nodes + self._nodes = list() + self._walltime = timedelta(seconds=walltime) + self._remaining = None + + @property + def job_id(self): + return f'{self._job_id:05d}' + + @property + def nr_nodes(self): + return self._nr_nodes + + @property + def nodes(self): + return self._nodes.copy() + + @property + def walltime(self): + return self._walltime + + @property + def remaining(self): + return self._remaining + + def start(self, nodes): + self._nodes = nodes.copy() + self._remaining = self._walltime + + def update(self, delta): + self._remaining -= delta + + @property + def is_done(self): + return self._remaining <= timedelta(seconds=0) + + def __repr__(self): + return f'{self.job_id} {self.nr_nodes} {self.walltime}' + + +class ResoureManager: + + def __init__(self, nodes): + self._free_nodes = list(nodes) + self._busy_nodes = set() + self._queue = list() + self._running = set() + self._completed = set() + self._next_id = 1 + + @property + def nr_nodes(self): + return len(self._free_nodes) + len(self._busy_nodes) + + @property + def nr_free_nodes(self): + return len(self._free_nodes) + + @property + def running_jobs(self): + return self._running + + def is_free(self, node): + return node in self._free_nodes + + def is_busy(self, node): + return node in self._busy_nodes + + def qstat(self): + for job in self._queue: + print(job, 'Q') + for job in self._running: + print(job, job.remaining, 'R') + + def submit(self): + self._queue.append(Job(self._next_id, + random.randrange(1, self.nr_nodes), + random.randint(5, 50))) + self._next_id += 1 + + def cycle(self, delta): + # check running jobs to see whether they are done + for job in self._running: + job.update(delta) + if job.is_done: + nodes = job.nodes + for node in nodes: + self._busy_nodes.remove(node) + self._free_nodes.extend(nodes) + self._completed.add(job) + for job in self._completed: + if job in self._running: + self._running.remove(job) + # start all jobs that can be started + for job in self._queue: + if not self._free_nodes: + break + if job.nr_nodes <= self.nr_free_nodes: + nodes = [self._free_nodes.pop(0) for _ in range(job.nr_nodes)] + for node in nodes: + self._busy_nodes.add(node) + job.start(nodes) + self._running.add(job) + for job in self._running: + if job in self._queue: + self._queue.remove(job) + + +if __name__ == '__main__': + arg_parser = ArgumentParser(description='generate data') + arg_parser.add_argument('--nr-nodes', type=int, default=1, + help='number of nodes to generate data for') + arg_parser.add_argument('--nr-timesteps', type=int, default=10, + help='number of timesteps') + arg_parser.add_argument('--nr-jobs', type=int, default=5, + help='number of jobs to submit') + arg_parser.add_argument('file', help='prefix name of the file to write to') + options = arg_parser.parse_args() + time = datetime.now() + delta = timedelta(seconds=5) + nodes = [f'node_{i + 1:03d}' for i in range(options.nr_nodes)] + resource_manager = ResoureManager(nodes) + for _ in range(options.nr_jobs): + resource_manager.submit() + with open(f'{options.file}_load.csv', 'w', newline='') as csv_load_file: + csv_load_writer = csv.writer(csv_load_file) + csv_load_writer.writerow(['time', 'node', 'cpu_load', 'mem_load']) + with open(f'{options.file}_jobs.csv', 'w', newline='') as csv_jobs_file: + csv_jobs_writer = csv.writer(csv_jobs_file) + csv_jobs_writer.writerow(['time', 'job_id', 'node']) + for _ in range(options.nr_timesteps): + print('-'*36) + print(f'---- {time} ----') + time_str = datetime.strftime(time, '%Y-%m-%d %H:%M:%S') + resource_manager.qstat() + for node in nodes: + if resource_manager.is_busy(node): + cpu_load = f'{50 + 50*random.random():.2f}' + mem_load = f'{10 + 90*random.random():.2f}' + else: + cpu_load = f'{10*random.random():.2f}' + mem_load = f'{10*random.random():.2f}' + csv_load_writer.writerow([time_str, node, cpu_load, mem_load]) + resource_manager.cycle(delta) + for job in resource_manager.running_jobs: + for node in job.nodes: + csv_jobs_writer.writerow([time_str, job.job_id, node]) + time += delta diff --git a/source-code/dash/jobs_overview.py b/source-code/dash/jobs_overview.py new file mode 100755 index 0000000..be0a0a8 --- /dev/null +++ b/source-code/dash/jobs_overview.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser +import dash +import dash_core_components as dcc +import dash_html_components as html +from dash.dependencies import Input, Output +from datetime import datetime +import pandas as pd + + +def generate_overview_table(jobs): + jobs_summary = jobs[['job_id', 'node']]\ + .drop_duplicates()\ + .groupby('job_id')\ + .count() + jobs_summary = jobs_summary.merge(jobs.groupby('job_id')[['time']].min(), + on='job_id') + jobs_summary = jobs_summary.merge(jobs.groupby('job_id')[['time']].max(), + on='job_id') + jobs_summary.columns = ['nodes', 'start time', 'end time'] + jobs_summary.insert(0, 'job_id', jobs_summary.index) + return html.Table([ + html.Thead( + html.Tr([html.Th(column) for column in jobs_summary.columns]) + ), + html.Tbody([ + html.Tr([ + html.Td(jobs_summary.loc[job_id][column]) + for column in jobs_summary.columns]) + for job_id in jobs_summary.index] + )] + ) + +def generate_job_menu(jobs): + job_list = [{'label': str(job_id), 'value': str(job_id)} + for job_id in jobs.job_id.unique()] + if len(job_list): + return dcc.Dropdown( + id='job_menu', + options=job_list, + value=job_list[0]['value'], + placeholder='Select a job...', + ) + + + +if __name__ == '__main__': + arg_parsr = ArgumentParser(description='visualize job performance') + arg_parsr.add_argument('--job-file', required=True, + help='file with job/node information') + arg_parsr.add_argument('--load-file', required=True, + help='file with node/load information') + options = arg_parsr.parse_args() + jobs = pd.read_csv(options.job_file, parse_dates=True, + date_parser=pd.to_datetime) + loads = pd.read_csv(options.load_file, parse_dates=True, + date_parser=pd.to_datetime) + external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css'] + app = dash.Dash(__name__, external_stylesheets=external_stylesheets) + app.layout = html.Div(children=[ + html.H1(children='Job overview'), + html.Div(children=[ + html.Div(children=[generate_overview_table(jobs)], + style={'width': '70%', 'display': 'inline-block', + 'margin': '20pt'}), + html.Div(children=[ + html.Label('Jobs'), + generate_job_menu(jobs), + ], style={'width': '10%', 'float': 'left', 'display': 'inline-block', + 'margin': '20pt'}) + ]), + html.Div(children=[ + html.Div(children=[ dcc.Graph(id='cpu_load_graph')], + style={'width': '45%', 'display': 'inline-block'}), + html.Div(children=[dcc.Graph(id='mem_load_graph')], + style={'widht': '45%', 'float': 'right', 'display': 'inline-block'}) + ]) + ]) + + @app.callback( + [Output('cpu_load_graph', 'figure'), Output('mem_load_graph', 'figure')], + [Input('job_menu', 'value')]) + def update(job_id): + data = loads.merge(jobs.query(f'job_id == {job_id}'), on=['time', 'node'], + how='right') + figures = list() + for quantity in ['cpu', 'mem']: + nodes_data = list() + for node_df in data.groupby('node'): + node_data = dict() + node_data['name'] = node_df[0] + node_data['x'] = node_df[1].time + node_data['y'] = node_df[1][f'{quantity}_load'] + node_data['mode'] = ['markers', 'line'], + nodes_data.append(node_data) + figures.append({ + 'data': nodes_data, + 'layout': dict( + xaxis={'title': 'time'}, + yaxis={'title': f'{quantity} load'}, + hovermode='closest'), + }) + return figures + + + app.run_server(debug=True)