From 14135b9c7d47348176028a5510bec81a83def3c5 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 15 Sep 2020 21:09:34 +0200 Subject: [PATCH] Emit dataframe to different data sinks - Export dataframe to different data sinks like SQLite, DuckDB, InfluxDB and CrateDB - Query results with SQL, based on in-memory DuckDB - Many features from cli.py now available through io.py - Tests for cli.py, run.py and io.py --- .coveragerc | 2 - .flake8 | 2 +- CHANGELOG.rst | 2 + README.rst | 10 +- docs/pages/api.rst | 86 ++- docs/pages/cli.rst | 60 +- docs/pages/development.rst | 2 +- noxfile.py | 4 +- poetry.lock | 1222 +++++++++++++++++++----------------- pyproject.toml | 4 +- tests/test_cli.py | 12 + tests/test_io.py | 177 ++++++ tests/test_run.py | 18 + wetterdienst/__init__.py | 1 + wetterdienst/cli.py | 155 +++-- wetterdienst/io.py | 369 +++++++++++ 16 files changed, 1436 insertions(+), 690 deletions(-) create mode 100644 tests/test_io.py create mode 100644 tests/test_run.py create mode 100644 wetterdienst/io.py diff --git a/.coveragerc b/.coveragerc index 1c162ea2e..fdb4c4338 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,7 +7,5 @@ source = show_missing = true fail_under = 0 omit = - wetterdienst/cli.py - wetterdienst/run.py tests/* noxfile.py diff --git a/.flake8 b/.flake8 index 74a039484..1567f0d6f 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ select = B,B9,BLK,C,E,F,W,S ignore = E203,W503 max-line-length = 88 -per-file-ignores = tests/*:S101, wetterdienst/__init__.py:F401, wetterdienst/cli.py:E501,B950 +per-file-ignores = tests/*:S101, wetterdienst/__init__.py:F401, wetterdienst/cli.py:E501,B950, wetterdienst/io.py:E501,B950 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 049bae77f..2694d8bae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,8 @@ Development - Add TTL-based persistent caching using dogpile.cache - Add ``example/radolan.py`` and adjust documentation +- Export dataframe to different data sinks like SQLite, DuckDB, InfluxDB and CrateDB +- Query results with SQL, based on in-memory DuckDB 0.7.0 (16.09.2020) ================== diff --git a/README.rst b/README.rst index be3ed97c0..8d109f4f1 100644 --- a/README.rst +++ b/README.rst @@ -64,13 +64,15 @@ Details - Get metadata for a set of Parameter, PeriodType and TimeResolution. - Get station(s) nearby a selected location for a given set. - Store/recover collected data. -- Docker image to run the library dockerized. -- Client to run the library from command line. +- Command line interface. +- Run SQL queries on the results. +- Export results to databases and other data sinks. +- Public Docker image on ghcr.io. Setup ***** -Run the following to make ``wetterdienst`` available in your current environment: +Run this to make ``wetterdienst`` available in your current environment: .. code-block:: bash @@ -115,7 +117,7 @@ Documentation We strongly recommend reading the full documentation, which will be updated continuously as we make progress with this library: - - https://wetterdienst.readthedocs.io/ +- https://wetterdienst.readthedocs.io/ For the whole functionality, check out the `Wetterdienst API`_ section of our documentation, which will be constantly updated. To stay up to date with the diff --git a/docs/pages/api.rst b/docs/pages/api.rst index 1f71f1ba9..514533bdb 100644 --- a/docs/pages/api.rst +++ b/docs/pages/api.rst @@ -1,19 +1,30 @@ ### API ### -The API offers access to different data products. They are -outlined in more detail within the :ref:`data-coverage` chapter. .. contents:: :local: :depth: 1 +************ +Introduction +************ +The API offers access to different data products. They are +outlined in more detail within the :ref:`data-coverage` chapter. +Please also check out complete examples about how to use the API in the +`example `_ +folder. +In order to explore all features interactively, +you might want to try the :ref:`cli`. + + ************ Observations ************ Acquire historical weather data through requesting by *parameter*, *time resolution* and *period type*. + Request arguments ================= The options *parameter*, *time resolution* and *period type* can be used in three ways: @@ -146,9 +157,68 @@ can be used to download the observation data: Et voila: We just got the data we wanted for our location and are ready to analyse the temperature on historical developments. -Please also check out more advanced examples in the -`example `_ -folder on Github. + +SQL support +=========== +Querying data using SQL is provided by an in-memory DuckDB_ database. +In order to explore what is possible, please have a look at the `DuckDB SQL introduction`_. + +.. code-block:: python + + from wetterdienst import DWDStationRequest, DataPackage + from wetterdienst import Parameter, PeriodType, TimeResolution + + request = DWDStationRequest( + station_ids=[1048], + parameter=[Parameter.TEMPERATURE_AIR], + time_resolution=TimeResolution.HOURLY, + start_date="2019-01-01", + end_date="2020-01-01", + tidy_data=True, + humanize_column_names=True, + prefer_local=True, + write_file=True, + ) + + data = DataPackage(request=request) + data.lowercase_fieldnames() + df = data.filter_by_sql("SELECT * FROM data WHERE element='temperature_air_200' AND value < -7.0;") + print(df) + + +Data export +=========== +Data can be exported to SQLite_, DuckDB_, InfluxDB_, CrateDB_ and more targets. +A target is identified by a connection string. + +Examples: + +- sqlite:///dwd.sqlite?table=weather +- duckdb:///dwd.duckdb?table=weather +- influxdb://localhost/?database=dwd&table=weather +- crate://localhost/?database=dwd&table=weather + +.. code-block:: python + + from wetterdienst import DWDStationRequest, DataPackage + from wetterdienst import Parameter, PeriodType, TimeResolution + + request = DWDStationRequest( + station_ids=[1048], + parameter=[Parameter.TEMPERATURE_AIR], + time_resolution=TimeResolution.HOURLY, + start_date="2019-01-01", + end_date="2020-01-01", + tidy_data=True, + humanize_column_names=True, + prefer_local=True, + write_file=True, + ) + + data = DataPackage(request=request) + data.lowercase_fieldnames() + data.export("influxdb://localhost/?database=dwd&table=weather") + ****** MOSMIX @@ -193,3 +263,9 @@ For a more thorough example, please have a look at `example/radolan.py`_. .. _wradlib: https://wradlib.org/ .. _example/radolan.py: https://github.com/earthobservations/wetterdienst/blob/master/example/radolan.py + +.. _SQLite: https://www.sqlite.org/ +.. _DuckDB: https://duckdb.org/docs/sql/introduction +.. _DuckDB SQL introduction: https://duckdb.org/docs/sql/introduction +.. _InfluxDB: https://github.com/influxdata/influxdb +.. _CrateDB: https://github.com/crate/crate diff --git a/docs/pages/cli.rst b/docs/pages/cli.rst index 13963ecbb..55e46a37a 100644 --- a/docs/pages/cli.rst +++ b/docs/pages/cli.rst @@ -1,3 +1,5 @@ +.. _cli: + ###################### Command line interface ###################### @@ -7,10 +9,11 @@ Command line interface $ wetterdienst --help Usage: - wetterdienst stations --parameter= --resolution= --period= [--station=] [--latitude=] [--longitude=] [--number=] [--distance=] [--persist] [--format=] - wetterdienst readings --parameter= --resolution= --period= --station= [--persist] [--date=] [--format=] - wetterdienst readings --parameter= --resolution= --period= --latitude= --longitude= [--number=] [--distance=] [--persist] [--date=] [--format=] + wetterdienst stations --parameter= --resolution= --period= [--station=] [--latitude=] [--longitude=] [--number=] [--distance=] [--persist] [--sql=] [--format=] + wetterdienst readings --parameter= --resolution= --period= --station= [--persist] [--date=] [--sql=] [--format=] [--target=] + wetterdienst readings --parameter= --resolution= --period= --latitude= --longitude= [--number=] [--distance=] [--persist] [--date=] [--sql=] [--format=] [--target=] wetterdienst about [parameters] [resolutions] [periods] + wetterdienst about coverage [--parameter=] [--resolution=] [--period=] wetterdienst --version wetterdienst (-h | --help) @@ -26,7 +29,9 @@ Command line interface --persist Save and restore data to filesystem w/o going to the network --date= Date for filtering data. Can be either a single date(time) or an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. + --sql= SQL query to apply to DataFrame. --format= Output format. [Default: json] + --target= Output target for storing data into different data sinks. --version Show version information --debug Enable debug messages -h --help Show this screen @@ -87,3 +92,52 @@ Command line interface # Acquire stations and readings by geoposition, request stations within specific radius. wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30 + + Examples using SQL filtering: + + # Find stations by state. + wetterdienst stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'" + + # Find stations by name (LIKE query). + wetterdienst stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')" + + # Find stations by name (regexp query). + wetterdienst stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))" + + # Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees. + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE element='temperature_air_max_200' AND value < 2.0;" + + Examples for inquiring metadata: + + # Display list of available parameters (air_temperature, precipitation, pressure, ...) + wetterdienst about parameters + + # Display list of available resolutions (10_minutes, hourly, daily, ...) + wetterdienst about resolutions + + # Display list of available periods (historical, recent, now) + wetterdienst about periods + + # Display coverage/correlation between parameters, resolutions and periods. + # This can answer questions like ... + wetterdienst about coverage + + # Tell me all periods and resolutions available for 'air_temperature'. + wetterdienst about coverage --parameter=air_temperature + + # Tell me all parameters available for 'daily' resolution. + wetterdienst about coverage --resolution=daily + + Examples for exporting data to databases: + + # Shortcut command for fetching readings from DWD + alias fetch="wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent" + + # Store readings to DuckDB + fetch --target="duckdb://database=dwd.duckdb&table=weather" + + # Store readings to InfluxDB + fetch --target="influxdb://localhost/?database=dwd&table=weather" + + # Store readings to CrateDB + fetch --target="crate://localhost/?database=dwd&table=weather" diff --git a/docs/pages/development.rst b/docs/pages/development.rst index 49e60006b..d7b7fd9b7 100644 --- a/docs/pages/development.rst +++ b/docs/pages/development.rst @@ -50,6 +50,6 @@ This will inform you in case of problems with tests and your code format. In order to run the tests more **quickly**:: - poetry install --extras=excel + poetry install --extras=sql --extras=excel poetry shell pytest -vvvv -m "not (remote or slow)" diff --git a/noxfile.py b/noxfile.py index 3bb59d71e..c06a3a0ea 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,7 +11,7 @@ @nox.session(python=["3.6", "3.7", "3.8"]) def tests(session): """Run tests.""" - session.run("poetry", "install", "--no-dev", "--extras=excel", external=True) + session.run("poetry", "install", "--no-dev", "--extras=sql", "--extras=excel", external=True) install_with_constraints(session, "pytest", "pytest-notebook", "matplotlib", "mock") session.run("pytest") @@ -19,7 +19,7 @@ def tests(session): @nox.session(python=["3.7"]) def coverage(session: Session) -> None: """Run tests and upload coverage data.""" - session.run("poetry", "install", "--no-dev", "--extras=excel", external=True) + session.run("poetry", "install", "--no-dev", "--extras=sql", "--extras=excel", external=True) install_with_constraints( session, "coverage[toml]", diff --git a/poetry.lock b/poetry.lock index 5081064ee..705155ca8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,85 +1,85 @@ [[package]] -name = "aiofiles" -version = "0.4.0" -description = "File support for asyncio." category = "main" +description = "File support for asyncio." +name = "aiofiles" optional = false python-versions = "*" +version = "0.4.0" [[package]] -name = "alabaster" -version = "0.7.12" -description = "A configurable sidebar-enabled Sphinx theme" category = "main" +description = "A configurable sidebar-enabled Sphinx theme" +name = "alabaster" optional = true python-versions = "*" +version = "0.7.12" [[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "main" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +name = "appdirs" optional = false python-versions = "*" +version = "1.4.4" [[package]] -name = "appnope" -version = "0.1.0" -description = "Disable App Nap on OS X 10.9" category = "main" +description = "Disable App Nap on OS X 10.9" +marker = "sys_platform == \"darwin\" or platform_system == \"Darwin\"" +name = "appnope" optional = false python-versions = "*" -marker = "sys_platform == \"darwin\" or platform_system == \"Darwin\"" +version = "0.1.0" [[package]] -name = "argcomplete" -version = "1.12.0" -description = "Bash tab completion for argparse" category = "dev" +description = "Bash tab completion for argparse" +name = "argcomplete" optional = false python-versions = "*" - -[package.extras] -test = ["coverage", "flake8", "pexpect", "wheel"] +version = "1.12.0" [package.dependencies] [package.dependencies.importlib-metadata] -version = ">=0.23,<2" python = ">=3.6,<3.7" +version = ">=0.23,<2" + +[package.extras] +test = ["coverage", "flake8", "pexpect", "wheel"] [[package]] -name = "argon2-cffi" -version = "20.1.0" -description = "The secure Argon2 password hashing algorithm." category = "dev" +description = "The secure Argon2 password hashing algorithm." +name = "argon2-cffi" optional = false python-versions = "*" +version = "20.1.0" + +[package.dependencies] +cffi = ">=1.0.0" +six = "*" [package.extras] dev = ["coverage (>=5.0.2)", "hypothesis", "pytest", "sphinx", "wheel", "pre-commit"] docs = ["sphinx"] tests = ["coverage (>=5.0.2)", "hypothesis", "pytest"] -[package.dependencies] -cffi = ">=1.0.0" -six = "*" - [[package]] -name = "atomicwrites" -version = "1.4.0" -description = "Atomic file writes." category = "dev" +description = "Atomic file writes." +marker = "sys_platform == \"win32\"" +name = "atomicwrites" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -marker = "sys_platform == \"win32\"" +version = "1.4.0" [[package]] -name = "attrs" -version = "20.2.0" -description = "Classes Without Boilerplate" category = "dev" +description = "Classes Without Boilerplate" +name = "attrs" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.2.0" [package.extras] dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] @@ -88,65 +88,61 @@ tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six tests_no_zope = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] [[package]] -name = "babel" -version = "2.8.0" -description = "Internationalization utilities" category = "main" +description = "Internationalization utilities" +name = "babel" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.8.0" [package.dependencies] pytz = ">=2015.7" [[package]] -name = "backcall" -version = "0.2.0" -description = "Specifications for callback functions passed in to an API" category = "main" +description = "Specifications for callback functions passed in to an API" +name = "backcall" optional = false python-versions = "*" +version = "0.2.0" [[package]] -name = "bandit" -version = "1.6.2" -description = "Security oriented static analyser for python code." category = "dev" +description = "Security oriented static analyser for python code." +name = "bandit" optional = false python-versions = "*" +version = "1.6.2" [package.dependencies] -colorama = ">=0.3.9" GitPython = ">=1.0.1" PyYAML = ">=3.13" +colorama = ">=0.3.9" six = ">=1.10.0" stevedore = ">=1.20.0" [[package]] -name = "beautifulsoup4" -version = "4.9.1" -description = "Screen-scraping library" category = "main" +description = "Screen-scraping library" +name = "beautifulsoup4" optional = false python-versions = "*" +version = "4.9.1" + +[package.dependencies] +soupsieve = [">1.2", "<2.0"] [package.extras] html5lib = ["html5lib"] lxml = ["lxml"] -[package.dependencies] -soupsieve = [">1.2", "<2.0"] - [[package]] -name = "black" -version = "20.8b1" -description = "The uncompromising code formatter." category = "dev" +description = "The uncompromising code formatter." +name = "black" optional = false python-versions = ">=3.6" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +version = "20.8b1" [package.dependencies] appdirs = "*" @@ -159,16 +155,20 @@ typed-ast = ">=1.4.0" typing-extensions = ">=3.7.4" [package.dependencies.dataclasses] -version = ">=0.6" python = "<3.7" +version = ">=0.6" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] -name = "bleach" -version = "3.2.1" -description = "An easy safelist-based HTML-sanitizing tool." category = "dev" +description = "An easy safelist-based HTML-sanitizing tool." +name = "bleach" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "3.2.1" [package.dependencies] packaging = "*" @@ -176,118 +176,118 @@ six = ">=1.9.0" webencodings = "*" [[package]] -name = "cachetools" -version = "3.1.1" -description = "Extensible memoizing collections and decorators" category = "main" +description = "Extensible memoizing collections and decorators" +name = "cachetools" optional = false python-versions = "*" +version = "3.1.1" [[package]] -name = "certifi" -version = "2020.6.20" -description = "Python package for providing Mozilla's CA Bundle." category = "main" +description = "Python package for providing Mozilla's CA Bundle." +name = "certifi" optional = false python-versions = "*" +version = "2020.6.20" [[package]] -name = "cffi" -version = "1.14.3" -description = "Foreign Function Interface for Python calling C code." category = "dev" +description = "Foreign Function Interface for Python calling C code." +name = "cffi" optional = false python-versions = "*" +version = "1.14.3" [package.dependencies] pycparser = "*" [[package]] -name = "chardet" -version = "3.0.4" -description = "Universal encoding detector for Python 2 and 3" category = "main" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" optional = false python-versions = "*" +version = "3.0.4" [[package]] -name = "click" -version = "7.1.2" -description = "Composable command line interface toolkit" category = "dev" +description = "Composable command line interface toolkit" +name = "click" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "7.1.2" [[package]] -name = "colorama" -version = "0.4.3" -description = "Cross-platform colored terminal text." category = "main" +description = "Cross-platform colored terminal text." +name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.4.3" [[package]] -name = "colorlog" -version = "4.2.1" -description = "Log formatting with colors!" category = "dev" +description = "Log formatting with colors!" +name = "colorlog" optional = false python-versions = "*" +version = "4.2.1" [package.dependencies] colorama = "*" [[package]] -name = "coverage" -version = "5.2.1" -description = "Code coverage measurement for Python" category = "dev" +description = "Code coverage measurement for Python" +name = "coverage" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" - -[package.extras] -toml = ["toml"] +version = "5.2.1" [package.dependencies] [package.dependencies.toml] -version = "*" optional = true +version = "*" + +[package.extras] +toml = ["toml"] [[package]] -name = "css-html-js-minify" -version = "2.5.5" -description = "CSS HTML JS Minifier" category = "main" +description = "CSS HTML JS Minifier" +name = "css-html-js-minify" optional = true python-versions = ">=3.6" +version = "2.5.5" [[package]] -name = "cycler" -version = "0.10.0" -description = "Composable style cycles" category = "main" +description = "Composable style cycles" +name = "cycler" optional = true python-versions = "*" +version = "0.10.0" [package.dependencies] six = "*" [[package]] -name = "dataclasses" -version = "0.6" -description = "A backport of the dataclasses module for Python 3.6" category = "dev" +description = "A backport of the dataclasses module for Python 3.6" +marker = "python_version < \"3.7\"" +name = "dataclasses" optional = false python-versions = "*" -marker = "python_version < \"3.7\"" +version = "0.6" [[package]] -name = "dateparser" -version = "0.7.6" -description = "Date parsing library designed to parse dates from HTML pages" category = "main" +description = "Date parsing library designed to parse dates from HTML pages" +name = "dateparser" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.7.6" [package.dependencies] python-dateutil = "*" @@ -296,100 +296,111 @@ regex = "!=2019.02.19" tzlocal = "*" [[package]] -name = "decorator" -version = "4.4.2" -description = "Decorators for Humans" category = "main" +description = "Decorators for Humans" +name = "decorator" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "4.4.2" [[package]] -name = "defusedxml" -version = "0.6.0" -description = "XML bomb protection for Python stdlib modules" category = "dev" +description = "XML bomb protection for Python stdlib modules" +name = "defusedxml" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.6.0" [[package]] -name = "distlib" -version = "0.3.1" -description = "Distribution utilities" category = "dev" +description = "Distribution utilities" +name = "distlib" optional = false python-versions = "*" +version = "0.3.1" [[package]] -name = "docopt" -version = "0.6.2" -description = "Pythonic argument parser, that will make you smile" category = "main" +description = "Pythonic argument parser, that will make you smile" +name = "docopt" optional = false python-versions = "*" +version = "0.6.2" [[package]] -name = "docutils" -version = "0.16" -description = "Docutils -- Python Documentation Utilities" category = "main" +description = "Docutils -- Python Documentation Utilities" +name = "docutils" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.16" [[package]] -name = "dogpile.cache" -version = "1.0.2" -description = "A caching front-end based on the Dogpile lock." category = "main" +description = "A caching front-end based on the Dogpile lock." +name = "dogpile.cache" optional = false python-versions = ">=3.6" +version = "1.0.2" [package.dependencies] decorator = ">=4.0.0" stevedore = ">=3.0.0" [[package]] -name = "entrypoints" -version = "0.3" -description = "Discover and load entry points from installed packages." -category = "dev" -optional = false -python-versions = ">=2.7" - -[[package]] -name = "et-xmlfile" -version = "1.0.1" -description = "An implementation of lxml.xmlfile for the standard library" category = "main" +description = "DuckDB embedded database" +name = "duckdb" optional = true python-versions = "*" +version = "0.2.2.dev175" + +[package.dependencies] +numpy = ">=1.14" [[package]] -name = "filelock" -version = "3.0.12" -description = "A platform independent file lock." category = "dev" +description = "Discover and load entry points from installed packages." +name = "entrypoints" optional = false +python-versions = ">=2.7" +version = "0.3" + +[[package]] +category = "main" +description = "An implementation of lxml.xmlfile for the standard library" +name = "et-xmlfile" +optional = true python-versions = "*" +version = "1.0.1" + +[[package]] +category = "dev" +description = "A platform independent file lock." +name = "filelock" +optional = false +python-versions = "*" +version = "3.0.12" [[package]] -name = "fire" -version = "0.3.1" -description = "A library for automatically generating command line interfaces." category = "main" +description = "A library for automatically generating command line interfaces." +name = "fire" optional = false python-versions = "*" +version = "0.3.1" [package.dependencies] six = "*" termcolor = "*" [[package]] -name = "flake8" -version = "3.8.3" -description = "the modular source code checker: pep8 pyflakes and co" category = "dev" +description = "the modular source code checker: pep8 pyflakes and co" +name = "flake8" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +version = "3.8.3" [package.dependencies] mccabe = ">=0.6.0,<0.7.0" @@ -397,16 +408,16 @@ pycodestyle = ">=2.6.0a1,<2.7.0" pyflakes = ">=2.2.0,<2.3.0" [package.dependencies.importlib-metadata] -version = "*" python = "<3.8" +version = "*" [[package]] -name = "flake8-bandit" -version = "2.1.2" -description = "Automated security testing with bandit and flake8." category = "dev" +description = "Automated security testing with bandit and flake8." +name = "flake8-bandit" optional = false python-versions = "*" +version = "2.1.2" [package.dependencies] bandit = "*" @@ -415,140 +426,137 @@ flake8-polyfill = "*" pycodestyle = "*" [[package]] -name = "flake8-black" -version = "0.2.1" -description = "flake8 plugin to call black as a code style validator" category = "dev" +description = "flake8 plugin to call black as a code style validator" +name = "flake8-black" optional = false python-versions = "*" +version = "0.2.1" [package.dependencies] black = "*" flake8 = ">=3.0.0" [[package]] -name = "flake8-bugbear" -version = "20.1.4" -description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." category = "dev" +description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." +name = "flake8-bugbear" optional = false python-versions = ">=3.6" +version = "20.1.4" [package.dependencies] attrs = ">=19.2.0" flake8 = ">=3.0.0" [[package]] -name = "flake8-polyfill" -version = "1.0.2" -description = "Polyfill package for Flake8 plugins" category = "dev" +description = "Polyfill package for Flake8 plugins" +name = "flake8-polyfill" optional = false python-versions = "*" +version = "1.0.2" [package.dependencies] flake8 = "*" [[package]] -name = "gitdb" -version = "4.0.5" -description = "Git Object Database" category = "dev" +description = "Git Object Database" +name = "gitdb" optional = false python-versions = ">=3.4" +version = "4.0.5" [package.dependencies] smmap = ">=3.0.1,<4" [[package]] -name = "gitpython" -version = "3.1.8" -description = "Python Git Library" category = "dev" +description = "Python Git Library" +name = "gitpython" optional = false python-versions = ">=3.4" +version = "3.1.8" [package.dependencies] gitdb = ">=4.0.1,<5" [[package]] -name = "h5py" -version = "2.10.0" -description = "Read and write HDF5 files from Python" category = "main" +description = "Read and write HDF5 files from Python" +name = "h5py" optional = false python-versions = "*" +version = "2.10.0" [package.dependencies] numpy = ">=1.7" six = "*" [[package]] -name = "idna" -version = "2.10" -description = "Internationalized Domain Names in Applications (IDNA)" category = "main" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.10" [[package]] -name = "imagesize" -version = "1.2.0" -description = "Getting image size from png/jpeg/jpeg2000/gif file" category = "main" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +name = "imagesize" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.2.0" [[package]] -name = "importlib-metadata" -version = "1.7.0" -description = "Read metadata from Python packages" category = "main" +description = "Read metadata from Python packages" +marker = "python_version < \"3.8\"" +name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -marker = "python_version < \"3.8\"" +version = "1.7.0" + +[package.dependencies] +zipp = ">=0.5" [package.extras] docs = ["sphinx", "rst.linker"] testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] -[package.dependencies] -zipp = ">=0.5" - [[package]] -name = "importlib-resources" -version = "3.0.0" -description = "Read resources from Python packages" category = "dev" +description = "Read resources from Python packages" +name = "importlib-resources" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[package.extras] -docs = ["sphinx", "rst.linker", "jaraco.packaging"] +version = "3.0.0" [package.dependencies] [package.dependencies.zipp] -version = ">=0.4" python = "<3.8" +version = ">=0.4" + +[package.extras] +docs = ["sphinx", "rst.linker", "jaraco.packaging"] [[package]] -name = "iniconfig" -version = "1.0.1" -description = "iniconfig: brain-dead simple config-ini parsing" category = "dev" +description = "iniconfig: brain-dead simple config-ini parsing" +name = "iniconfig" optional = false python-versions = "*" +version = "1.0.1" [[package]] -name = "ipykernel" -version = "5.3.4" -description = "IPython Kernel for Jupyter" category = "dev" +description = "IPython Kernel for Jupyter" +name = "ipykernel" optional = false python-versions = ">=3.5" - -[package.extras] -test = ["pytest (!=5.3.4)", "pytest-cov", "flaky", "nose"] +version = "5.3.4" [package.dependencies] appnope = "*" @@ -557,24 +565,16 @@ jupyter-client = "*" tornado = ">=4.2" traitlets = ">=4.1.0" +[package.extras] +test = ["pytest (!=5.3.4)", "pytest-cov", "flaky", "nose"] + [[package]] -name = "ipython" -version = "7.16.1" -description = "IPython: Productive Interactive Computing" category = "main" +description = "IPython: Productive Interactive Computing" +name = "ipython" optional = false python-versions = ">=3.6" - -[package.extras] -all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.14)", "pygments", "qtconsole", "requests", "testpath"] -doc = ["Sphinx (>=1.3)"] -kernel = ["ipykernel"] -nbconvert = ["nbconvert"] -nbformat = ["nbformat"] -notebook = ["notebook", "ipywidgets"] -parallel = ["ipyparallel"] -qtconsole = ["qtconsole"] -test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.14)"] +version = "7.16.1" [package.dependencies] appnope = "*" @@ -589,62 +589,69 @@ pygments = "*" setuptools = ">=18.5" traitlets = ">=4.2" +[package.extras] +all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.14)", "pygments", "qtconsole", "requests", "testpath"] +doc = ["Sphinx (>=1.3)"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["notebook", "ipywidgets"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.14)"] + [[package]] -name = "ipython-genutils" -version = "0.2.0" -description = "Vestigial utilities from IPython" category = "main" +description = "Vestigial utilities from IPython" +name = "ipython-genutils" optional = false python-versions = "*" +version = "0.2.0" [[package]] -name = "jdcal" -version = "1.4.1" -description = "Julian dates from proleptic Gregorian and Julian calendars." category = "main" +description = "Julian dates from proleptic Gregorian and Julian calendars." +name = "jdcal" optional = true python-versions = "*" +version = "1.4.1" [[package]] -name = "jedi" -version = "0.17.2" -description = "An autocompletion tool for Python that can be used for text editors." category = "main" +description = "An autocompletion tool for Python that can be used for text editors." +name = "jedi" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.17.2" + +[package.dependencies] +parso = ">=0.7.0,<0.8.0" [package.extras] qa = ["flake8 (3.7.9)"] testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] -[package.dependencies] -parso = ">=0.7.0,<0.8.0" - [[package]] -name = "jinja2" -version = "2.11.2" -description = "A very fast and expressive template engine." category = "main" +description = "A very fast and expressive template engine." +name = "jinja2" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -i18n = ["Babel (>=0.8)"] +version = "2.11.2" [package.dependencies] MarkupSafe = ">=0.23" +[package.extras] +i18n = ["Babel (>=0.8)"] + [[package]] -name = "jsonschema" -version = "3.2.0" -description = "An implementation of JSON Schema validation for Python" category = "dev" +description = "An implementation of JSON Schema validation for Python" +name = "jsonschema" optional = false python-versions = "*" - -[package.extras] -format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] -format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"] +version = "3.2.0" [package.dependencies] attrs = ">=17.4.0" @@ -653,19 +660,20 @@ setuptools = "*" six = ">=1.11.0" [package.dependencies.importlib-metadata] -version = "*" python = "<3.8" +version = "*" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"] [[package]] -name = "jupyter-client" -version = "6.1.7" -description = "Jupyter protocol implementation and client libraries" category = "dev" +description = "Jupyter protocol implementation and client libraries" +name = "jupyter-client" optional = false python-versions = ">=3.5" - -[package.extras] -test = ["ipykernel", "ipython", "mock", "pytest", "pytest-asyncio", "async-generator", "pytest-timeout"] +version = "6.1.7" [package.dependencies] jupyter-core = ">=4.6.0" @@ -674,33 +682,36 @@ pyzmq = ">=13" tornado = ">=4.1" traitlets = "*" +[package.extras] +test = ["ipykernel", "ipython", "mock", "pytest", "pytest-asyncio", "async-generator", "pytest-timeout"] + [[package]] -name = "jupyter-core" -version = "4.6.3" -description = "Jupyter core package. A base package on which Jupyter projects rely." category = "dev" +description = "Jupyter core package. A base package on which Jupyter projects rely." +name = "jupyter-core" optional = false python-versions = "!=3.0,!=3.1,!=3.2,!=3.3,!=3.4,>=2.7" +version = "4.6.3" [package.dependencies] pywin32 = ">=1.0" traitlets = "*" [[package]] -name = "kiwisolver" -version = "1.2.0" -description = "A fast implementation of the Cassowary constraint solver" category = "main" +description = "A fast implementation of the Cassowary constraint solver" +name = "kiwisolver" optional = true python-versions = ">=3.6" +version = "1.2.0" [[package]] -name = "lxml" -version = "4.5.2" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +name = "lxml" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" +version = "4.5.2" [package.extras] cssselect = ["cssselect (>=0.7)"] @@ -709,20 +720,20 @@ htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] [[package]] -name = "markupsafe" -version = "1.1.1" -description = "Safely add untrusted strings to HTML/XML markup." category = "main" +description = "Safely add untrusted strings to HTML/XML markup." +name = "markupsafe" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +version = "1.1.1" [[package]] -name = "matplotlib" -version = "3.3.2" -description = "Python plotting package" category = "main" +description = "Python plotting package" +name = "matplotlib" optional = true python-versions = ">=3.6" +version = "3.3.2" [package.dependencies] certifi = ">=2020.06.20" @@ -734,28 +745,28 @@ pyparsing = ">=2.0.3,<2.0.4 || >2.0.4,<2.1.2 || >2.1.2,<2.1.6 || >2.1.6" python-dateutil = ">=2.1" [[package]] -name = "mccabe" -version = "0.6.1" -description = "McCabe checker, plugin for flake8" category = "dev" +description = "McCabe checker, plugin for flake8" +name = "mccabe" optional = false python-versions = "*" +version = "0.6.1" [[package]] -name = "mistune" -version = "0.8.4" -description = "The fastest markdown parser in pure Python" category = "dev" +description = "The fastest markdown parser in pure Python" +name = "mistune" optional = false python-versions = "*" +version = "0.8.4" [[package]] -name = "mock" -version = "4.0.2" -description = "Rolling backport of unittest.mock for all Pythons" category = "dev" +description = "Rolling backport of unittest.mock for all Pythons" +name = "mock" optional = false python-versions = ">=3.6" +version = "4.0.2" [package.extras] build = ["twine", "wheel", "blurb"] @@ -763,50 +774,43 @@ docs = ["sphinx"] test = ["pytest", "pytest-cov"] [[package]] -name = "more-itertools" -version = "8.5.0" -description = "More routines for operating on iterables, beyond itertools" category = "dev" +description = "More routines for operating on iterables, beyond itertools" +name = "more-itertools" optional = false python-versions = ">=3.5" +version = "8.5.0" [[package]] -name = "munch" -version = "2.5.0" -description = "A dot-accessible dictionary (a la JavaScript objects)" category = "main" +description = "A dot-accessible dictionary (a la JavaScript objects)" +name = "munch" optional = false python-versions = "*" +version = "2.5.0" + +[package.dependencies] +six = "*" [package.extras] testing = ["pytest", "coverage", "astroid (>=1.5.3,<1.6.0)", "pylint (>=1.7.2,<1.8.0)", "astroid (>=2.0)", "pylint (>=2.3.1,<2.4.0)"] yaml = ["PyYAML (>=5.1.0)"] -[package.dependencies] -six = "*" - [[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." category = "dev" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +name = "mypy-extensions" optional = false python-versions = "*" +version = "0.4.3" [[package]] -name = "nbconvert" -version = "5.6.1" -description = "Converting Jupyter Notebooks" category = "dev" +description = "Converting Jupyter Notebooks" +name = "nbconvert" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -all = ["pytest", "pytest-cov", "ipykernel", "jupyter-client (>=5.3.1)", "ipywidgets (>=7)", "pebble", "tornado (>=4.0)", "sphinx (>=1.5.1)", "sphinx-rtd-theme", "nbsphinx (>=0.2.12)", "sphinxcontrib-github-alt", "ipython", "mock"] -docs = ["sphinx (>=1.5.1)", "sphinx-rtd-theme", "nbsphinx (>=0.2.12)", "sphinxcontrib-github-alt", "ipython", "jupyter-client (>=5.3.1)"] -execute = ["jupyter-client (>=5.3.1)"] -serve = ["tornado (>=4.0)"] -test = ["pytest", "pytest-cov", "ipykernel", "jupyter-client (>=5.3.1)", "ipywidgets (>=7)", "pebble", "mock"] +version = "5.6.1" [package.dependencies] bleach = "*" @@ -821,21 +825,24 @@ pygments = "*" testpath = "*" traitlets = ">=4.2" +[package.extras] +all = ["pytest", "pytest-cov", "ipykernel", "jupyter-client (>=5.3.1)", "ipywidgets (>=7)", "pebble", "tornado (>=4.0)", "sphinx (>=1.5.1)", "sphinx-rtd-theme", "nbsphinx (>=0.2.12)", "sphinxcontrib-github-alt", "ipython", "mock"] +docs = ["sphinx (>=1.5.1)", "sphinx-rtd-theme", "nbsphinx (>=0.2.12)", "sphinxcontrib-github-alt", "ipython", "jupyter-client (>=5.3.1)"] +execute = ["jupyter-client (>=5.3.1)"] +serve = ["tornado (>=4.0)"] +test = ["pytest", "pytest-cov", "ipykernel", "jupyter-client (>=5.3.1)", "ipywidgets (>=7)", "pebble", "mock"] + [[package]] -name = "nbdime" -version = "2.0.0" -description = "Diff and merge of Jupyter Notebooks" category = "dev" +description = "Diff and merge of Jupyter Notebooks" +name = "nbdime" optional = false python-versions = ">=3.5" - -[package.extras] -docs = ["sphinx", "recommonmark", "sphinx-rtd-theme"] -test = ["pytest (>=3.6)", "pytest-cov", "pytest-timeout", "pytest-tornado5 (>=2)", "jsonschema", "mock", "requests", "tabulate"] +version = "2.0.0" [package.dependencies] -colorama = "*" GitPython = "<2.1.4 || >2.1.4,<2.1.5 || >2.1.5,<2.1.6 || >2.1.6" +colorama = "*" jinja2 = ">=2.9" nbformat = "*" notebook = "*" @@ -844,16 +851,17 @@ requests = "*" six = "*" tornado = "*" +[package.extras] +docs = ["sphinx", "recommonmark", "sphinx-rtd-theme"] +test = ["pytest (>=3.6)", "pytest-cov", "pytest-timeout", "pytest-tornado5 (>=2)", "jsonschema", "mock", "requests", "tabulate"] + [[package]] -name = "nbformat" -version = "5.0.7" -description = "The Jupyter Notebook format" category = "dev" +description = "The Jupyter Notebook format" +name = "nbformat" optional = false python-versions = ">=3.5" - -[package.extras] -test = ["pytest", "pytest-cov", "testpath"] +version = "5.0.7" [package.dependencies] ipython-genutils = "*" @@ -861,19 +869,19 @@ jsonschema = ">=2.4,<2.5.0 || >2.5.0" jupyter-core = "*" traitlets = ">=4.1" +[package.extras] +test = ["pytest", "pytest-cov", "testpath"] + [[package]] -name = "notebook" -version = "6.1.4" -description = "A web-based notebook environment for interactive computing" category = "dev" +description = "A web-based notebook environment for interactive computing" +name = "notebook" optional = false python-versions = ">=3.5" - -[package.extras] -docs = ["sphinx", "nbsphinx", "sphinxcontrib-github-alt"] -test = ["nose", "coverage", "requests", "nose-warnings-filters", "nbval", "nose-exclude", "selenium", "pytest", "pytest-cov", "requests-unixsocket"] +version = "6.1.4" [package.dependencies] +Send2Trash = "*" argon2-cffi = "*" ipykernel = "*" ipython-genutils = "*" @@ -884,21 +892,21 @@ nbconvert = "*" nbformat = "*" prometheus-client = "*" pyzmq = ">=17" -Send2Trash = "*" terminado = ">=0.8.3" tornado = ">=5.0" traitlets = ">=4.2.1" +[package.extras] +docs = ["sphinx", "nbsphinx", "sphinxcontrib-github-alt"] +test = ["nose", "coverage", "requests", "nose-warnings-filters", "nbval", "nose-exclude", "selenium", "pytest", "pytest-cov", "requests-unixsocket"] + [[package]] -name = "nox" -version = "2020.8.22" -description = "Flexible test automation." category = "dev" +description = "Flexible test automation." +name = "nox" optional = false python-versions = ">=3.5" - -[package.extras] -tox_to_nox = ["jinja2", "tox"] +version = "2020.8.22" [package.dependencies] argcomplete = ">=1.9.4,<2.0" @@ -907,245 +915,244 @@ py = ">=1.4.0,<2.0.0" virtualenv = ">=14.0.0" [package.dependencies.importlib-metadata] -version = "*" python = "<3.8" +version = "*" + +[package.extras] +tox_to_nox = ["jinja2", "tox"] [[package]] -name = "numexpr" -version = "2.7.1" -description = "Fast numerical expression evaluator for NumPy" category = "main" +description = "Fast numerical expression evaluator for NumPy" +name = "numexpr" optional = false python-versions = "*" +version = "2.7.1" [package.dependencies] numpy = ">=1.7" [[package]] -name = "numpy" -version = "1.18.3" -description = "NumPy is the fundamental package for array computing with Python." category = "main" +description = "NumPy is the fundamental package for array computing with Python." +name = "numpy" optional = false python-versions = ">=3.5" +version = "1.18.3" [[package]] -name = "openpyxl" -version = "3.0.5" -description = "A Python library to read/write Excel 2010 xlsx/xlsm files" category = "main" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +name = "openpyxl" optional = true python-versions = ">=3.6," +version = "3.0.5" [package.dependencies] et-xmlfile = "*" jdcal = "*" [[package]] -name = "packaging" -version = "20.4" -description = "Core utilities for Python packages" category = "main" +description = "Core utilities for Python packages" +name = "packaging" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.4" [package.dependencies] pyparsing = ">=2.0.2" six = "*" [[package]] -name = "pandas" -version = "1.0.4" -description = "Powerful data structures for data analysis, time series, and statistics" category = "main" +description = "Powerful data structures for data analysis, time series, and statistics" +name = "pandas" optional = false python-versions = ">=3.6.1" - -[package.extras] -test = ["pytest (>=4.0.2)", "pytest-xdist", "hypothesis (>=3.58)"] +version = "1.0.4" [package.dependencies] numpy = ">=1.13.3" python-dateutil = ">=2.6.1" pytz = ">=2017.2" +[package.extras] +test = ["pytest (>=4.0.2)", "pytest-xdist", "hypothesis (>=3.58)"] + [[package]] -name = "pandocfilters" -version = "1.4.2" -description = "Utilities for writing pandoc filters in python" category = "dev" +description = "Utilities for writing pandoc filters in python" +name = "pandocfilters" optional = false python-versions = "*" +version = "1.4.2" [[package]] -name = "parso" -version = "0.7.1" -description = "A Python Parser" category = "main" +description = "A Python Parser" +name = "parso" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.7.1" [package.extras] testing = ["docopt", "pytest (>=3.0.7)"] [[package]] -name = "pathspec" -version = "0.8.0" -description = "Utility library for gitignore style pattern matching of file paths." category = "dev" +description = "Utility library for gitignore style pattern matching of file paths." +name = "pathspec" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.8.0" [[package]] -name = "pbr" -version = "5.5.0" -description = "Python Build Reasonableness" category = "main" +description = "Python Build Reasonableness" +name = "pbr" optional = false python-versions = ">=2.6" +version = "5.5.0" [[package]] -name = "pexpect" -version = "4.8.0" -description = "Pexpect allows easy control of interactive console applications." category = "main" +description = "Pexpect allows easy control of interactive console applications." +marker = "sys_platform != \"win32\"" +name = "pexpect" optional = false python-versions = "*" -marker = "sys_platform != \"win32\"" +version = "4.8.0" [package.dependencies] ptyprocess = ">=0.5" [[package]] -name = "pickleshare" -version = "0.7.5" -description = "Tiny 'shelve'-like database with concurrency support" category = "main" +description = "Tiny 'shelve'-like database with concurrency support" +name = "pickleshare" optional = false python-versions = "*" +version = "0.7.5" [[package]] -name = "pillow" -version = "7.2.0" -description = "Python Imaging Library (Fork)" category = "main" +description = "Python Imaging Library (Fork)" +name = "pillow" optional = true python-versions = ">=3.5" +version = "7.2.0" [[package]] -name = "pluggy" -version = "0.13.1" -description = "plugin and hook calling mechanisms for python" category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.extras] -dev = ["pre-commit", "tox"] +version = "0.13.1" [package.dependencies] [package.dependencies.importlib-metadata] -version = ">=0.12" python = "<3.8" +version = ">=0.12" + +[package.extras] +dev = ["pre-commit", "tox"] [[package]] -name = "prometheus-client" -version = "0.8.0" -description = "Python client for the Prometheus monitoring system." category = "dev" +description = "Python client for the Prometheus monitoring system." +name = "prometheus-client" optional = false python-versions = "*" +version = "0.8.0" [package.extras] twisted = ["twisted"] [[package]] -name = "prompt-toolkit" -version = "3.0.7" -description = "Library for building powerful interactive command lines in Python" category = "main" +description = "Library for building powerful interactive command lines in Python" +name = "prompt-toolkit" optional = false python-versions = ">=3.6.1" +version = "3.0.7" [package.dependencies] wcwidth = "*" [[package]] -name = "ptyprocess" -version = "0.6.0" -description = "Run a subprocess in a pseudo terminal" category = "main" +description = "Run a subprocess in a pseudo terminal" +marker = "sys_platform != \"win32\" or os_name != \"nt\"" +name = "ptyprocess" optional = false python-versions = "*" -marker = "sys_platform != \"win32\" or os_name != \"nt\"" +version = "0.6.0" [[package]] -name = "py" -version = "1.9.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "dev" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.9.0" [[package]] -name = "pycodestyle" -version = "2.6.0" -description = "Python style guide checker" category = "dev" +description = "Python style guide checker" +name = "pycodestyle" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.6.0" [[package]] -name = "pycparser" -version = "2.20" -description = "C parser in Python" category = "dev" +description = "C parser in Python" +name = "pycparser" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.20" [[package]] -name = "pyflakes" -version = "2.2.0" -description = "passive checker of Python programs" category = "dev" +description = "passive checker of Python programs" +name = "pyflakes" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.2.0" [[package]] -name = "pygments" -version = "2.7.1" -description = "Pygments is a syntax highlighting package written in Python." category = "main" +description = "Pygments is a syntax highlighting package written in Python." +name = "pygments" optional = false python-versions = ">=3.5" +version = "2.7.1" [[package]] -name = "pyparsing" -version = "2.4.7" -description = "Python parsing module" category = "main" +description = "Python parsing module" +name = "pyparsing" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.7" [[package]] -name = "pyrsistent" -version = "0.17.3" -description = "Persistent/Functional/Immutable data structures" category = "dev" +description = "Persistent/Functional/Immutable data structures" +name = "pyrsistent" optional = false python-versions = ">=3.5" +version = "0.17.3" [[package]] -name = "pytest" -version = "6.0.2" -description = "pytest: simple powerful testing with Python" category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" optional = false python-versions = ">=3.5" - -[package.extras] -checkqa_mypy = ["mypy (0.780)"] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +version = "6.0.2" [package.dependencies] atomicwrites = ">=1.0" @@ -1159,36 +1166,35 @@ py = ">=1.8.2" toml = "*" [package.dependencies.importlib-metadata] -version = ">=0.12" python = "<3.8" +version = ">=0.12" + +[package.extras] +checkqa_mypy = ["mypy (0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] -name = "pytest-cov" -version = "2.10.1" -description = "Pytest plugin for measuring coverage." category = "dev" +description = "Pytest plugin for measuring coverage." +name = "pytest-cov" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] +version = "2.10.1" [package.dependencies] coverage = ">=4.4" pytest = ">=4.6" +[package.extras] +testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] + [[package]] -name = "pytest-notebook" -version = "0.6.1" -description = "A pytest plugin for testing Jupyter Notebooks" category = "dev" +description = "A pytest plugin for testing Jupyter Notebooks" +name = "pytest-notebook" optional = false python-versions = ">=3.6" - -[package.extras] -code_style = ["pre-commit (>=2.7.0,<2.8.0)", "doc8 (>=0.8.0,<0.9.0)"] -docs = ["sphinx (>=2)", "pyyaml", "sphinx-book-theme (>=0.0.36,<0.1.0)", "myst-nb (>=0.10.1,<0.11.0)", "coverage (>=4.5.1,<4.6.0)"] -testing = ["coverage (>=4.5.1,<4.6.0)", "pytest-cov", "pytest-regressions", "black (19.3b0)", "beautifulsoup4 (4.8.0)"] +version = "0.6.1" [package.dependencies] attrs = ">=19,<21" @@ -1200,96 +1206,97 @@ nbdime = "*" nbformat = "*" pytest = ">=3.5.0" +[package.extras] +code_style = ["pre-commit (>=2.7.0,<2.8.0)", "doc8 (>=0.8.0,<0.9.0)"] +docs = ["sphinx (>=2)", "pyyaml", "sphinx-book-theme (>=0.0.36,<0.1.0)", "myst-nb (>=0.10.1,<0.11.0)", "coverage (>=4.5.1,<4.6.0)"] +testing = ["coverage (>=4.5.1,<4.6.0)", "pytest-cov", "pytest-regressions", "black (19.3b0)", "beautifulsoup4 (4.8.0)"] + [[package]] -name = "python-dateutil" -version = "2.8.1" -description = "Extensions to the standard Python datetime module" category = "main" +description = "Extensions to the standard Python datetime module" +name = "python-dateutil" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +version = "2.8.1" [package.dependencies] six = ">=1.5" [[package]] -name = "python-slugify" -version = "4.0.1" -description = "A Python Slugify application that handles Unicode" category = "main" +description = "A Python Slugify application that handles Unicode" +name = "python-slugify" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -unidecode = ["Unidecode (>=1.1.1)"] +version = "4.0.1" [package.dependencies] text-unidecode = ">=1.3" [package.dependencies.Unidecode] -version = ">=1.1.1" optional = true +version = ">=1.1.1" + +[package.extras] +unidecode = ["Unidecode (>=1.1.1)"] [[package]] -name = "pytz" -version = "2020.1" -description = "World timezone definitions, modern and historical" category = "main" +description = "World timezone definitions, modern and historical" +name = "pytz" optional = false python-versions = "*" +version = "2020.1" [[package]] -name = "pywin32" -version = "228" -description = "Python for Window Extensions" category = "dev" +description = "Python for Window Extensions" +marker = "sys_platform == \"win32\"" +name = "pywin32" optional = false python-versions = "*" -marker = "sys_platform == \"win32\"" +version = "228" [[package]] -name = "pywinpty" -version = "0.5.7" -description = "Python bindings for the winpty library" category = "dev" +description = "Python bindings for the winpty library" +marker = "os_name == \"nt\"" +name = "pywinpty" optional = false python-versions = "*" -marker = "os_name == \"nt\"" +version = "0.5.7" [[package]] -name = "pyyaml" -version = "5.3.1" -description = "YAML parser and emitter for Python" category = "dev" +description = "YAML parser and emitter for Python" +name = "pyyaml" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "5.3.1" [[package]] -name = "pyzmq" -version = "19.0.2" -description = "Python bindings for 0MQ" category = "dev" +description = "Python bindings for 0MQ" +name = "pyzmq" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" +version = "19.0.2" [[package]] -name = "regex" -version = "2020.7.14" -description = "Alternative regular expression module, to replace re." category = "main" +description = "Alternative regular expression module, to replace re." +name = "regex" optional = false python-versions = "*" +version = "2020.7.14" [[package]] -name = "requests" -version = "2.24.0" -description = "Python HTTP for Humans." category = "main" +description = "Python HTTP for Humans." +name = "requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] +version = "2.24.0" [package.dependencies] certifi = ">=2017.4.17" @@ -1297,79 +1304,78 @@ chardet = ">=3.0.2,<4" idna = ">=2.5,<3" urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + [[package]] -name = "scipy" -version = "1.4.1" -description = "SciPy: Scientific Library for Python" category = "main" +description = "SciPy: Scientific Library for Python" +name = "scipy" optional = false python-versions = ">=3.5" +version = "1.4.1" [package.dependencies] numpy = ">=1.13.3" [[package]] -name = "send2trash" -version = "1.5.0" -description = "Send file to trash natively under Mac OS X, Windows and Linux." category = "dev" +description = "Send file to trash natively under Mac OS X, Windows and Linux." +name = "send2trash" optional = false python-versions = "*" +version = "1.5.0" [[package]] -name = "six" -version = "1.15.0" -description = "Python 2 and 3 compatibility utilities" category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "smmap" -version = "3.0.4" -description = "A pure Python implementation of a sliding window memory map manager" +version = "1.15.0" + +[[package]] category = "dev" +description = "A pure Python implementation of a sliding window memory map manager" +name = "smmap" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.0.4" [[package]] -name = "snowballstemmer" -version = "2.0.0" -description = "This package provides 26 stemmers for 25 languages generated from Snowball algorithms." category = "main" +description = "This package provides 26 stemmers for 25 languages generated from Snowball algorithms." +name = "snowballstemmer" optional = true python-versions = "*" +version = "2.0.0" [[package]] -name = "soupsieve" -version = "1.9.6" -description = "A modern CSS selector implementation for Beautiful Soup." category = "main" +description = "A modern CSS selector implementation for Beautiful Soup." +name = "soupsieve" optional = false python-versions = "*" +version = "1.9.6" [[package]] -name = "sphinx" -version = "3.2.1" -description = "Python documentation generator" category = "main" +description = "Python documentation generator" +name = "sphinx" optional = true python-versions = ">=3.5" - -[package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "flake8-import-order", "mypy (>=0.780)", "docutils-stubs"] -test = ["pytest", "pytest-cov", "html5lib", "typed-ast", "cython"] +version = "3.2.1" [package.dependencies] +Jinja2 = ">=2.3" +Pygments = ">=2.0" alabaster = ">=0.7,<0.8" babel = ">=1.3" colorama = ">=0.3.5" docutils = ">=0.12" imagesize = "*" -Jinja2 = ">=2.3" packaging = "*" -Pygments = ">=2.0" requests = ">=2.5.0" setuptools = "*" snowballstemmer = ">=1.1" @@ -1380,31 +1386,33 @@ sphinxcontrib-jsmath = "*" sphinxcontrib-qthelp = "*" sphinxcontrib-serializinghtml = "*" +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["flake8 (>=3.5.0)", "flake8-import-order", "mypy (>=0.780)", "docutils-stubs"] +test = ["pytest", "pytest-cov", "html5lib", "typed-ast", "cython"] + [[package]] -name = "sphinx-autodoc-typehints" -version = "1.11.0" -description = "Type hints (PEP 484) support for the Sphinx autodoc extension" category = "main" +description = "Type hints (PEP 484) support for the Sphinx autodoc extension" +name = "sphinx-autodoc-typehints" optional = true python-versions = ">=3.5.2" +version = "1.11.0" + +[package.dependencies] +Sphinx = ">=3.0" [package.extras] test = ["pytest (>=3.1.0)", "typing-extensions (>=3.5)", "sphobjinv (>=2.0)", "dataclasses"] type_comments = ["typed-ast (>=1.4.0)"] -[package.dependencies] -Sphinx = ">=3.0" - [[package]] -name = "sphinx-material" -version = "0.0.30" -description = "Material sphinx theme" category = "main" +description = "Material sphinx theme" +name = "sphinx-material" optional = true python-versions = ">=3.6" - -[package.extras] -dev = ["black (19.10b0)"] +version = "0.0.30" [package.dependencies] beautifulsoup4 = "*" @@ -1413,136 +1421,147 @@ lxml = "*" sphinx = ">=2.0" [package.dependencies.python-slugify] -version = "*" extras = ["unidecode"] +version = "*" + +[package.extras] +dev = ["black (19.10b0)"] [[package]] -name = "sphinxcontrib-applehelp" -version = "1.0.2" -description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" category = "main" +description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" +name = "sphinxcontrib-applehelp" optional = true python-versions = ">=3.5" +version = "1.0.2" [package.extras] lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] [[package]] -name = "sphinxcontrib-devhelp" -version = "1.0.2" -description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." category = "main" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +name = "sphinxcontrib-devhelp" optional = true python-versions = ">=3.5" +version = "1.0.2" [package.extras] lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] [[package]] -name = "sphinxcontrib-htmlhelp" -version = "1.0.3" -description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" category = "main" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +name = "sphinxcontrib-htmlhelp" optional = true python-versions = ">=3.5" +version = "1.0.3" [package.extras] lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest", "html5lib"] [[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -description = "A sphinx extension which renders display math in HTML via JavaScript" category = "main" +description = "A sphinx extension which renders display math in HTML via JavaScript" +name = "sphinxcontrib-jsmath" optional = true python-versions = ">=3.5" +version = "1.0.1" [package.extras] test = ["pytest", "flake8", "mypy"] [[package]] -name = "sphinxcontrib-qthelp" -version = "1.0.3" -description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." category = "main" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +name = "sphinxcontrib-qthelp" optional = true python-versions = ">=3.5" +version = "1.0.3" [package.extras] lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] [[package]] -name = "sphinxcontrib-serializinghtml" -version = "1.1.4" -description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." category = "main" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +name = "sphinxcontrib-serializinghtml" optional = true python-versions = ">=3.5" +version = "1.1.4" [package.extras] lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] [[package]] -name = "sphinxcontrib-svg2pdfconverter" -version = "1.1.0" -description = "Sphinx SVG to PDF converter extension" category = "main" +description = "Sphinx SVG to PDF converter extension" +name = "sphinxcontrib-svg2pdfconverter" optional = true python-versions = "~=3.4" - -[package.extras] -CairoSVG = ["cairosvg (>=1.0)"] +version = "1.1.0" [package.dependencies] Sphinx = ">=1.6.3" +[package.extras] +CairoSVG = ["cairosvg (>=1.0)"] + [[package]] -name = "stevedore" -version = "3.2.2" -description = "Manage dynamic plugins for Python applications" category = "main" +description = "Manage dynamic plugins for Python applications" +name = "stevedore" optional = false python-versions = ">=3.6" +version = "3.2.2" [package.dependencies] pbr = ">=2.0.0,<2.1.0 || >2.1.0" [package.dependencies.importlib-metadata] -version = ">=1.7.0" python = "<3.8" +version = ">=1.7.0" + +[[package]] +category = "dev" +description = "A Python micro-lib to create stubs for non-existing modules." +name = "surrogate" +optional = false +python-versions = "*" +version = "0.1" [[package]] -name = "tables" -version = "3.6.1" -description = "Hierarchical datasets for Python" category = "main" +description = "Hierarchical datasets for Python" +name = "tables" optional = false python-versions = ">=3.5" +version = "3.6.1" [package.dependencies] numexpr = ">=2.6.2" numpy = ">=1.9.3" [[package]] -name = "termcolor" -version = "1.1.0" -description = "ANSII Color formatting for output in terminal." category = "main" +description = "ANSII Color formatting for output in terminal." +name = "termcolor" optional = false python-versions = "*" +version = "1.1.0" [[package]] -name = "terminado" -version = "0.9.1" -description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." category = "dev" +description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." +name = "terminado" optional = false python-versions = ">=3.6" +version = "0.9.1" [package.dependencies] ptyprocess = "*" @@ -1550,106 +1569,106 @@ pywinpty = ">=0.5" tornado = ">=4" [[package]] -name = "testpath" -version = "0.4.4" -description = "Test utilities for code working with files and commands" category = "dev" +description = "Test utilities for code working with files and commands" +name = "testpath" optional = false python-versions = "*" +version = "0.4.4" [package.extras] test = ["pathlib2"] [[package]] -name = "text-unidecode" -version = "1.3" -description = "The most basic Text::Unidecode port" category = "main" +description = "The most basic Text::Unidecode port" +name = "text-unidecode" optional = true python-versions = "*" +version = "1.3" [[package]] -name = "toml" -version = "0.10.1" -description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" +description = "Python Library for Tom's Obvious, Minimal Language" +name = "toml" optional = false python-versions = "*" +version = "0.10.1" [[package]] -name = "tomlkit" -version = "0.7.0" -description = "Style preserving TOML library" category = "main" +description = "Style preserving TOML library" +name = "tomlkit" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.7.0" [[package]] -name = "tornado" -version = "6.0.4" -description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." category = "dev" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +name = "tornado" optional = false python-versions = ">= 3.5" +version = "6.0.4" [[package]] -name = "traitlets" -version = "4.3.3" -description = "Traitlets Python config system" category = "main" +description = "Traitlets Python config system" +name = "traitlets" optional = false python-versions = "*" - -[package.extras] -test = ["pytest", "mock"] +version = "4.3.3" [package.dependencies] decorator = "*" ipython-genutils = "*" six = "*" +[package.extras] +test = ["pytest", "mock"] + [[package]] -name = "typed-ast" -version = "1.4.1" -description = "a fork of Python 2 and 3 ast modules with type comment support" category = "dev" +description = "a fork of Python 2 and 3 ast modules with type comment support" +name = "typed-ast" optional = false python-versions = "*" +version = "1.4.1" [[package]] -name = "typing-extensions" -version = "3.7.4.3" -description = "Backported and Experimental Type Hints for Python 3.5+" category = "dev" +description = "Backported and Experimental Type Hints for Python 3.5+" +name = "typing-extensions" optional = false python-versions = "*" +version = "3.7.4.3" [[package]] -name = "tzlocal" -version = "2.1" -description = "tzinfo object for the local timezone" category = "main" +description = "tzinfo object for the local timezone" +name = "tzlocal" optional = false python-versions = "*" +version = "2.1" [package.dependencies] pytz = "*" [[package]] -name = "unidecode" -version = "1.1.1" -description = "ASCII transliterations of Unicode text" category = "main" +description = "ASCII transliterations of Unicode text" +name = "unidecode" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.1.1" [[package]] -name = "urllib3" -version = "1.25.10" -description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +name = "urllib3" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "1.25.10" [package.extras] brotli = ["brotlipy (>=0.6.0)"] @@ -1657,16 +1676,12 @@ secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0 socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] -name = "virtualenv" -version = "20.0.31" -description = "Virtual Python Environment builder" category = "dev" +description = "Virtual Python Environment builder" +name = "virtualenv" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" - -[package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] -testing = ["coverage (>=5)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "pytest-xdist (>=1.31.0)", "packaging (>=20.0)", "xonsh (>=0.9.16)"] +version = "20.0.31" [package.dependencies] appdirs = ">=1.4.3,<2" @@ -1675,51 +1690,56 @@ filelock = ">=3.0.0,<4" six = ">=1.9.0,<2" [package.dependencies.importlib-metadata] -version = ">=0.12,<2" python = "<3.8" +version = ">=0.12,<2" [package.dependencies.importlib-resources] -version = ">=1.0" python = "<3.7" +version = ">=1.0" + +[package.extras] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] +testing = ["coverage (>=5)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "pytest-xdist (>=1.31.0)", "packaging (>=20.0)", "xonsh (>=0.9.16)"] [[package]] -name = "wcwidth" -version = "0.2.5" -description = "Measures the displayed width of unicode strings in a terminal" category = "main" +description = "Measures the displayed width of unicode strings in a terminal" +name = "wcwidth" optional = false python-versions = "*" +version = "0.2.5" [[package]] -name = "webencodings" -version = "0.5.1" -description = "Character encoding aliases for legacy web content" category = "dev" +description = "Character encoding aliases for legacy web content" +name = "webencodings" optional = false python-versions = "*" +version = "0.5.1" [[package]] -name = "zipp" -version = "3.1.0" -description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" +description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" +name = "zipp" optional = false python-versions = ">=3.6" -marker = "python_version < \"3.8\"" +version = "3.1.0" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [extras] -ipython = ["ipython", "ipython-genutils", "matplotlib"] +docs = ["sphinx", "sphinx-material", "tomlkit", "sphinx-autodoc-typehints", "sphinxcontrib-svg2pdfconverter", "matplotlib", "ipython"] excel = ["openpyxl"] -docs = ["sphinx", "sphinx-material", "importlib_metadata", "tomlkit", "sphinx-autodoc-typehints", "sphinxcontrib-svg2pdfconverter", "matplotlib", "ipython"] +ipython = ["ipython", "ipython-genutils", "matplotlib"] +sql = ["duckdb"] [metadata] +content-hash = "63259b3343372bd3c711d82f8fc2e447dadea9f5acf5db87fce9423e1e3cc9e0" lock-version = "1.0" python-versions = "^3.6.1" -content-hash = "1c9563884927ee4d592e83aaf307968b96146551793de8c0e984a6eeb078d2d6" [metadata.files] aiofiles = [ @@ -1817,19 +1837,16 @@ cffi = [ {file = "cffi-1.14.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c687778dda01832555e0af205375d649fa47afeaeeb50a201711f9a9573323b8"}, {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, {file = "cffi-1.14.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:03d3d238cc6c636a01cf55b9b2e1b6531a7f2f4103fabb5a744231582e68ecc7"}, {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, {file = "cffi-1.14.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c2a33558fdbee3df370399fe1712d72464ce39c66436270f3664c03f94971aff"}, {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, {file = "cffi-1.14.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d9a7dc7cf8b1101af2602fe238911bcc1ac36d239e0a577831f5dac993856e9"}, @@ -1930,6 +1947,23 @@ docutils = [ "dogpile.cache" = [ {file = "dogpile.cache-1.0.2.tar.gz", hash = "sha256:64fda39d25b46486a4876417ca03a4af06f35bfadba9f59613f9b3d748aa21ef"}, ] +duckdb = [ + {file = "duckdb-0.2.2.dev175-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c34a2ca5e10d83d5d0f5b3535f5860e008a6b805b28f25ed6c71bd9023b952cf"}, + {file = "duckdb-0.2.2.dev175-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8a962c24f814f5f94c39f2fadc3c11420690e5dd83b24aa504e5a7b45c4be607"}, + {file = "duckdb-0.2.2.dev175-cp36-cp36m-win32.whl", hash = "sha256:28d5d70c30bb9eb5c312d8f420b3d23429d6b903d2eba127b4c6b91a017dd1ba"}, + {file = "duckdb-0.2.2.dev175-cp36-cp36m-win_amd64.whl", hash = "sha256:151bd75a50648bef010a41a06191581e4923a39005c21079cf77b0b62d5e5a67"}, + {file = "duckdb-0.2.2.dev175-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:22bf1c41145f42819622319257db40e4435a9c9c7e5261b098cfa9051c2a9c81"}, + {file = "duckdb-0.2.2.dev175-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:1e9cd2f10b8241ffc767b88e6ae8d95d468764a7e2ef3b2d9dbae3afb90e535b"}, + {file = "duckdb-0.2.2.dev175-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:86685c2c1a95898008fd6cdb4e49cbcc639f4e279354f6e2453840537cb17a4a"}, + {file = "duckdb-0.2.2.dev175-cp37-cp37m-win32.whl", hash = "sha256:947e084a96dec104febe062d66f53b2422ee7f42d05f139a7feb412722189c29"}, + {file = "duckdb-0.2.2.dev175-cp37-cp37m-win_amd64.whl", hash = "sha256:7fb2304beff76110e975d56e9883571714b7e56d46e509db0d378a4766d0be43"}, + {file = "duckdb-0.2.2.dev175-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c6fd95718896b2123f1b75433454302822db5ee479bd13d25f36a06a347a978"}, + {file = "duckdb-0.2.2.dev175-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:da141b5fd17ffddb861658cbc0bad573cc7896a297bc3743bfea20b3623056fc"}, + {file = "duckdb-0.2.2.dev175-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7f1ec4a8e7e3c889d171053b8ebbf2e1c1ed3179b4bd664c2dc3503309930643"}, + {file = "duckdb-0.2.2.dev175-cp38-cp38-win32.whl", hash = "sha256:92e94782d0d9622ce901cdc9a4e35007fa63024219d00d6f1f56e9bf5222e9ce"}, + {file = "duckdb-0.2.2.dev175-cp38-cp38-win_amd64.whl", hash = "sha256:1cc76dcc1087d03249f0860660756323724518b5f55b039ec5884447aacfcf37"}, + {file = "duckdb-0.2.2.dev175.tar.gz", hash = "sha256:88fa147cadd76eef39c1d3b03ca935480d47080d8d5e46f3a9f546023266bfeb"}, +] entrypoints = [ {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, @@ -2061,16 +2095,19 @@ kiwisolver = [ {file = "kiwisolver-1.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:443c2320520eda0a5b930b2725b26f6175ca4453c61f739fef7a5847bd262f74"}, {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:efcf3397ae1e3c3a4a0a0636542bcad5adad3b1dd3e8e629d0b6e201347176c8"}, {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fccefc0d36a38c57b7bd233a9b485e2f1eb71903ca7ad7adacad6c28a56d62d2"}, + {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:be046da49fbc3aa9491cc7296db7e8d27bcf0c3d5d1a40259c10471b014e4e0c"}, {file = "kiwisolver-1.2.0-cp36-none-win32.whl", hash = "sha256:60a78858580761fe611d22127868f3dc9f98871e6fdf0a15cc4203ed9ba6179b"}, {file = "kiwisolver-1.2.0-cp36-none-win_amd64.whl", hash = "sha256:556da0a5f60f6486ec4969abbc1dd83cf9b5c2deadc8288508e55c0f5f87d29c"}, {file = "kiwisolver-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7cc095a4661bdd8a5742aaf7c10ea9fac142d76ff1770a0f84394038126d8fc7"}, {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c955791d80e464da3b471ab41eb65cf5a40c15ce9b001fdc5bbc241170de58ec"}, {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:603162139684ee56bcd57acc74035fceed7dd8d732f38c0959c8bd157f913fec"}, + {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:63f55f490b958b6299e4e5bdac66ac988c3d11b7fafa522800359075d4fa56d1"}, {file = "kiwisolver-1.2.0-cp37-none-win32.whl", hash = "sha256:03662cbd3e6729f341a97dd2690b271e51a67a68322affab12a5b011344b973c"}, {file = "kiwisolver-1.2.0-cp37-none-win_amd64.whl", hash = "sha256:4eadb361baf3069f278b055e3bb53fa189cea2fd02cb2c353b7a99ebb4477ef1"}, {file = "kiwisolver-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c31bc3c8e903d60a1ea31a754c72559398d91b5929fcb329b1c3a3d3f6e72113"}, {file = "kiwisolver-1.2.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d52b989dc23cdaa92582ceb4af8d5bcc94d74b2c3e64cd6785558ec6a879793e"}, {file = "kiwisolver-1.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e586b28354d7b6584d8973656a7954b1c69c93f708c0c07b77884f91640b7657"}, + {file = "kiwisolver-1.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:38d05c9ecb24eee1246391820ed7137ac42a50209c203c908154782fced90e44"}, {file = "kiwisolver-1.2.0-cp38-none-win32.whl", hash = "sha256:d069ef4b20b1e6b19f790d00097a5d5d2c50871b66d10075dab78938dc2ee2cf"}, {file = "kiwisolver-1.2.0-cp38-none-win_amd64.whl", hash = "sha256:18d749f3e56c0480dccd1714230da0f328e6e4accf188dd4e6884bdd06bf02dd"}, {file = "kiwisolver-1.2.0.tar.gz", hash = "sha256:247800260cd38160c362d211dcaf4ed0f7816afb5efe56544748b21d6ad6d17f"}, @@ -2333,6 +2370,8 @@ pillow = [ {file = "Pillow-7.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8"}, {file = "Pillow-7.2.0-cp38-cp38-win32.whl", hash = "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f"}, {file = "Pillow-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6"}, + {file = "Pillow-7.2.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:9c87ef410a58dd54b92424ffd7e28fd2ec65d2f7fc02b76f5e9b2067e355ebf6"}, + {file = "Pillow-7.2.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e901964262a56d9ea3c2693df68bc9860b8bdda2b04768821e4c44ae797de117"}, {file = "Pillow-7.2.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d"}, {file = "Pillow-7.2.0.tar.gz", hash = "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626"}, ] @@ -2585,6 +2624,9 @@ stevedore = [ {file = "stevedore-3.2.2-py3-none-any.whl", hash = "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62"}, {file = "stevedore-3.2.2.tar.gz", hash = "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0"}, ] +surrogate = [ + {file = "surrogate-0.1.tar.gz", hash = "sha256:edebec660d728325be1d52cab40d778d4c75ba04f927f4aba12d35f730b2df03"}, +] tables = [ {file = "tables-3.6.1-2-cp36-cp36m-win32.whl", hash = "sha256:db163df08ded7804d596dee14d88397f6c55cdf4671b3992cb885c0b3890a54d"}, {file = "tables-3.6.1-2-cp36-cp36m-win_amd64.whl", hash = "sha256:fd63c94960f8208cb13d41033a3114c0242e7737cb578f2454c6a087c5d246ec"}, diff --git a/pyproject.toml b/pyproject.toml index eac2b27fa..c21e497f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,7 @@ sphinx-autodoc-typehints = { version = "1.11.0", optional = true } sphinxcontrib-svg2pdfconverter = { version = "1.1.0", optional = true } tomlkit = { version = "0.7.0", optional = true } +duckdb = { version = "^0.2.2.dev175", optional = true } [tool.poetry.extras] ipython = ["ipython", "ipython-genutils", "matplotlib"] @@ -108,6 +109,7 @@ docs = [ "matplotlib", "ipython" ] +sql = ["duckdb"] [tool.poetry.dev-dependencies] nox = "^2020.8.22" @@ -122,7 +124,7 @@ pytest-cov = "^2.10.1" pytest-notebook = "^0.6.0" nbconvert = ">=5.0, <6.0" mock = "^4.0" -pygments = "^2.7.0" +surrogate = "^0.1" [tool.poetry.scripts] wetterdienst = 'wetterdienst.cli:run' diff --git a/tests/test_cli.py b/tests/test_cli.py index 7d4e013a1..ae82055e0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -56,6 +56,18 @@ def test_cli_about_periods(capsys): assert "now" in response +def test_cli_about_coverage(capsys): + + sys.argv = ["wetterdienst", "about", "coverage"] + cli.run() + stdout, stderr = capsys.readouterr() + + response = stdout + assert "TimeResolution.ANNUAL" in response + assert "Parameter.CLIMATE_SUMMARY" in response + assert "PeriodType.HISTORICAL" in response + + def invoke_wetterdienst_stations(format="json"): argv = shlex.split( f"wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --num=5 --format={format}" # noqa:E501,B950 diff --git a/tests/test_io.py b/tests/test_io.py new file mode 100644 index 000000000..5d1a65de6 --- /dev/null +++ b/tests/test_io.py @@ -0,0 +1,177 @@ +import json +import mock +import pandas as pd +import pytest +from surrogate import surrogate + +from wetterdienst import DWDStationRequest, Parameter, TimeResolution, PeriodType +from wetterdienst.additionals.time_handling import parse_datetime +from wetterdienst.io import DataPackage + + +original = pd.DataFrame.from_dict( + [ + { + "STATION_ID": 1048, + "PARAMETER": "climate_summary", + "ELEMENT": "temperature_air_max_200", + "DATE": parse_datetime("2019-12-28T00:00:00.000"), + "VALUE": 1.3, + "QUALITY": None, + } + ] +) + + +def test_lowercase_fieldnames(): + + dp = DataPackage(df=original) + dp.lowercase_fieldnames() + + assert list(dp.df.columns) == [ + "station_id", + "parameter", + "element", + "date", + "value", + "quality", + ] + + +def test_filter_by_date(): + + dp = DataPackage(df=original) + df = dp.filter_by_date("2019-12-28", TimeResolution.HOURLY) + assert not df.empty + + dp = DataPackage(df=original) + df = dp.filter_by_date("2019-12-27", TimeResolution.HOURLY) + assert df.empty + + +def test_filter_by_date_interval(): + + dp = DataPackage(df=original) + df = dp.filter_by_date("2019-12-27/2019-12-29", TimeResolution.HOURLY) + assert not df.empty + + dp = DataPackage(df=original) + df = dp.filter_by_date("2020/2022", TimeResolution.HOURLY) + assert df.empty + + +def test_filter_by_sql(): + + dp = DataPackage(df=original) + dp.lowercase_fieldnames() + df = dp.filter_by_sql( + "SELECT * FROM data WHERE element='temperature_air_max_200' AND value < 1.5" + ) + assert not df.empty + + dp = DataPackage(df=original) + dp.lowercase_fieldnames() + df = dp.filter_by_sql( + "SELECT * FROM data WHERE element='temperature_air_max_200' AND value > 1.5" + ) + assert df.empty + + +def test_format_json(): + + dp = DataPackage(df=original) + dp.lowercase_fieldnames() + output = dp.format("json") + + response = json.loads(output) + station_ids = list(set([reading["station_id"] for reading in response])) + + assert 1048 in station_ids + + +def test_format_csv(): + + dp = DataPackage(df=original) + dp.lowercase_fieldnames() + output = dp.format("csv").strip() + + assert ( + output + == """ +station_id,parameter,element,date,value,quality +1048,climate_summary,temperature_air_max_200,2019-12-28T00-00-00,1.3, +""".strip() + ) + + +def test_format_unknown(): + + dp = DataPackage(df=original) + + with pytest.raises(KeyError): + dp.format("foobar") + + +def test_request(): + + request = DWDStationRequest( + station_ids=[1048], + parameter=Parameter.CLIMATE_SUMMARY, + time_resolution=TimeResolution.DAILY, + period_type=PeriodType.RECENT, + ) + + dp = DataPackage(request=request) + assert not dp.df.empty + + +@surrogate("duckdb.connect") +def test_export_duckdb(): + + request = DWDStationRequest( + station_ids=[1048], + parameter=Parameter.CLIMATE_SUMMARY, + time_resolution=TimeResolution.DAILY, + period_type=PeriodType.RECENT, + ) + + mock_connection = mock.MagicMock() + with mock.patch( + "duckdb.connect", side_effect=[mock_connection], create=True + ) as mock_connect: + + dp = DataPackage(request=request) + dp.export("duckdb:///test.duckdb?table=testdrive") + + mock_connect.assert_called_once_with(database="test.duckdb", read_only=False) + mock_connection.register.assert_called_once() + mock_connection.execute.assert_called() + mock_connection.table.assert_called_once_with("testdrive") + # a.table.to_df.assert_called() + mock_connection.close.assert_called_once() + + +@surrogate("influxdb.dataframe_client.DataFrameClient") +def test_export_influxdb(): + + request = DWDStationRequest( + station_ids=[1048], + parameter=Parameter.CLIMATE_SUMMARY, + time_resolution=TimeResolution.DAILY, + period_type=PeriodType.RECENT, + ) + + mock_client = mock.MagicMock() + with mock.patch( + "influxdb.dataframe_client.DataFrameClient", + side_effect=[mock_client], + create=True, + ) as mock_connect: + + dp = DataPackage(request=request) + dp.lowercase_fieldnames() + dp.export("influxdb://localhost/?database=dwd&table=weather") + + mock_connect.assert_called_once_with(database="dwd") + mock_client.create_database.assert_called_once_with("dwd") + mock_client.write_points.assert_called_once() diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 000000000..ee60a64d0 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,18 @@ +import sys +import shlex +import mock +import runpy + + +@mock.patch( + "wetterdienst.data_collection.collect_climate_observations_data", side_effect=[None] +) +def test_run(mock_ccod): + args = ( + "run.py collect_climate_observations_data " + '"[1048]" "kl" "daily" "recent" /app/dwd_data/ ' + "False False True False True False" + ) + sys.argv = shlex.split(args) + runpy.run_module("wetterdienst.run", run_name="__main__") + mock_ccod.assert_called_once() diff --git a/wetterdienst/__init__.py b/wetterdienst/__init__.py index 95adc5040..05fe40b60 100644 --- a/wetterdienst/__init__.py +++ b/wetterdienst/__init__.py @@ -18,6 +18,7 @@ collect_radolan_data, ) from wetterdienst.api import DWDStationRequest, DWDRadolanRequest +from wetterdienst.io import DataPackage # Single-sourcing the package version # https://cjolowicz.github.io/posts/hypermodern-python-06-ci-cd/ diff --git a/wetterdienst/cli.py b/wetterdienst/cli.py index ef20b6fa9..8fd943bdf 100644 --- a/wetterdienst/cli.py +++ b/wetterdienst/cli.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import sys -import json import logging from datetime import datetime, timedelta @@ -14,14 +13,12 @@ get_nearby_stations, discover_climate_observations, ) -from wetterdienst.additionals.geo_location import stations_to_geojson -from wetterdienst.additionals.time_handling import mktimerange, parse_datetime from wetterdienst.additionals.util import normalize_options, setup_logging, read_list from wetterdienst.api import DWDStationRequest -from wetterdienst.enumerations.column_names_enumeration import DWDMetaColumns from wetterdienst.enumerations.parameter_enumeration import Parameter from wetterdienst.enumerations.period_type_enumeration import PeriodType from wetterdienst.enumerations.time_resolution_enumeration import TimeResolution +from wetterdienst.io import DataPackage log = logging.getLogger(__name__) @@ -29,9 +26,9 @@ def run(): """ Usage: - wetterdienst stations --parameter= --resolution= --period= [--station=] [--latitude=] [--longitude=] [--number=] [--distance=] [--persist] [--format=] - wetterdienst readings --parameter= --resolution= --period= --station= [--persist] [--date=] [--format=] - wetterdienst readings --parameter= --resolution= --period= --latitude= --longitude= [--number=] [--distance=] [--persist] [--date=] [--format=] + wetterdienst stations --parameter= --resolution= --period= [--station=] [--latitude=] [--longitude=] [--number=] [--distance=] [--persist] [--sql=] [--format=] + wetterdienst readings --parameter= --resolution= --period= --station= [--persist] [--date=] [--sql=] [--format=] [--target=] + wetterdienst readings --parameter= --resolution= --period= --latitude= --longitude= [--number=] [--distance=] [--persist] [--date=] [--sql=] [--format=] [--target=] wetterdienst about [parameters] [resolutions] [periods] wetterdienst about coverage [--parameter=] [--resolution=] [--period=] wetterdienst --version @@ -49,7 +46,9 @@ def run(): --persist Save and restore data to filesystem w/o going to the network --date= Date for filtering data. Can be either a single date(time) or an ISO-8601 time interval, see https://en.wikipedia.org/wiki/ISO_8601#Time_intervals. + --sql= SQL query to apply to DataFrame. --format= Output format. [Default: json] + --target= Output target for storing data into different data sinks. --version Show version information --debug Enable debug messages -h --help Show this screen @@ -111,7 +110,21 @@ def run(): wetterdienst stations --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 wetterdienst readings --resolution=daily --parameter=kl --period=recent --lat=49.9195 --lon=8.9671 --distance=25 --date=2020-06-30 - Examples for inquring metadata: + Examples using SQL filtering: + + # Find stations by state. + wetterdienst stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE state='Sachsen'" + + # Find stations by name (LIKE query). + wetterdienst stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE lower(station_name) LIKE lower('%dresden%')" + + # Find stations by name (regexp query). + wetterdienst stations --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE regexp_matches(lower(station_name), lower('.*dresden.*'))" + + # Filter measurements: Display daily climate observation readings where the maximum temperature is below two degrees. + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --sql="SELECT * FROM data WHERE element='temperature_air_max_200' AND value < 2.0;" + + Examples for inquiring metadata: # Display list of available parameters (air_temperature, precipitation, pressure, ...) wetterdienst about parameters @@ -132,6 +145,20 @@ def run(): # Tell me all parameters available for 'daily' resolution. wetterdienst about coverage --resolution=daily + Examples for exporting data to databases: + + # Shortcut command for fetching readings from DWD + alias fetch="wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent" + + # Store readings to DuckDB + fetch --target="duckdb://database=dwd.duckdb&table=weather" + + # Store readings to InfluxDB + fetch --target="influxdb://localhost/?database=dwd&table=weather" + + # Store readings to CrateDB + fetch --target="crate://localhost/?database=dwd&table=weather" + """ # Read command line options. @@ -150,6 +177,11 @@ def run(): about(options) return + # Sanity checks. + if options.readings and options.format == "geojson": + raise KeyError("GeoJSON format only available for stations output") + + # Acquire station list. if options.stations: df = metadata_for_climate_observations( parameter=options.parameter, @@ -168,11 +200,16 @@ def run(): log.error("No data available for given constraints") sys.exit(1) + data = DataPackage(df=df) + + # Acquire observations. elif options.readings: + # Use list of station identifiers. if options.station: station_ids = read_list(options.station) + # Use coordinates for a nearby search to determine list of stations. elif options.latitude and options.longitude: df = get_nearby(options) station_ids = df.STATION_ID.unique() @@ -180,6 +217,7 @@ def run(): else: raise KeyError("Either --station or --lat, --lon required") + # Funnel all parameters to the workhorse. request = DWDStationRequest( station_ids=station_ids, parameter=read_list(options.parameter), @@ -190,86 +228,41 @@ def run(): humanize_column_names=True, tidy_data=True, ) - data = list(request.collect_data()) - if not data: - log.error("No data available for given constraints") + # Collect data and merge together. + data = DataPackage() + try: + data.collect(request) + + except ValueError as ex: + log.error(ex) sys.exit(1) - df = pd.concat(data) - - if options.readings: - - if options.date: - - # Filter by time interval. - if "/" in options.date: - date_from, date_to = options.date.split("/") - date_from = parse_datetime(date_from) - date_to = parse_datetime(date_to) - if request.time_resolution in ( - TimeResolution.ANNUAL, - TimeResolution.MONTHLY, - ): - date_from, date_to = mktimerange( - request.time_resolution, date_from, date_to - ) - expression = (date_from <= df[DWDMetaColumns.FROM_DATE.value]) & ( - df[DWDMetaColumns.TO_DATE.value] <= date_to - ) - else: - expression = (date_from <= df[DWDMetaColumns.DATE.value]) & ( - df[DWDMetaColumns.DATE.value] <= date_to - ) - df = df[expression] - - # Filter by date. - else: - date = parse_datetime(options.date) - if request.time_resolution in ( - TimeResolution.ANNUAL, - TimeResolution.MONTHLY, - ): - date_from, date_to = mktimerange(request.time_resolution, date) - expression = (date_from <= df[DWDMetaColumns.FROM_DATE.value]) & ( - df[DWDMetaColumns.TO_DATE.value] <= date_to - ) - else: - expression = date == df[DWDMetaColumns.DATE.value] - df = df[expression] + # Filter readings by datetime expression. + if options.readings and options.date: + data.filter_by_date(options.date, request.time_resolution) # Make column names lowercase. - df = df.rename(columns=str.lower) - for attribute in DWDMetaColumns.PARAMETER, DWDMetaColumns.ELEMENT: - attribute_name = attribute.value.lower() - if attribute_name in df: - df[attribute_name] = df[attribute_name].str.lower() - - # Output as JSON. - if options.format == "json": - output = df.to_json(orient="records", date_format="iso", indent=4) - - # Output as GeoJSON. - elif options.format == "geojson": - if options.readings: - raise KeyError("GeoJSON format only available for stations output") - output = json.dumps(stations_to_geojson(df), indent=4) - - # Output as CSV. - elif options.format == "csv": - output = df.to_csv(index=False, date_format="%Y-%m-%dT%H-%M-%S") - - # Output as XLSX. - # FIXME: Make --format=excel write to a designated file. - elif options.format == "excel": - # TODO: Obtain output file name from command line. - output_filename = "output.xlsx" - log.info(f"Writing {output_filename}") - df.to_excel(output_filename, index=False) + data.lowercase_fieldnames() + + # Apply filtering by SQL. + if options.sql: + log.info(f"Filtering with SQL: {options.sql}") + data.filter_by_sql(options.sql) + + # Emit to data sink, e.g. write to database. + if options.target: + log.info(f"Writing data to target {options.target}") + data.export(options.target) return - else: - log.error('Output format must be one of "json", "geojson", "csv", "excel".') + # Render to output format. + try: + output = data.format(options.format) + except KeyError as ex: + log.error( + f'{ex}. Output format must be one of "json", "geojson", "csv", "excel".' + ) sys.exit(1) print(output) diff --git a/wetterdienst/io.py b/wetterdienst/io.py new file mode 100644 index 000000000..b115ebe7e --- /dev/null +++ b/wetterdienst/io.py @@ -0,0 +1,369 @@ +import json +import logging +from typing import Union +from urllib.parse import urlparse, parse_qs + +import pandas as pd + +from wetterdienst import DWDStationRequest, TimeResolution +from wetterdienst.additionals.geo_location import stations_to_geojson +from wetterdienst.additionals.time_handling import parse_datetime, mktimerange +from wetterdienst.enumerations.column_names_enumeration import DWDMetaColumns + +log = logging.getLogger(__name__) + + +class DataPackage: + """ + Postprocessing DWD data. + + This aids in collecting, filtering, formatting and emitting data + acquired through the core machinery. + """ + + def __init__( + self, df: pd.DataFrame = None, request: Union[DWDStationRequest] = None + ): + self.df = df + self.request = request + + if self.request is not None: + self.collect(self.request) + + def collect(self, request: Union[DWDStationRequest]): + """ + Collect all data from ``DWDStationRequest`` and assign to ``self.df``. + + :param request: The DWDStationRequest instance. + :return: self + """ + + data = list(request.collect_data()) + + if not data: + raise ValueError("No data available for given constraints") + + self.df = pd.concat(data) + + return self + + def filter_by_date( + self, date: str, time_resolution: TimeResolution + ) -> pd.DataFrame: + """ + Filter Pandas DataFrame by date or date interval. + + Accepts different kinds of date formats, like: + + - 2020-05-01 + - 2020-06-15T12 + - 2020-05 + - 2019 + - 2020-05-01/2020-05-05 + - 2017-01/2019-12 + - 2010/2020 + + :param date: + :param time_resolution: + :return: Filtered DataFrame + """ + + # Filter by date interval. + if "/" in date: + date_from, date_to = date.split("/") + date_from = parse_datetime(date_from) + date_to = parse_datetime(date_to) + if time_resolution in ( + TimeResolution.ANNUAL, + TimeResolution.MONTHLY, + ): + date_from, date_to = mktimerange(time_resolution, date_from, date_to) + expression = (date_from <= self.df[DWDMetaColumns.FROM_DATE.value]) & ( + self.df[DWDMetaColumns.TO_DATE.value] <= date_to + ) + else: + expression = (date_from <= self.df[DWDMetaColumns.DATE.value]) & ( + self.df[DWDMetaColumns.DATE.value] <= date_to + ) + df = self.df[expression] + + # Filter by specific date. + else: + date = parse_datetime(date) + if time_resolution in ( + TimeResolution.ANNUAL, + TimeResolution.MONTHLY, + ): + date_from, date_to = mktimerange(time_resolution, date) + expression = (date_from <= self.df[DWDMetaColumns.FROM_DATE.value]) & ( + self.df[DWDMetaColumns.TO_DATE.value] <= date_to + ) + else: + expression = date == self.df[DWDMetaColumns.DATE.value] + df = self.df[expression] + + return df + + def lowercase_fieldnames(self): + """ + Make Pandas DataFrame column names lowercase. + + :return: Mungled DataFrame + """ + self.df = self.df.rename(columns=str.lower) + for attribute in DWDMetaColumns.PARAMETER, DWDMetaColumns.ELEMENT: + attribute_name = attribute.value.lower() + if attribute_name in self.df: + self.df[attribute_name] = self.df[attribute_name].str.lower() + return self + + def filter_by_sql(self, sql: str) -> pd.DataFrame: + """ + Filter Pandas DataFrame using an SQL query. + The virtual table name is "data", so queries + should look like ``SELECT * FROM data;``. + + This implementation is based on DuckDB, so please + have a look at its SQL documentation. + + - https://duckdb.org/docs/sql/introduction + + :param sql: A SQL expression. + :return: Filtered DataFrame + """ + import duckdb + + return duckdb.query(self.df, "data", sql).df() + + def format(self, format: str) -> str: + """ + Format/render Pandas DataFrame to given output format. + + :param format: One of json, geojson, csv, excel. + :return: Rendered payload. + """ + + # Output as JSON. + if format == "json": + output = self.df.to_json(orient="records", date_format="iso", indent=4) + + # Output as GeoJSON. + elif format == "geojson": + output = json.dumps(stations_to_geojson(self.df), indent=4) + + # Output as CSV. + elif format == "csv": + output = self.df.to_csv(index=False, date_format="%Y-%m-%dT%H-%M-%S") + + # Output as XLSX. + # FIXME: Make --format=excel write to a designated file. + elif format == "excel": + # TODO: Obtain output file name from command line. + output_filename = "output.xlsx" + log.info(f"Writing {output_filename}") + self.df.to_excel(output_filename, index=False) + output = None + + else: + raise KeyError("Unknown output format") + + return output + + def export(self, target: str): + """ + Emit Pandas DataFrame to target. A target + is identified by a connection string. + + Examples: + + - duckdb://dwd.duckdb?table=weather + - influxdb://localhost/?database=dwd&table=weather + - crate://localhost/?database=dwd&table=weather + + Dispatch data to different data sinks. Currently, SQLite, DuckDB, + InfluxDB and CrateDB are implemented. However, through the SQLAlchemy + layer, it should actually work with any supported SQL database. + + - https://docs.sqlalchemy.org/en/13/dialects/ + + :param target: Target connection string. + :return: self + """ + + database, tablename = ConnectionString(target).get() + + if target.startswith("duckdb://"): + """ + ==================== + DuckDB database sink + ==================== + + Install Python driver:: + + pip install duckdb + + Acquire data:: + + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --target="duckdb:///dwd.duckdb?table=weather" + + Example queries:: + + python -c 'import duckdb; c = duckdb.connect(database="dwd.duckdb"); print(c.table("weather"))' # noqa + python -c 'import duckdb; c = duckdb.connect(database="dwd.duckdb"); print(c.execute("SELECT * FROM weather").df())' # noqa + + """ + log.info(f"Writing to DuckDB {database, tablename}") + import duckdb + + connection = duckdb.connect(database=database, read_only=False) + connection.register("origin", self.df) + connection.execute(f"DROP TABLE IF EXISTS {tablename};") + connection.execute(f"CREATE TABLE {tablename} AS SELECT * FROM origin;") + + weather_table = connection.table(tablename) + print(weather_table) + print("Cardinalities:") + print(weather_table.to_df().count()) + connection.close() + log.info("Writing to DuckDB finished") + + elif target.startswith("influxdb://"): + """ + ====================== + InfluxDB database sink + ====================== + + Install Python driver:: + + pip install influxdb + + Run database:: + + docker run --publish "8086:8086" influxdb/influxdb:1.8.2 + + Acquire data:: + + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --target="influxdb://localhost/?database=dwd&table=weather" + + Example queries:: + + http 'localhost:8086/query?db=dwd&q=SELECT * FROM weather;' + http 'localhost:8086/query?db=dwd&q=SELECT COUNT(*) FROM weather;' + """ + log.info(f"Writing to InfluxDB {database, tablename}") + from influxdb.dataframe_client import DataFrameClient + + # Setup the connection. + c = DataFrameClient(database=database) + c.create_database(database) + + # Mungle the data frame. + df = self.df.set_index(pd.DatetimeIndex(self.df["date"])) + df = df.drop(["date"], axis=1) + df = df.dropna() + + # Write to InfluxDB. + c.write_points( + dataframe=df, + measurement=tablename, + tag_columns=["station_id", "parameter", "element"], + ) + log.info("Writing to InfluxDB finished") + + elif target.startswith("crate://"): + """ + ===================== + CrateDB database sink + ===================== + + Install Python driver:: + + pip install crate[sqlalchemy] crash + + Run database:: + + docker run --publish "4200:4200" --env CRATE_HEAP_SIZE=512M crate/crate:4.2.4 + + Acquire data:: + + wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent --target="crate://localhost/?database=dwd&table=weather" + + Example queries:: + + crash -c 'select * from weather;' + crash -c 'select count(*) from weather;' + crash -c "select *, date_format('%Y-%m-%dT%H:%i:%s.%fZ', date) as datetime from weather order by datetime limit 10;" # noqa + + """ + log.info("Writing to CrateDB") + self.df.to_sql( + name=tablename, + con=target, + if_exists="replace", + index=False, + ) + log.info("Writing to CrateDB finished") + + else: + """ + ======================== + SQLAlchemy database sink + ======================== + + Install Python driver:: + + pip install sqlalchemy + + Examples:: + + # Prepare + alias fetch='wetterdienst readings --station=1048,4411 --parameter=kl --resolution=daily --period=recent' + + # Acquire data. + fetch --target="sqlite:///dwd.sqlite?table=weather" + + # Query data. + sqlite3 dwd.sqlite "SELECT * FROM weather;" + + """ + log.info("Writing to SQL database") + self.df.to_sql( + name=tablename, + con=target, + if_exists="replace", + index=False, + ) + log.info("Writing to SQL database finished") + + return self + + +class ConnectionString: + def __init__(self, url): + self.url_raw = url + self.url = urlparse(url) + + def get_query_param(self, name): + query = parse_qs(self.url.query) + try: + return query[name][0] + except (KeyError, IndexError): + return None + + def get_table(self): + return self.get_query_param("table") or "weather" + + def get_database(self): + database = None + if self.url.netloc: + database = self.get_query_param("database") + else: + if self.url.path.startswith("/"): + database = self.url.path[1:] + + return database or "dwd" + + def get(self): + database = self.get_database() + tablename = self.get_table() + return database, tablename