From 3ff19055e0286be9d1ddd5707e8e0709edcdeab3 Mon Sep 17 00:00:00 2001 From: Hanne Opseth Rygg Date: Tue, 23 Apr 2024 11:19:09 +0200 Subject: [PATCH 1/5] make progress bar dependent on log level --- datareservoirio/client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/datareservoirio/client.py b/datareservoirio/client.py index 5c1f54dc..4b7cddb5 100644 --- a/datareservoirio/client.py +++ b/datareservoirio/client.py @@ -543,7 +543,9 @@ def get_samples_aggregate_page(url): timeout=_TIMEOUT_DEAULT, ) - progress_bar = tqdm(unit=" pages", desc="Downloading aggregate data") + if log.getEffectiveLevel() < logging.WARNING: + progress_bar = tqdm(unit=" pages", desc="Downloading aggregate data") + while next_page_link: response = get_samples_aggregate_page(next_page_link) response.raise_for_status() @@ -559,7 +561,7 @@ def get_samples_aggregate_page(url): ] # update the progress bar - if content: + if content and log.getEffectiveLevel() < logging.WARNING: progress_bar.update(1) new_df = pd.DataFrame( @@ -567,8 +569,8 @@ def get_samples_aggregate_page(url): ).astype({"values": "float64"}, errors="ignore") df = pd.concat([df, new_df]) - - progress_bar.close() + if log.getEffectiveLevel() < logging.WARNING: + progress_bar.close() series = df.set_index("index").squeeze("columns").copy(deep=True) return series From dcb44b0510a3cdef26c349a4c4b78e28d3fef332 Mon Sep 17 00:00:00 2001 From: Hanne Opseth Rygg Date: Tue, 23 Apr 2024 11:23:30 +0200 Subject: [PATCH 2/5] black --- datareservoirio/appdirs.py | 1 + tests/response_cases.py | 1 + 2 files changed, 2 insertions(+) diff --git a/datareservoirio/appdirs.py b/datareservoirio/appdirs.py index 367c664f..3fddea2f 100644 --- a/datareservoirio/appdirs.py +++ b/datareservoirio/appdirs.py @@ -3,6 +3,7 @@ https://github.com/pypa/pip/blob/master/src/pip/_internal/utils/appdirs.py and modified to suit our purposes. """ + from __future__ import absolute_import, division, print_function import os diff --git a/tests/response_cases.py b/tests/response_cases.py index 8a609afd..827cbde3 100644 --- a/tests/response_cases.py +++ b/tests/response_cases.py @@ -15,6 +15,7 @@ Note that ``url`` is defined as part of the key in RESPONSE_CASES. See ``requests.Response`` source code for more details. """ + from pathlib import Path TEST_PATH = Path(__file__).parent From dbb32bb5c88aed9b37f42517914451a10f97d4b7 Mon Sep 17 00:00:00 2001 From: Hanne Opseth Rygg Date: Tue, 23 Apr 2024 11:27:47 +0200 Subject: [PATCH 3/5] update python version --- .github/workflows/ci.yml | 4 ++-- .github/workflows/deploy_public.yml | 4 ++-- pyproject.toml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d48891b4..853dd0f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 @@ -38,4 +38,4 @@ jobs: - name: Test doc build with tox run: tox -e docs - if: ${{ (matrix.python-version == '3.10') && (matrix.os == 'ubuntu-latest')}} + if: ${{ (matrix.python-version == '3.11') && (matrix.os == 'ubuntu-latest')}} diff --git a/.github/workflows/deploy_public.yml b/.github/workflows/deploy_public.yml index 0b22db19..e854d0ba 100644 --- a/.github/workflows/deploy_public.yml +++ b/.github/workflows/deploy_public.yml @@ -35,10 +35,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install dependencies run: | diff --git a/pyproject.toml b/pyproject.toml index 32b0233a..d2920d68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,14 +11,14 @@ dynamic = ["version"] description = "DataReservoir.io Python API" readme = "README.rst" license = { file="LICENSE" } -requires-python = ">3.9" +requires-python = ">3.10" classifiers = [ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ "numpy", @@ -69,7 +69,7 @@ deps = [testenv:docs] -basepython = python3.10 +basepython = python3.11 commands = sphinx-build -W -b html -d {toxworkdir}/docs_doctree docs {toxworkdir}/docs_out deps = sphinx==5.3.0 From 13c26be9a5ddd46e21a11971a972834a2cd1aa40 Mon Sep 17 00:00:00 2001 From: Hanne Opseth Rygg Date: Thu, 25 Apr 2024 08:32:30 +0200 Subject: [PATCH 4/5] handle warning --- datareservoirio/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datareservoirio/client.py b/datareservoirio/client.py index 4b7cddb5..44e8d901 100644 --- a/datareservoirio/client.py +++ b/datareservoirio/client.py @@ -571,7 +571,9 @@ def get_samples_aggregate_page(url): df = pd.concat([df, new_df]) if log.getEffectiveLevel() < logging.WARNING: progress_bar.close() - series = df.set_index("index").squeeze("columns").copy(deep=True) + series = ( + df.infer_objects().set_index("index").squeeze("columns").copy(deep=True) + ) return series From c24c17e90ad03c70a2011ca899c93bea0b6cfcbd Mon Sep 17 00:00:00 2001 From: Hanne Opseth Rygg Date: Thu, 25 Apr 2024 08:48:49 +0200 Subject: [PATCH 5/5] documentation on log level --- docs/user_guide/advanced_config.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/advanced_config.rst b/docs/user_guide/advanced_config.rst index d551f990..0b9c5e9b 100644 --- a/docs/user_guide/advanced_config.rst +++ b/docs/user_guide/advanced_config.rst @@ -107,7 +107,10 @@ Logging To simplify debugging, enable logging for the logger named 'datareservoirio'. This is especially helpful if you experience undesired behavior in your application. If your logging requirements are solely related to :py:mod:`datareservoirio`, you can use the following code. This will provide you with an understanding of the progress made in some -of the processes in the package. It is recommended to use this logging. +of the processes in the package. +In particular, when using :py:meth:`Client.get_samples_aggregate`, lowering the log level below WARNING triggers a progress bar during data collection. +The default log level for the logger named 'datareservoirio' is WARNING. +It is recommended to use this logging. .. code-block:: python