From 15cf4d84fd3ce6b867043679f64f734d7e4590cd Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 09:55:57 +0100 Subject: [PATCH 1/7] remove mypy.overrides and dependencies --- pyproject.toml | 43 - src/neptune/legacy/__init__.py | 452 ------- src/neptune/legacy/api_exceptions.py | 305 ----- src/neptune/legacy/backend.py | 203 --- src/neptune/legacy/checkpoint.py | 22 - src/neptune/legacy/constants.py | 24 - src/neptune/legacy/envs.py | 24 - src/neptune/legacy/exceptions.py | 350 ----- src/neptune/legacy/experiments.py | 1084 --------------- src/neptune/legacy/git_info.py | 18 - src/neptune/legacy/internal/__init__.py | 15 - src/neptune/legacy/internal/abort.py | 75 -- .../legacy/internal/api_clients/__init__.py | 21 - .../internal/api_clients/backend_factory.py | 33 - .../internal/api_clients/client_config.py | 66 - .../internal/api_clients/credentials.py | 99 -- .../hosted_api_clients/__init__.py | 15 - .../hosted_alpha_leaderboard_api_client.py | 1173 ----------------- .../hosted_backend_api_client.py | 216 --- .../api_clients/hosted_api_clients/mixins.py | 114 -- .../api_clients/hosted_api_clients/utils.py | 120 -- .../internal/api_clients/offline_backend.py | 184 --- .../legacy/internal/backends/__init__.py | 15 - .../backends/hosted_neptune_backend.py | 20 - .../legacy/internal/channels/__init__.py | 15 - .../legacy/internal/channels/channels.py | 132 -- .../channels/channels_values_sender.py | 201 --- .../legacy/internal/execution/__init__.py | 15 - .../internal/execution/execution_context.py | 173 --- .../legacy/internal/experiments/__init__.py | 15 - .../legacy/internal/notebooks/__init__.py | 15 - src/neptune/legacy/internal/notebooks/comm.py | 53 - .../legacy/internal/notebooks/notebooks.py | 49 - .../legacy/internal/streams/__init__.py | 15 - .../legacy/internal/streams/channel_writer.py | 77 -- .../internal/streams/stdstream_uploader.py | 60 - .../legacy/internal/threads/__init__.py | 15 - .../internal/threads/aborting_thread.py | 65 - .../hardware_metric_reporting_thread.py | 47 - .../legacy/internal/threads/neptune_thread.py | 41 - .../legacy/internal/threads/ping_thread.py | 44 - src/neptune/legacy/internal/utils/__init__.py | 15 - .../internal/utils/alpha_integration.py | 237 ---- .../legacy/internal/utils/deprecation.py | 30 - src/neptune/legacy/internal/utils/http.py | 78 -- .../legacy/internal/utils/http_utils.py | 78 -- src/neptune/legacy/internal/utils/image.py | 98 -- .../legacy/internal/utils/source_code.py | 82 -- .../legacy/internal/websockets/__init__.py | 15 - .../legacy/internal/websockets/message.py | 123 -- .../reconnecting_websocket_factory.py | 31 - .../websockets/websocket_message_processor.py | 36 - src/neptune/legacy/model.py | 144 -- src/neptune/legacy/notebook.py | 92 -- src/neptune/legacy/oauth.py | 20 - src/neptune/legacy/patterns.py | 17 - src/neptune/legacy/projects.py | 596 --------- src/neptune/legacy/sessions.py | 231 ---- src/neptune/legacy/utils.py | 41 - tests/e2e/standard/test_legacy_client.py | 129 -- tests/unit/neptune/legacy/__init__.py | 0 tests/unit/neptune/legacy/api_models.py | 19 - .../neptune/legacy/api_objects_factory.py | 155 --- tests/unit/neptune/legacy/assertions.py | 29 - .../legacy/experiments_object_factory.py | 35 - .../neptune/legacy/http_objects_factory.py | 28 - .../unit/neptune/legacy/internal/__init__.py | 15 - .../legacy/internal/backends/__init__.py | 15 - .../backends/test_hosted_neptune_backend.py | 180 --- .../internal/backends/test_noop_object.py | 64 - .../legacy/internal/channels/__init__.py | 15 - .../channels/test_channels_values_sender.py | 318 ----- .../legacy/internal/hardware/__init__.py | 15 - .../internal/hardware/gauges/__init__.py | 15 - .../hardware/gauges/gauges_fixture.py | 67 - .../hardware/gauges/test_cpu_gauges.py | 68 - .../hardware/gauges/test_gpu_gauges.py | 77 -- .../hardware/gauges/test_memory_gauges.py | 57 - .../internal/hardware/metrics/__init__.py | 15 - .../test_metric_reporter_integration.py | 121 -- .../test_metric_service_integration.py | 154 --- .../hardware/metrics/test_metrics_factory.py | 163 --- .../internal/hardware/resources/__init__.py | 15 - .../test_system_resource_info_factory.py | 169 --- .../legacy/internal/storage/__init__.py | 15 - .../internal/storage/test_datastream.py | 75 -- .../storage/test_upload_storage_utils.py | 64 - .../neptune/legacy/internal/utils/__init__.py | 0 .../legacy/internal/utils/test_image.py | 137 -- .../neptune/legacy/oauth_objects_factory.py | 38 - .../neptune/legacy/project_test_fixture.py | 34 - tests/unit/neptune/legacy/random_utils.py | 64 - .../legacy/test_alpha_integration_backend.py | 138 -- .../neptune/legacy/test_channel_writer.py | 82 -- tests/unit/neptune/legacy/test_experiment.py | 165 --- tests/unit/neptune/legacy/test_git_info.py | 68 - tests/unit/neptune/legacy/test_imports.py | 112 -- tests/unit/neptune/legacy/test_oauth.py | 161 --- tests/unit/neptune/legacy/test_project.py | 388 ------ tests/unit/neptune/legacy/test_session.py | 76 -- tests/unit/neptune/legacy/test_utils.py | 316 ----- 101 files changed, 11558 deletions(-) delete mode 100644 src/neptune/legacy/__init__.py delete mode 100644 src/neptune/legacy/api_exceptions.py delete mode 100644 src/neptune/legacy/backend.py delete mode 100644 src/neptune/legacy/checkpoint.py delete mode 100644 src/neptune/legacy/constants.py delete mode 100644 src/neptune/legacy/envs.py delete mode 100644 src/neptune/legacy/exceptions.py delete mode 100644 src/neptune/legacy/experiments.py delete mode 100644 src/neptune/legacy/git_info.py delete mode 100644 src/neptune/legacy/internal/__init__.py delete mode 100644 src/neptune/legacy/internal/abort.py delete mode 100644 src/neptune/legacy/internal/api_clients/__init__.py delete mode 100644 src/neptune/legacy/internal/api_clients/backend_factory.py delete mode 100644 src/neptune/legacy/internal/api_clients/client_config.py delete mode 100644 src/neptune/legacy/internal/api_clients/credentials.py delete mode 100644 src/neptune/legacy/internal/api_clients/hosted_api_clients/__init__.py delete mode 100644 src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py delete mode 100644 src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_backend_api_client.py delete mode 100644 src/neptune/legacy/internal/api_clients/hosted_api_clients/mixins.py delete mode 100644 src/neptune/legacy/internal/api_clients/hosted_api_clients/utils.py delete mode 100644 src/neptune/legacy/internal/api_clients/offline_backend.py delete mode 100644 src/neptune/legacy/internal/backends/__init__.py delete mode 100644 src/neptune/legacy/internal/backends/hosted_neptune_backend.py delete mode 100644 src/neptune/legacy/internal/channels/__init__.py delete mode 100644 src/neptune/legacy/internal/channels/channels.py delete mode 100644 src/neptune/legacy/internal/channels/channels_values_sender.py delete mode 100644 src/neptune/legacy/internal/execution/__init__.py delete mode 100644 src/neptune/legacy/internal/execution/execution_context.py delete mode 100644 src/neptune/legacy/internal/experiments/__init__.py delete mode 100644 src/neptune/legacy/internal/notebooks/__init__.py delete mode 100644 src/neptune/legacy/internal/notebooks/comm.py delete mode 100644 src/neptune/legacy/internal/notebooks/notebooks.py delete mode 100644 src/neptune/legacy/internal/streams/__init__.py delete mode 100644 src/neptune/legacy/internal/streams/channel_writer.py delete mode 100644 src/neptune/legacy/internal/streams/stdstream_uploader.py delete mode 100644 src/neptune/legacy/internal/threads/__init__.py delete mode 100644 src/neptune/legacy/internal/threads/aborting_thread.py delete mode 100644 src/neptune/legacy/internal/threads/hardware_metric_reporting_thread.py delete mode 100644 src/neptune/legacy/internal/threads/neptune_thread.py delete mode 100644 src/neptune/legacy/internal/threads/ping_thread.py delete mode 100644 src/neptune/legacy/internal/utils/__init__.py delete mode 100644 src/neptune/legacy/internal/utils/alpha_integration.py delete mode 100644 src/neptune/legacy/internal/utils/deprecation.py delete mode 100644 src/neptune/legacy/internal/utils/http.py delete mode 100644 src/neptune/legacy/internal/utils/http_utils.py delete mode 100644 src/neptune/legacy/internal/utils/image.py delete mode 100644 src/neptune/legacy/internal/utils/source_code.py delete mode 100644 src/neptune/legacy/internal/websockets/__init__.py delete mode 100644 src/neptune/legacy/internal/websockets/message.py delete mode 100644 src/neptune/legacy/internal/websockets/reconnecting_websocket_factory.py delete mode 100644 src/neptune/legacy/internal/websockets/websocket_message_processor.py delete mode 100644 src/neptune/legacy/model.py delete mode 100644 src/neptune/legacy/notebook.py delete mode 100644 src/neptune/legacy/oauth.py delete mode 100644 src/neptune/legacy/patterns.py delete mode 100644 src/neptune/legacy/projects.py delete mode 100644 src/neptune/legacy/sessions.py delete mode 100644 src/neptune/legacy/utils.py delete mode 100644 tests/e2e/standard/test_legacy_client.py delete mode 100644 tests/unit/neptune/legacy/__init__.py delete mode 100644 tests/unit/neptune/legacy/api_models.py delete mode 100644 tests/unit/neptune/legacy/api_objects_factory.py delete mode 100644 tests/unit/neptune/legacy/assertions.py delete mode 100644 tests/unit/neptune/legacy/experiments_object_factory.py delete mode 100644 tests/unit/neptune/legacy/http_objects_factory.py delete mode 100644 tests/unit/neptune/legacy/internal/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/backends/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/backends/test_hosted_neptune_backend.py delete mode 100644 tests/unit/neptune/legacy/internal/backends/test_noop_object.py delete mode 100644 tests/unit/neptune/legacy/internal/channels/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/channels/test_channels_values_sender.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/gauges/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/gauges/gauges_fixture.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/gauges/test_cpu_gauges.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/gauges/test_gpu_gauges.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/gauges/test_memory_gauges.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/metrics/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_reporter_integration.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_service_integration.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/metrics/test_metrics_factory.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/resources/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/hardware/resources/test_system_resource_info_factory.py delete mode 100644 tests/unit/neptune/legacy/internal/storage/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/storage/test_datastream.py delete mode 100644 tests/unit/neptune/legacy/internal/storage/test_upload_storage_utils.py delete mode 100644 tests/unit/neptune/legacy/internal/utils/__init__.py delete mode 100644 tests/unit/neptune/legacy/internal/utils/test_image.py delete mode 100644 tests/unit/neptune/legacy/oauth_objects_factory.py delete mode 100644 tests/unit/neptune/legacy/project_test_fixture.py delete mode 100644 tests/unit/neptune/legacy/random_utils.py delete mode 100644 tests/unit/neptune/legacy/test_alpha_integration_backend.py delete mode 100644 tests/unit/neptune/legacy/test_channel_writer.py delete mode 100644 tests/unit/neptune/legacy/test_experiment.py delete mode 100644 tests/unit/neptune/legacy/test_git_info.py delete mode 100644 tests/unit/neptune/legacy/test_imports.py delete mode 100644 tests/unit/neptune/legacy/test_oauth.py delete mode 100644 tests/unit/neptune/legacy/test_project.py delete mode 100644 tests/unit/neptune/legacy/test_session.py delete mode 100644 tests/unit/neptune/legacy/test_utils.py diff --git a/pyproject.toml b/pyproject.toml index 9e2661b62..00cc6f1bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,10 +15,6 @@ python = "^3.7" importlib-metadata = { version = "*", python = "<3.8" } typing-extensions = ">=3.10.0" -# Missing compatibility layer between Python 2 and Python 3 -six = ">=1.12.0" -future = ">=0.17.1" - # Utility packaging = "*" click = ">=7.0" @@ -292,45 +288,6 @@ module = [ "neptune.internal.utils.uncaught_exception_handler", "neptune.internal.websockets.websocket_signals_background_job", "neptune.internal.websockets.websockets_factory", - "neptune.legacy", - "neptune.legacy.api_exceptions", - "neptune.legacy.backend", - "neptune.legacy.checkpoint", - "neptune.legacy.exceptions", - "neptune.legacy.experiments", - "neptune.legacy.internal.abort", - "neptune.legacy.internal.api_clients.backend_factory", - "neptune.legacy.internal.api_clients.client_config", - "neptune.legacy.internal.api_clients.credentials", - "neptune.legacy.internal.api_clients.hosted_api_clients.hosted_alpha_leaderboard_api_client", - "neptune.legacy.internal.api_clients.hosted_api_clients.hosted_backend_api_client", - "neptune.legacy.internal.api_clients.hosted_api_clients.mixins", - "neptune.legacy.internal.api_clients.hosted_api_clients.utils", - "neptune.legacy.internal.api_clients.offline_backend", - "neptune.legacy.internal.channels.channels", - "neptune.legacy.internal.channels.channels_values_sender", - "neptune.legacy.internal.execution.execution_context", - "neptune.legacy.internal.notebooks.comm", - "neptune.legacy.internal.notebooks.notebooks", - "neptune.legacy.internal.streams.channel_writer", - "neptune.legacy.internal.streams.stdstream_uploader", - "neptune.legacy.internal.threads.aborting_thread", - "neptune.legacy.internal.threads.hardware_metric_reporting_thread", - "neptune.legacy.internal.threads.neptune_thread", - "neptune.legacy.internal.threads.ping_thread", - "neptune.legacy.internal.utils.alpha_integration", - "neptune.legacy.internal.utils.deprecation", - "neptune.legacy.internal.utils.http", - "neptune.legacy.internal.utils.http_utils", - "neptune.legacy.internal.utils.image", - "neptune.legacy.internal.utils.source_code", - "neptune.legacy.internal.websockets.message", - "neptune.legacy.internal.websockets.reconnecting_websocket_factory", - "neptune.legacy.internal.websockets.websocket_message_processor", - "neptune.legacy.model", - "neptune.legacy.notebook", - "neptune.legacy.projects", - "neptune.legacy.sessions", "neptune.logging.logger", "neptune.management.exceptions", "neptune.management.internal.api", diff --git a/src/neptune/legacy/__init__.py b/src/neptune/legacy/__init__.py deleted file mode 100644 index dd7c680a8..000000000 --- a/src/neptune/legacy/__init__.py +++ /dev/null @@ -1,452 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -__all__ = [ - "Project", - "Session", - "session", - "project", - "ANONYMOUS", - "ANONYMOUS_API_TOKEN", - "CURRENT_KWARGS", - "init", - "set_project", - "create_experiment", - "get_experiment", - "append_tag", - "append_tags", - "remove_tag", - "set_property", - "remove_property", - "send_metric", - "log_metric", - "send_text", - "log_text", - "send_image", - "log_image", - "send_artifact", - "delete_artifacts", - "log_artifact", - "stop", - "InvalidNeptuneBackend", - "NeptuneIncorrectImportException", - "NeptuneUninitializedException", - "envs", - "constants", -] - - -import logging -import os -import threading - -from neptune.common.utils import assure_project_qualified_name -from neptune.legacy import ( - constants, - envs, -) -from neptune.legacy.exceptions import ( - InvalidNeptuneBackend, - NeptuneIncorrectImportException, - NeptuneUninitializedException, -) -from neptune.legacy.internal.api_clients.backend_factory import backend_factory -from neptune.legacy.internal.utils.deprecation import legacy_client_deprecation -from neptune.legacy.projects import Project -from neptune.legacy.sessions import Session - -session = None -project = None - -__lock = threading.RLock() - -_logger = logging.getLogger(__name__) - -"""Access Neptune as an anonymous user. -You can pass this value as api_token during init() call, either by an environment variable or passing it directly -""" -ANONYMOUS = constants.ANONYMOUS - -"""Anonymous user API token. -You can pass this value as api_token during init() call, either by an environment variable or passing it directly -""" -ANONYMOUS_API_TOKEN = constants.ANONYMOUS_API_TOKEN - - -CURRENT_KWARGS = ( - "project", - "run", - "custom_run_id", - "mode", - "name", - "description", - "tags", - "source_files", - "capture_stdout", - "capture_stderr", - "capture_hardware_metrics", - "fail_on_exception", - "monitoring_namespace", - "flush_period", -) - - -def _check_for_extra_kwargs(caller_name, kwargs: dict): - for name in CURRENT_KWARGS: - if name in kwargs: - raise NeptuneIncorrectImportException() - if kwargs: - first_key = next(iter(kwargs.keys())) - raise TypeError(f"{caller_name}() got an unexpected keyword argument '{first_key}'") - - -@legacy_client_deprecation -def init(project_qualified_name=None, api_token=None, proxies=None, backend=None, **kwargs): - """Initialize `Neptune client library `_ to work with - specific project. - - Authorize user, sets value of global variable ``project`` to :class:`~neptune.projects.Project` object - that can be used to create or list experiments, notebooks, etc. - - Args: - project_qualified_name (:obj:`str`, optional, default is ``None``): - Qualified name of a project in a form of ``namespace/project_name``. - If ``None``, the value of ``NEPTUNE_PROJECT`` environment variable will be taken. - - api_token (:obj:`str`, optional, default is ``None``): - User's API token. If ``None``, the value of ``NEPTUNE_API_TOKEN`` environment variable will be taken. - - .. note:: - - It is strongly recommended to use ``NEPTUNE_API_TOKEN`` environment variable rather than - placing your API token in plain text in your source code. - - proxies (:obj:`dict`, optional, default is ``None``): - Argument passed to HTTP calls made via the `Requests `_ library. - For more information see their proxies - `section `_. - - .. note:: - - Only `http` and `https` keys are supported by all features. - - .. deprecated :: 0.4.4 - - Instead, use: - - .. code :: python3 - - from neptune.legacy import HostedNeptuneBackendApiClient - neptune.init(backend=HostedNeptuneBackendApiClient(proxies=...)) - - backend (:class:`~neptune.ApiClient`, optional, default is ``None``): - By default, Neptune client library sends logs, metrics, images, etc to Neptune servers: - either publicly available SaaS, or an on-premises installation. - - You can also pass the default backend instance explicitly to specify its parameters: - - .. code :: python3 - - from neptune.legacy import HostedNeptuneBackendApiClient - neptune.init(backend=HostedNeptuneBackendApiClient(...)) - - Passing an instance of :class:`~neptune.OfflineApiClient` makes your code run without communicating - with Neptune servers. - - .. code :: python3 - - from neptune.legacy import OfflineApiClient - neptune.init(backend=OfflineApiClient()) - - .. note:: - Instead of passing a ``neptune.OfflineApiClient`` instance as ``backend``, you can set an - environment variable ``NEPTUNE_BACKEND=offline`` to override the default behaviour. - - Returns: - :class:`~neptune.projects.Project` object that is used to create or list experiments, notebooks, etc. - - Raises: - `NeptuneMissingApiTokenException`: When ``api_token`` is None - and ``NEPTUNE_API_TOKEN`` environment variable was not set. - `NeptuneMissingProjectQualifiedNameException`: When ``project_qualified_name`` is None - and ``NEPTUNE_PROJECT`` environment variable was not set. - `InvalidApiKey`: When given ``api_token`` is malformed. - `Unauthorized`: When given ``api_token`` is invalid. - - Examples: - - .. code:: python3 - - # minimal invoke - neptune.init() - - # specifying project name - neptune.init('jack/sandbox') - - # running offline - neptune.init(backend=neptune.OfflineApiClient()) - """ - - _check_for_extra_kwargs(init.__name__, kwargs) - project_qualified_name = assure_project_qualified_name(project_qualified_name) - - with __lock: - global session, project - - if backend is None: - backend_name = os.getenv(envs.BACKEND) - backend = backend_factory( - backend_name=backend_name, - api_token=api_token, - proxies=proxies, - ) - - session = Session(backend=backend) - project = session.get_project(project_qualified_name) - - return project - - -@legacy_client_deprecation -def set_project(project_qualified_name): - """Setups `Neptune client library `_ to work with specific project. - - | Sets value of global variable ``project`` to :class:`~neptune.projects.Project` object - that can be used to create or list experiments, notebooks, etc. - | If Neptune client library was not previously initialized via :meth:`~neptune.init` call - it will be initialized with API token taken from ``NEPTUNE_API_TOKEN`` environment variable. - - Args: - project_qualified_name (:obj:`str`): - Qualified name of a project in a form of ``namespace/project_name``. - - Returns: - :class:`~neptune.projects.Project` object that is used to create or list experiments, notebooks, etc. - - Raises: - `NeptuneMissingApiTokenException`: When library was not initialized previously by ``init`` call and - ``NEPTUNE_API_TOKEN`` environment variable is not set. - - Examples: - - .. code:: python3 - - # minimal invoke - neptune.set_project('jack/sandbox') - """ - - with __lock: - global session, project - - if session is None: - init(project_qualified_name=project_qualified_name) - else: - project = session.get_project(project_qualified_name) - - return project - - -@legacy_client_deprecation -def create_experiment( - name=None, - description=None, - params=None, - properties=None, - tags=None, - upload_source_files=None, - abort_callback=None, - logger=None, - upload_stdout=True, - upload_stderr=True, - send_hardware_metrics=True, - run_monitoring_thread=True, - handle_uncaught_exceptions=True, - git_info=None, - hostname=None, - notebook_id=None, -): - """Create and start Neptune experiment. - - Alias for: :meth:`~neptune.projects.Project.create_experiment` - """ - - global project - if project is None: - raise NeptuneUninitializedException() - - return project.create_experiment( - name=name, - description=description, - params=params, - properties=properties, - tags=tags, - upload_source_files=upload_source_files, - abort_callback=abort_callback, - logger=logger, - upload_stdout=upload_stdout, - upload_stderr=upload_stderr, - send_hardware_metrics=send_hardware_metrics, - run_monitoring_thread=run_monitoring_thread, - handle_uncaught_exceptions=handle_uncaught_exceptions, - git_info=git_info, - hostname=hostname, - notebook_id=notebook_id, - ) - - -@legacy_client_deprecation -def get_experiment(): - global project - if project is None: - raise NeptuneUninitializedException() - - return project._get_current_experiment() - - -@legacy_client_deprecation -def append_tag(tag, *tags): - """Append tag(s) to the experiment on the top of experiments view. - - Alias for: :meth:`~neptune.experiments.Experiment.append_tag` - """ - get_experiment().append_tag(tag, *tags) - - -@legacy_client_deprecation -def append_tags(tag, *tags): - """Append tag(s) to the experiment on the top of experiments view. - - Alias for: :meth:`~neptune.experiments.Experiment.append_tags` - """ - get_experiment().append_tag(tag, *tags) - - -@legacy_client_deprecation -def remove_tag(tag): - """Removes single tag from experiment. - - Alias for: :meth:`~neptune.experiments.Experiment.remove_tag` - """ - get_experiment().remove_tag(tag) - - -@legacy_client_deprecation -def set_property(key, value): - """Set `key-value` pair as an experiment property. - - If property with given ``key`` does not exist, it adds a new one. - - Alias for: :meth:`~neptune.experiments.Experiment.set_property` - """ - get_experiment().set_property(key, value) - - -@legacy_client_deprecation -def remove_property(key): - """Removes a property with given key. - - Alias for: :meth:`~neptune.experiments.Experiment.remove_property` - """ - get_experiment().remove_property(key) - - -@legacy_client_deprecation -def send_metric(channel_name, x, y=None, timestamp=None): - """Log metrics (numeric values) in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_metric` - """ - return get_experiment().send_metric(channel_name, x, y, timestamp) - - -@legacy_client_deprecation -def log_metric(log_name, x, y=None, timestamp=None): - """Log metrics (numeric values) in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_metric` - """ - return get_experiment().log_metric(log_name, x, y, timestamp) - - -@legacy_client_deprecation -def send_text(channel_name, x, y=None, timestamp=None): - """Log text data in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_text` - """ - return get_experiment().send_text(channel_name, x, y, timestamp) - - -@legacy_client_deprecation -def log_text(log_name, x, y=None, timestamp=None): - """Log text data in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_text` - """ - return get_experiment().send_text(log_name, x, y, timestamp) - - -@legacy_client_deprecation -def send_image(channel_name, x, y=None, name=None, description=None, timestamp=None): - """Log image data in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_image` - """ - return get_experiment().send_image(channel_name, x, y, name, description, timestamp) - - -@legacy_client_deprecation -def log_image(log_name, x, y=None, image_name=None, description=None, timestamp=None): - """Log image data in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_image` - """ - return get_experiment().send_image(log_name, x, y, image_name, description, timestamp) - - -@legacy_client_deprecation -def send_artifact(artifact, destination=None): - """Save an artifact (file) in experiment storage. - - Alias for :meth:`~neptune.experiments.Experiment.log_artifact` - """ - return get_experiment().log_artifact(artifact, destination) - - -@legacy_client_deprecation -def delete_artifacts(path): - """Delete an artifact (file/directory) from experiment storage. - - Alias for :meth:`~neptune.experiments.Experiment.delete_artifacts` - """ - return get_experiment().delete_artifacts(path) - - -@legacy_client_deprecation -def log_artifact(artifact, destination=None): - """Save an artifact (file) in experiment storage. - - Alias for :meth:`~neptune.experiments.Experiment.log_artifact` - """ - return get_experiment().log_artifact(artifact, destination) - - -@legacy_client_deprecation -def stop(traceback=None): - """Marks experiment as finished (succeeded or failed). - - Alias for :meth:`~neptune.experiments.Experiment.stop` - """ - get_experiment().stop(traceback) diff --git a/src/neptune/legacy/api_exceptions.py b/src/neptune/legacy/api_exceptions.py deleted file mode 100644 index 2ff11d3e3..000000000 --- a/src/neptune/legacy/api_exceptions.py +++ /dev/null @@ -1,305 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from neptune.common.exceptions import STYLES -from neptune.legacy import envs -from neptune.legacy.exceptions import NeptuneException - - -class NeptuneApiException(NeptuneException): - pass - - -class NeptuneSSLVerificationError(NeptuneException): - def __init__(self): - message = """ -{h1} -----NeptuneSSLVerificationError----------------------------------------------------------------------- -{end} - -Neptune client was unable to verify your SSL Certificate. - -{bold}What could go wrong?{end} - - You are behind a proxy that inspects traffic to Neptune servers. - - Contact your network administrator - - Your on-prem installation's SSL/TLS certificate is not recognized due to a custom Certificate Authority (CA). - - To check run the following command in terminal: - {bash}curl https:///api/backend/echo {end} - - Where is the address that you use to access Neptune UI i.e. abc.com - - Contact your network administrator if you get the following output: - {fail}"curl: (60) server certificate verification failed..."{end} - - Your machine software is not up-to-date. - - Minimal OS requirements: - - Windows >= XP SP3 - - macOS >= 10.12.1 - - Ubuntu >= 12.04 - - Debian >= 8 - -{bold}What can I do?{end} -You can manually configure Neptune to skip all SSL checks. To do that -set the NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE environment variable to 'TRUE'. -{bold}Note that might mean your connection is less secure{end}. - -Linux/Unix -In your terminal run: - {bash}export NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE='TRUE'{end} - -Windows -In your terminal run: - {bash}set NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE='TRUE'{end} - -Jupyter notebook -In your code cell: - {bash}%env NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE='TRUE'{end} - -You may also want to check the following docs pages: - - https://docs.neptune.ai/api/environment_variables/#neptune_allow_self_signed_certificate - - -{correct}Need help?{end}-> https://docs.neptune.ai/getting_help -""" - super().__init__(message.format(**STYLES)) - - -class ConnectionLost(NeptuneApiException): - def __init__(self): - super(ConnectionLost, self).__init__("Connection lost. Please try again.") - - -class ServerError(NeptuneApiException): - def __init__(self): - message = """ -{h1} -----ServerError----------------------------------------------------------------------- -{end} -Neptune Client Library encountered an unexpected Server Error. - -Please try again later or contact Neptune support. -""" - super(ServerError, self).__init__(message.format(**STYLES)) - - -class Unauthorized(NeptuneApiException): - def __init__(self): - message = """ -{h1} -----Unauthorized----------------------------------------------------------------------- -{end} -You have no permission to access given resource. - - - Verify your API token is correct. - See: https://docs-legacy.neptune.ai/security-and-privacy/api-tokens/how-to-find-and-set-neptune-api-token.html - - - Verify if you set your Project qualified name correctly - The correct project qualified name should look like this {correct}WORKSPACE/PROJECT_NAME{end}. - It has two parts: - - {correct}WORKSPACE{end}: which can be your username or your organization name - - {correct}PROJECT_NAME{end}: which is the actual project name you chose - - - Ask your organization administrator to grant you necessary privileges to the project -""" - super(Unauthorized, self).__init__(message.format(**STYLES)) - - -class Forbidden(NeptuneApiException): - def __init__(self): - message = """ -{h1} -----Forbidden----------------------------------------------------------------------- -{end} -You have no permission to access given resource. - - - Verify your API token is correct. - See: https://docs-legacy.neptune.ai/security-and-privacy/api-tokens/how-to-find-and-set-neptune-api-token.html - - - Verify if you set your Project qualified name correctly - The correct project qualified name should look like this {correct}WORKSPACE/PROJECT_NAME{end}. - It has two parts: - - {correct}WORKSPACE{end}: which can be your username or your organization name - - {correct}PROJECT_NAME{end}: which is the actual project name you chose - - - Ask your organization administrator to grant you necessary privileges to the project -""" - super(Forbidden, self).__init__(message.format(**STYLES)) - - -class InvalidApiKey(NeptuneApiException): - def __init__(self): - message = """ -{h1} -----InvalidApiKey----------------------------------------------------------------------- -{end} -Your API token is invalid. - -Learn how to get it in this docs page: -https://docs-legacy.neptune.ai/security-and-privacy/api-tokens/how-to-find-and-set-neptune-api-token.html - -There are two options to add it: - - specify it in your code - - set an environment variable in your operating system. - -{h2}CODE{end} -Pass the token to {bold}neptune.init(){end} via {bold}api_token{end} argument: - {python}neptune.init(project_qualified_name='WORKSPACE_NAME/PROJECT_NAME', api_token='YOUR_API_TOKEN'){end} - -{h2}ENVIRONMENT VARIABLE{end} {correct}(Recommended option){end} -or export or set an environment variable depending on your operating system: - - {correct}Linux/Unix{end} - In your terminal run: - {bash}export {env_api_token}=YOUR_API_TOKEN{end} - - {correct}Windows{end} - In your CMD run: - {bash}set {env_api_token}=YOUR_API_TOKEN{end} - -and skip the {bold}api_token{end} argument of {bold}neptune.init(){end}: - {python}neptune.init(project_qualified_name='WORKSPACE_NAME/PROJECT_NAME'){end} - -You may also want to check the following docs pages: - - https://docs-legacy.neptune.ai/security-and-privacy/api-tokens/how-to-find-and-set-neptune-api-token.html - - https://docs-legacy.neptune.ai/getting-started/quick-starts/log_first_experiment.html - -{correct}Need help?{end}-> https://docs-legacy.neptune.ai/getting-started/getting-help.html -""" - super(InvalidApiKey, self).__init__(message.format(env_api_token=envs.API_TOKEN_ENV_NAME, **STYLES)) - - -class WorkspaceNotFound(NeptuneApiException): - def __init__(self, namespace_name): - message = """ -{h1} -----WorkspaceNotFound------------------------------------------------------------------------- -{end} -Workspace {python}{workspace}{end} not found. - -Workspace is your username or a name of your team organization. -""" - super(WorkspaceNotFound, self).__init__(message.format(workspace=namespace_name, **STYLES)) - - -class ProjectNotFound(NeptuneApiException): - def __init__(self, project_identifier): - message = """ -{h1} -----ProjectNotFound------------------------------------------------------------------------- -{end} -Project {python}{project}{end} not found. - -Verify if your project's name was not misspelled. You can find proper name after logging into Neptune UI. -""" - super(ProjectNotFound, self).__init__(message.format(project=project_identifier, **STYLES)) - - -class PathInProjectNotFound(NeptuneApiException): - def __init__(self, path, project_identifier): - super(PathInProjectNotFound, self).__init__( - "Path {} was not found in project {}.".format(path, project_identifier) - ) - - -class PathInExperimentNotFound(NeptuneApiException): - def __init__(self, path, exp_identifier): - super().__init__(f"Path {path} was not found in experiment {exp_identifier}.") - - -class NotebookNotFound(NeptuneApiException): - def __init__(self, notebook_id, project=None): - if project: - super(NotebookNotFound, self).__init__( - "Notebook '{}' not found in project '{}'.".format(notebook_id, project) - ) - else: - super(NotebookNotFound, self).__init__("Notebook '{}' not found.".format(notebook_id)) - - -class ExperimentNotFound(NeptuneApiException): - def __init__(self, experiment_short_id, project_qualified_name): - super(ExperimentNotFound, self).__init__( - "Experiment '{exp}' not found in '{project}'.".format( - exp=experiment_short_id, project=project_qualified_name - ) - ) - - -class ChannelNotFound(NeptuneApiException): - def __init__(self, channel_id): - super(ChannelNotFound, self).__init__("Channel '{id}' not found.".format(id=channel_id)) - - -class ExperimentAlreadyFinished(NeptuneApiException): - def __init__(self, experiment_short_id): - super(ExperimentAlreadyFinished, self).__init__( - "Experiment '{}' is already finished.".format(experiment_short_id) - ) - - -class ExperimentLimitReached(NeptuneApiException): - def __init__(self): - super(ExperimentLimitReached, self).__init__("Experiment limit reached.") - - -class StorageLimitReached(NeptuneApiException): - def __init__(self): - super(StorageLimitReached, self).__init__("Storage limit reached.") - - -class ExperimentValidationError(NeptuneApiException): - pass - - -class ChannelAlreadyExists(NeptuneApiException): - def __init__(self, experiment_short_id, channel_name): - super(ChannelAlreadyExists, self).__init__( - "Channel with name '{}' already exists in experiment '{}'.".format(channel_name, experiment_short_id) - ) - - -class ChannelDoesNotExist(NeptuneApiException): - def __init__(self, experiment_short_id, channel_name): - super(ChannelDoesNotExist, self).__init__( - "Channel with name '{}' does not exist in experiment '{}'.".format(channel_name, experiment_short_id) - ) - - -class ChannelsValuesSendBatchError(NeptuneApiException): - @staticmethod - def _format_error(error): - return "{msg} (metricId: '{channelId}', x: {x})".format(msg=error.error, channelId=error.channelId, x=error.x) - - def __init__(self, experiment_short_id, batch_errors): - super(ChannelsValuesSendBatchError, self).__init__( - "Received batch errors sending channels' values to experiment {}. " - "Cause: {} " - "Skipping {} values.".format( - experiment_short_id, - self._format_error(batch_errors[0]) if batch_errors else "No errors", - len(batch_errors), - ) - ) - - -class ExperimentOperationErrors(NeptuneApiException): - """Handles minor errors returned by calling `client.executeOperations`""" - - def __init__(self, errors): - super().__init__() - self.errors = errors - - def __str__(self): - lines = ["Caused by:"] - for error in self.errors: - lines.append(f"\t* {error}") - return "\n".join(lines) diff --git a/src/neptune/legacy/backend.py b/src/neptune/legacy/backend.py deleted file mode 100644 index 675abd876..000000000 --- a/src/neptune/legacy/backend.py +++ /dev/null @@ -1,203 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from abc import ( - ABC, - abstractmethod, -) -from typing import Dict - -from neptune.legacy.model import ChannelWithLastValue - - -class ApiClient(ABC): - @property - @abstractmethod - def api_address(self): - pass - - @property - @abstractmethod - def display_address(self): - pass - - @property - @abstractmethod - def proxies(self): - pass - - -class BackendApiClient(ApiClient, ABC): - @abstractmethod - def get_project(self, project_qualified_name): - pass - - @abstractmethod - def get_projects(self, namespace): - pass - - @abstractmethod - def create_leaderboard_backend(self, project) -> "LeaderboardApiClient": - pass - - -class LeaderboardApiClient(ApiClient, ABC): - @abstractmethod - def get_project_members(self, project_identifier): - pass - - @abstractmethod - def get_leaderboard_entries( - self, - project, - entry_types=None, - ids=None, - states=None, - owners=None, - tags=None, - min_running_time=None, - ): - pass - - def websockets_factory(self, project_id, experiment_id): - return None - - @abstractmethod - def get_channel_points_csv(self, experiment, channel_internal_id, channel_name): - pass - - @abstractmethod - def get_metrics_csv(self, experiment): - pass - - @abstractmethod - def create_experiment( - self, - project, - name, - description, - params, - properties, - tags, - abortable, - monitored, - git_info, - hostname, - entrypoint, - notebook_id, - checkpoint_id, - ): - pass - - @abstractmethod - def upload_source_code(self, experiment, source_target_pairs): - pass - - @abstractmethod - def get_notebook(self, project, notebook_id): - pass - - @abstractmethod - def get_last_checkpoint(self, project, notebook_id): - pass - - @abstractmethod - def create_notebook(self, project): - pass - - @abstractmethod - def create_checkpoint(self, notebook_id, jupyter_path, _file=None): - pass - - @abstractmethod - def get_experiment(self, experiment_id): - pass - - @abstractmethod - def set_property(self, experiment, key, value): - pass - - @abstractmethod - def remove_property(self, experiment, key): - pass - - @abstractmethod - def update_tags(self, experiment, tags_to_add, tags_to_delete): - pass - - @abstractmethod - def create_channel(self, experiment, name, channel_type) -> ChannelWithLastValue: - pass - - @abstractmethod - def get_channels(self, experiment) -> Dict[str, object]: - pass - - @abstractmethod - def reset_channel(self, experiment, channel_id, channel_name, channel_type): - pass - - @abstractmethod - def create_system_channel(self, experiment, name, channel_type) -> ChannelWithLastValue: - pass - - @abstractmethod - def get_system_channels(self, experiment) -> Dict[str, object]: - pass - - @abstractmethod - def send_channels_values(self, experiment, channels_with_values): - pass - - @abstractmethod - def mark_failed(self, experiment, traceback): - pass - - @abstractmethod - def ping_experiment(self, experiment): - pass - - @abstractmethod - def create_hardware_metric(self, experiment, metric): - pass - - @abstractmethod - def send_hardware_metric_reports(self, experiment, metrics, metric_reports): - pass - - @abstractmethod - def log_artifact(self, experiment, artifact, destination=None): - pass - - @abstractmethod - def delete_artifacts(self, experiment, path): - pass - - @abstractmethod - def download_data(self, experiment, path, destination): - pass - - @abstractmethod - def download_sources(self, experiment, path=None, destination_dir=None): - pass - - @abstractmethod - def download_artifacts(self, experiment, path=None, destination_dir=None): - pass - - @abstractmethod - def download_artifact(self, experiment, path=None, destination_dir=None): - pass diff --git a/src/neptune/legacy/checkpoint.py b/src/neptune/legacy/checkpoint.py deleted file mode 100644 index 7bd593c3f..000000000 --- a/src/neptune/legacy/checkpoint.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class Checkpoint(object): - def __init__(self, _id, name, path): - self.id = _id - self.name = name - self.path = path diff --git a/src/neptune/legacy/constants.py b/src/neptune/legacy/constants.py deleted file mode 100644 index 1c5208437..000000000 --- a/src/neptune/legacy/constants.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2020, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Constants used by Neptune""" - -ANONYMOUS = "ANONYMOUS" - -ANONYMOUS_API_TOKEN = ( - "eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5haSIsImFwaV91cmwiOiJodHRwczovL3VpLm5lcHR1bmUuYW" - "kiLCJhcGlfa2V5IjoiYjcwNmJjOGYtNzZmOS00YzJlLTkzOWQtNGJhMDM2ZjkzMmU0In0=" -) diff --git a/src/neptune/legacy/envs.py b/src/neptune/legacy/envs.py deleted file mode 100644 index 369a4b866..000000000 --- a/src/neptune/legacy/envs.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -__all__ = ["PROJECT_ENV_NAME", "API_TOKEN_ENV_NAME", "NOTEBOOK_ID_ENV_NAME", "NOTEBOOK_PATH_ENV_NAME", "BACKEND"] - -from neptune.common.envs import ( - API_TOKEN_ENV_NAME, - BACKEND, - NOTEBOOK_ID_ENV_NAME, - NOTEBOOK_PATH_ENV_NAME, - PROJECT_ENV_NAME, -) diff --git a/src/neptune/legacy/exceptions.py b/src/neptune/legacy/exceptions.py deleted file mode 100644 index 403eb6f40..000000000 --- a/src/neptune/legacy/exceptions.py +++ /dev/null @@ -1,350 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -__all__ = [ - "EMPTY_STYLES", - "STYLES", - "UNIX_STYLES", - "WINDOWS_STYLES", - "FileNotFound", - "InvalidNotebookPath", - "NeptuneIncorrectProjectQualifiedNameException", - "NeptuneMissingProjectQualifiedNameException", - "NotADirectory", - "NotAFile", -] -from neptune.common.exceptions import ( - EMPTY_STYLES, - STYLES, - UNIX_STYLES, - WINDOWS_STYLES, - FileNotFound, - InvalidNotebookPath, - NeptuneIncorrectProjectQualifiedNameException, - NeptuneMissingProjectQualifiedNameException, - NotADirectory, - NotAFile, -) -from neptune.legacy import envs - - -class NeptuneException(Exception): - pass - - -class NeptuneUninitializedException(NeptuneException): - def __init__(self): - message = """ -{h1} -----NeptuneUninitializedException--------------------------------------------------------------------------------------- -{end} -You must initialize neptune-client before you create an experiment. - -Looks like you forgot to add: - {python}neptune.init(project_qualified_name='WORKSPACE_NAME/PROJECT_NAME', api_token='YOUR_API_TOKEN'){end} - -before you ran: - {python}neptune.create_experiment(){end} - -You may also want to check the following docs pages: - - https://docs-legacy.neptune.ai/getting-started/quick-starts/log_first_experiment.html - -{correct}Need help?{end}-> https://docs-legacy.neptune.ai/getting-started/getting-help.html -""".format( - **STYLES - ) - super(NeptuneUninitializedException, self).__init__(message) - - -class NoChannelValue(NeptuneException): - def __init__(self): - super(NoChannelValue, self).__init__("No channel value provided.") - - -class NeptuneLibraryNotInstalledException(NeptuneException): - def __init__(self, library): - message = """ -{h1} -----NeptuneLibraryNotInstalledException--------------------------------------------------------------------------------- -{end} -Looks like library {library} wasn't installed. - -To install run: - {bash}pip install {library}{end} - -You may also want to check the following docs pages: - - https://docs-legacy.neptune.ai/getting-started/installation/index.html - -{correct}Need help?{end}-> https://docs-legacy.neptune.ai/getting-started/getting-help.html -""" - super(NeptuneLibraryNotInstalledException, self).__init__(message.format(library=library, **STYLES)) - - -class InvalidChannelValue(NeptuneException): - def __init__(self, expected_type, actual_type): - super(InvalidChannelValue, self).__init__( - "Invalid channel value type. Expected: {expected}, actual: {actual}.".format( - expected=expected_type, actual=actual_type - ) - ) - - -class NeptuneNoExperimentContextException(NeptuneException): - def __init__(self): - message = """ -{h1} -----NeptuneNoExperimentContextException--------------------------------------------------------------------------------- -{end} -Neptune couldn't find an active experiment. - -Looks like you forgot to run: - {python}neptune.create_experiment(){end} - -You may also want to check the following docs pages: - - https://docs-legacy.neptune.ai/getting-started/quick-starts/log_first_experiment.html - -{correct}Need help?{end}-> https://docs-legacy.neptune.ai/getting-started/getting-help.html -""" - super(NeptuneNoExperimentContextException, self).__init__(message.format(**STYLES)) - - -class NeptuneMissingApiTokenException(NeptuneException): - def __init__(self): - message = """ -{h1} -----NeptuneMissingApiTokenException------------------------------------------------------------------------------------- -{end} -Neptune client couldn't find your API token. - -Learn how to get it in this docs page: -https://docs-legacy.neptune.ai/security-and-privacy/api-tokens/how-to-find-and-set-neptune-api-token.html - -There are two options to add it: - - specify it in your code - - set an environment variable in your operating system. - -{h2}CODE{end} -Pass the token to {bold}neptune.init(){end} via {bold}api_token{end} argument: - {python}neptune.init(project_qualified_name='WORKSPACE_NAME/PROJECT_NAME', api_token='YOUR_API_TOKEN'){end} - -{h2}ENVIRONMENT VARIABLE{end} {correct}(Recommended option){end} -or export or set an environment variable depending on your operating system: - - {correct}Linux/Unix{end} - In your terminal run: - {bash}export {env_api_token}=YOUR_API_TOKEN{end} - - {correct}Windows{end} - In your CMD run: - {bash}set {env_api_token}=YOUR_API_TOKEN{end} - -and skip the {bold}api_token{end} argument of {bold}neptune.init(){end}: - {python}neptune.init(project_qualified_name='WORKSPACE_NAME/PROJECT_NAME'){end} - -You may also want to check the following docs pages: - - https://docs-legacy.neptune.ai/security-and-privacy/api-tokens/how-to-find-and-set-neptune-api-token.html - - https://docs-legacy.neptune.ai/getting-started/quick-starts/log_first_experiment.html - -{correct}Need help?{end}-> https://docs-legacy.neptune.ai/getting-started/getting-help.html -""" - super(NeptuneMissingApiTokenException, self).__init__( - message.format(env_api_token=envs.API_TOKEN_ENV_NAME, **STYLES) - ) - - -class InvalidNeptuneBackend(NeptuneException): - def __init__(self, provided_backend_name): - super(InvalidNeptuneBackend, self).__init__( - 'Unknown {} "{}". ' - "Use this environment variable to modify neptune-client behaviour at runtime, " - "e.g. using {}=offline allows you to run your code without logging anything to Neptune" - "".format(envs.BACKEND, provided_backend_name, envs.BACKEND) - ) - - -class DeprecatedApiToken(NeptuneException): - def __init__(self, app_url): - super(DeprecatedApiToken, self).__init__( - "Your API token is deprecated. Please visit {} to get a new one.".format(app_url) - ) - - -class CannotResolveHostname(NeptuneException): - def __init__(self, host): - message = """ -{h1} -----CannotResolveHostname----------------------------------------------------------------------- -{end} -Neptune Client Library was not able to resolve hostname {host}. - -What should I do? - - Check if your computer is connected to the internet. - - Check if your computer should use any proxy to access internet. - If so, you may want to use {python}proxies{end} parameter of {python}neptune.init(){end} function. - See https://docs-legacy.neptune.ai/api-reference/neptune/index.html#neptune.init - and https://requests.readthedocs.io/en/master/user/advanced/#proxies -""" - super(CannotResolveHostname, self).__init__(message.format(host=host, **STYLES)) - - -class UnsupportedClientVersion(NeptuneException): - def __init__(self, version, minVersion, maxVersion): - super(UnsupportedClientVersion, self).__init__( - "This client version ({}) is not supported. Please install neptune-client{}".format( - version, - "==" + str(maxVersion) if maxVersion else ">=" + str(minVersion), - ) - ) - - -class UnsupportedInAlphaException(NeptuneException): - """Raised for operations which was available in old client, - but aren't supported in alpha version""" - - -class DownloadSourcesException(UnsupportedInAlphaException): - message = """ -{h1} -----DownloadSourcesException----------------------------------------------------------------------- -{end} -Neptune Client Library was not able to download single file from sources. - -Why am I seeing this? - Your project "{project}" has been migrated to new structure. - Old version of `neptune-api` is not supporting downloading particular source files. - We recommend you to use new version of api: `neptune.new`. - {correct}Need help?{end}-> https://docs.neptune.ai/getting_help - -If you don't want to adapt your code to new api yet, -you can use `download_sources` with `path` parameter set to None. -""" - - def __init__(self, experiment): - assert self.message is not None - super().__init__( - self.message.format( - project=experiment._project.internal_id, - **STYLES, - ) - ) - - -class DownloadArtifactsUnsupportedException(UnsupportedInAlphaException): - message = """ -{h1} -----DownloadArtifactsUnsupportedException----------------------------------------------------------------------- -{end} -Neptune Client Library was not able to download artifacts. -Function `download_artifacts` is deprecated. - -Why am I seeing this? - Your project "{project}" has been migrated to new structure. - Old version of `neptune-api` is not supporting downloading artifact directories. - We recommend you to use new version of api: `neptune.new`. - {correct}Need help?{end}-> https://docs.neptune.ai/getting_help - -If you don't want to adapt your code to new api yet, -you can use `download_artifact` and download files one by one. -""" - - def __init__(self, experiment): - assert self.message is not None - super().__init__( - self.message.format( - project=experiment._project.internal_id, - **STYLES, - ) - ) - - -class DownloadArtifactUnsupportedException(UnsupportedInAlphaException): - message = """ -{h1} -----DownloadArtifactUnsupportedException----------------------------------------------------------------------- -{end} -Neptune Client Library was not able to download attribute: "{artifact_path}". -It's not present in experiment {experiment} or is a directory. - -Why am I seeing this? - Your project "{project}" has been migrated to new structure. - Old version of `neptune-api` is not supporting downloading whole artifact directories. - We recommend you to use new version of api: `neptune.new`. - {correct}Need help?{end}-> https://docs.neptune.ai/getting_help - -If you don't want to adapt your code to new api yet: - - Make sure that artifact "{artifact_path}" is present in experiment "{experiment}". - - Make sure that you're addressing artifact which is file, not whole directory. -""" - - def __init__(self, artifact_path, experiment): - assert self.message is not None - super().__init__( - self.message.format( - artifact_path=artifact_path, - experiment=experiment.id, - project=experiment._project.internal_id, - **STYLES, - ) - ) - - -class DeleteArtifactUnsupportedInAlphaException(UnsupportedInAlphaException): - message = """ -{h1} -----DeleteArtifactUnsupportedInAlphaException----------------------------------------------------------------------- -{end} -Neptune Client Library was not able to delete attribute: "{artifact_path}". -It's not present in experiment {experiment} or is a directory. - -Why am I seeing this? - Your project "{project}" has been migrated to new structure. - Old version of `neptune-api` is not supporting deleting whole artifact directories. - We recommend you to use new version of api: `neptune.new`. - {correct}Need help?{end}-> https://docs.neptune.ai/getting_help - -If you don't want to adapt your code to new api yet: - - Make sure that artifact "{artifact_path}" is present in experiment "{experiment}". - - Make sure that you're addressing artifact which is file, not whole directory. -""" - - def __init__(self, artifact_path, experiment): - assert self.message is not None - super().__init__( - self.message.format( - artifact_path=artifact_path, - experiment=experiment.id, - project=experiment._project.internal_id, - **STYLES, - ) - ) - - -class NeptuneIncorrectImportException(NeptuneException): - def __init__(self): - message = """ -{h1} -----NeptuneIncorrectImportException---------------------------------------------------------------- -{end} -It seems you are trying to use the new Python API, but imported the legacy API. - -Simply update your import statement to: - {python}import neptune{end} - -You may also want to check the following docs pages: - - https://docs.neptune.ai/about/legacy/#migrating-to-neptunenew - -{correct}Need help?{end}-> https://docs.neptune.ai/getting_help -""" - super().__init__(message.format(**STYLES)) diff --git a/src/neptune/legacy/experiments.py b/src/neptune/legacy/experiments.py deleted file mode 100644 index 4e5b097c5..000000000 --- a/src/neptune/legacy/experiments.py +++ /dev/null @@ -1,1084 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import base64 -import logging -import traceback - -import pandas as pd -import six -from pandas.errors import EmptyDataError - -from neptune.common.experiments import LegacyExperiment -from neptune.common.utils import ( - align_channels_on_x, - is_float, - is_nan_or_inf, -) -from neptune.legacy.api_exceptions import ( - ChannelDoesNotExist, - ExperimentAlreadyFinished, -) -from neptune.legacy.exceptions import ( - InvalidChannelValue, - NeptuneIncorrectImportException, - NoChannelValue, -) -from neptune.legacy.internal.channels.channels import ( - ChannelNamespace, - ChannelType, - ChannelValue, -) -from neptune.legacy.internal.channels.channels_values_sender import ChannelsValuesSender -from neptune.legacy.internal.execution.execution_context import ExecutionContext -from neptune.legacy.internal.utils.deprecation import legacy_client_deprecation -from neptune.legacy.internal.utils.image import get_image_content - -_logger = logging.getLogger(__name__) - - -class Experiment(LegacyExperiment): - """A class for managing Neptune experiment. - - Each time User creates new experiment instance of this class is created. - It lets you manage experiment, :meth:`~neptune.experiments.Experiment.log_metric`, - :meth:`~neptune.experiments.Experiment.log_text`, - :meth:`~neptune.experiments.Experiment.log_image`, - :meth:`~neptune.experiments.Experiment.set_property`, - and much more. - - - Args: - backend (:obj:`neptune.ApiClient`): A ApiClient object - project (:obj:`neptune.Project`): The project this experiment belongs to - _id (:obj:`str`): Experiment id - internal_id (:obj:`str`): internal ID - - Example: - Assuming that `project` is an instance of :class:`~neptune.projects.Project`. - - .. code:: python3 - - experiment = project.create_experiment() - - Warning: - User should never create instances of this class manually. - Always use: :meth:`~neptune.projects.Project.create_experiment`. - - """ - - IMAGE_SIZE_LIMIT_MB = 15 - - @legacy_client_deprecation - def __init__(self, backend, project, _id, internal_id): - self._backend = backend - self._project = project - self._id = _id - self._internal_id = internal_id - self._channels_values_sender = ChannelsValuesSender(self) - self._execution_context = ExecutionContext(backend, self) - - def _raise_new_client_expected(self): - raise NeptuneIncorrectImportException() - - def __getitem__(self, item): - self._raise_new_client_expected() - - def __setitem__(self, key, value): - self._raise_new_client_expected() - - def __delitem__(self, item): - self._raise_new_client_expected() - - @property - def id(self): - """Experiment short id - - | Combination of project key and unique experiment number. - | Format is ``-``, for example: ``MPI-142``. - - Returns: - :obj:`str` - experiment short id - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - exp_id = experiment.id - - """ - return self._id - - @property - def name(self): - """Experiment name - - Returns: - :obj:`str` experiment name - - Examples: - Assuming that `project` is an instance of :class:`~neptune.projects.Project`. - - .. code:: python3 - - experiment = project.create_experiment('exp_name') - exp_name = experiment.name - """ - return self._backend.get_experiment(self._internal_id).name - - @property - def state(self): - """Current experiment state - - Possible values: `'running'`, `'succeeded'`, `'failed'`, `'aborted'`. - - Returns: - :obj:`str` - current experiment state - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - state_str = experiment.state - """ - return self._backend.get_experiment(self._internal_id).state - - @property - def internal_id(self): - return self._internal_id - - @property - def limits(self): - return {"channels": {"numeric": 1000, "text": 100, "image": 100}} - - def get_system_properties(self): - """Retrieve experiment properties. - - | Experiment properties are for example: `owner`, `created`, `name`, `hostname`. - | List of experiment properties may change over time. - - Returns: - :obj:`dict` - dictionary mapping a property name to value. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - sys_properties = experiment.get_system_properties - """ - experiment = self._backend.get_experiment(self._internal_id) - return { - "id": experiment.shortId, - "name": experiment.name, - "created": experiment.timeOfCreation, - "finished": experiment.timeOfCompletion, - "running_time": experiment.runningTime, - "owner": experiment.owner, - "storage_size": experiment.storageSize, - "channels_size": experiment.channelsSize, - "size": experiment.storageSize + experiment.channelsSize, - "tags": experiment.tags, - "notes": experiment.description, - "description": experiment.description, - "hostname": experiment.hostname, - } - - def get_tags(self): - """Get tags associated with experiment. - - Returns: - :obj:`list` of :obj:`str` with all tags for this experiment. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - experiment.get_tags() - """ - return self._backend.get_experiment(self._internal_id).tags - - def append_tag(self, tag, *tags): - """Append tag(s) to the current experiment. - - Alias: :meth:`~neptune.experiments.Experiment.append_tags`. - Only ``[a-zA-Z0-9]`` and ``-`` (dash) characters are allowed in tags. - - Args: - tag (single :obj:`str` or multiple :obj:`str` or :obj:`list` of :obj:`str`): - Tag(s) to add to the current experiment. - - * If :obj:`str` is passed, singe tag is added. - * If multiple - comma separated - :obj:`str` are passed, all of them are added as tags. - * If :obj:`list` of :obj:`str` is passed, all elements of the :obj:`list` are added as tags. - - Examples: - - .. code:: python3 - - neptune.append_tag('new-tag') # single tag - neptune.append_tag('first-tag', 'second-tag', 'third-tag') # few str - neptune.append_tag(['first-tag', 'second-tag', 'third-tag']) # list of str - """ - if isinstance(tag, list): - tags_list = tag - else: - tags_list = [tag] + list(tags) - self._backend.update_tags(experiment=self, tags_to_add=tags_list, tags_to_delete=[]) - - def append_tags(self, tag, *tags): - """Append tag(s) to the current experiment. - - Alias for: :meth:`~neptune.experiments.Experiment.append_tag` - """ - self.append_tag(tag, *tags) - - def remove_tag(self, tag): - """Removes single tag from the experiment. - - Args: - tag (:obj:`str`): Tag to be removed - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - # assuming experiment has tags: `['tag-1', 'tag-2']`. - experiment.remove_tag('tag-1') - - Note: - Removing a tag that is not assigned to this experiment is silently ignored. - """ - self._backend.update_tags(experiment=self, tags_to_add=[], tags_to_delete=[tag]) - - def get_channels(self): - """Alias for :meth:`~neptune.experiments.Experiment.get_logs`""" - return self.get_logs() - - def get_logs(self): - """Retrieve all log names along with their last values for this experiment. - - Returns: - :obj:`dict` - A dictionary mapping a log names to the log's last value. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - exp_logs = experiment.get_logs() - """ - - def get_channel_value(ch): - return float(ch.y) if ch.y is not None and ch.channelType == "numeric" else ch.y - - return {key: get_channel_value(ch) for key, ch in self._backend.get_channels(self).items()} - - def _get_system_channels(self): - return self._backend.get_system_channels(self) - - def send_metric(self, channel_name, x, y=None, timestamp=None): - """Log metrics (numeric values) in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_metric` - """ - return self.log_metric(channel_name, x, y, timestamp) - - def log_metric(self, log_name, x, y=None, timestamp=None): - """Log metrics (numeric values) in Neptune - - | If a log with provided ``log_name`` does not exist, it is created automatically. - | If log exists (determined by ``log_name``), then new value is appended to it. - - Args: - log_name (:obj:`str`): The name of log, i.e. `mse`, `loss`, `accuracy`. - x (:obj:`double`): Depending, whether ``y`` parameter is passed: - - * ``y`` not passed: The value of the log (data-point). - * ``y`` passed: Index of log entry being appended. Must be strictly increasing. - - y (:obj:`double`, optional, default is ``None``): The value of the log (data-point). - timestamp (:obj:`time`, optional, default is ``None``): - Timestamp to be associated with log entry. Must be Unix time. - If ``None`` is passed, `time.time() `_ - (Python 3.6 example) is invoked to obtain timestamp. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment` and - 'accuracy' log does not exists: - - .. code:: python3 - - # Both calls below have the same effect - - # Common invocation, providing log name and value - experiment.log_metric('accuracy', 0.5) - experiment.log_metric('accuracy', 0.65) - experiment.log_metric('accuracy', 0.8) - - # Providing both x and y params - experiment.log_metric('accuracy', 0, 0.5) - experiment.log_metric('accuracy', 1, 0.65) - experiment.log_metric('accuracy', 2, 0.8) - - # Common invocation, logging loss tensor in PyTorch - loss = torch.Tensor([0.89]) - experiment.log_metric('log-loss', loss) - - # Common invocation, logging metric tensor in Tensorflow - acc = tf.constant([0.93]) - experiment.log_metric('accuracy', acc) - f1_score = tf.constant(0.78) - experiment.log_metric('f1_score', f1_score) - - Note: - For efficiency, logs are uploaded in batches via a queue. - Hence, if you log a lot of data, you may experience slight delays in Neptune web application. - Note: - Passing either ``x`` or ``y`` coordinate as NaN or +/-inf causes this log entry to be ignored. - Warning is printed to ``stdout``. - """ - x, y = self._get_valid_x_y(x, y) - - if not is_float(y): - raise InvalidChannelValue(expected_type="float", actual_type=type(y).__name__) - - if is_nan_or_inf(y): - _logger.warning( - "Invalid metric value: %s for channel %s. " - "Metrics with nan or +/-inf values will not be sent to server", - y, - log_name, - ) - elif x is not None and is_nan_or_inf(x): - _logger.warning( - "Invalid metric x-coordinate: %s for channel %s. " - "Metrics with nan or +/-inf x-coordinates will not be sent to server", - x, - log_name, - ) - else: - value = ChannelValue(x, dict(numeric_value=y), timestamp) - self._channels_values_sender.send(log_name, ChannelType.NUMERIC.value, value) - - def send_text(self, channel_name, x, y=None, timestamp=None): - """Log text data in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_text` - """ - return self.log_text(channel_name, x, y, timestamp) - - def log_text(self, log_name, x, y=None, timestamp=None): - """Log text data in Neptune - - | If a log with provided ``log_name`` does not exist, it is created automatically. - | If log exists (determined by ``log_name``), then new value is appended to it. - - Args: - log_name (:obj:`str`): The name of log, i.e. `mse`, `my_text_data`, `timing_info`. - x (:obj:`double` or :obj:`str`): Depending, whether ``y`` parameter is passed: - - * ``y`` not passed: The value of the log (data-point). Must be ``str``. - * ``y`` passed: Index of log entry being appended. Must be strictly increasing. - - y (:obj:`str`, optional, default is ``None``): The value of the log (data-point). - timestamp (:obj:`time`, optional, default is ``None``): - Timestamp to be associated with log entry. Must be Unix time. - If ``None`` is passed, `time.time() `_ - (Python 3.6 example) is invoked to obtain timestamp. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - # common case, where log name and data are passed - neptune.log_text('my_text_data', str(data_item)) - - # log_name, x and timestamp are passed - neptune.log_text(log_name='logging_losses_as_text', - x=str(val_loss), - timestamp=1560430912) - - Note: - For efficiency, logs are uploaded in batches via a queue. - Hence, if you log a lot of data, you may experience slight delays in Neptune web application. - Note: - Passing ``x`` coordinate as NaN or +/-inf causes this log entry to be ignored. - Warning is printed to ``stdout``. - """ - x, y = self._get_valid_x_y(x, y) - - if x is not None and is_nan_or_inf(x): - x = None - - if not isinstance(y, six.string_types): - raise InvalidChannelValue(expected_type="str", actual_type=type(y).__name__) - - if x is not None and is_nan_or_inf(x): - _logger.warning( - "Invalid metric x-coordinate: %s for channel %s. " - "Metrics with nan or +/-inf x-coordinates will not be sent to server", - x, - log_name, - ) - else: - value = ChannelValue(x, dict(text_value=y), timestamp) - self._channels_values_sender.send(log_name, ChannelType.TEXT.value, value) - - def send_image(self, channel_name, x, y=None, name=None, description=None, timestamp=None): - """Log image data in Neptune. - - Alias for :meth:`~neptune.experiments.Experiment.log_image` - """ - return self.log_image(channel_name, x, y, name, description, timestamp) - - def log_image(self, log_name, x, y=None, image_name=None, description=None, timestamp=None): - """Log image data in Neptune - - | If a log with provided ``log_name`` does not exist, it is created automatically. - | If log exists (determined by ``log_name``), then new value is appended to it. - - Args: - log_name (:obj:`str`): The name of log, i.e. `bboxes`, `visualisations`, `sample_images`. - x (:obj:`double`): Depending, whether ``y`` parameter is passed: - - * ``y`` not passed: The value of the log (data-point). See ``y`` parameter. - * ``y`` passed: Index of log entry being appended. Must be strictly increasing. - - y (multiple types supported, optional, default is ``None``): - - The value of the log (data-point). Can be one of the following types: - - * :obj:`PIL image` - `Pillow docs `_ - * :obj:`matplotlib.figure.Figure` - `Matplotlib 3.1.1 docs `_ - * :obj:`str` - path to image file - * 2-dimensional :obj:`numpy.array` with values in the [0, 1] range - interpreted as grayscale image - * 3-dimensional :obj:`numpy.array` with values in the [0, 1] range - behavior depends on last dimension - - * if last dimension is 1 - interpreted as grayscale image - * if last dimension is 3 - interpreted as RGB image - * if last dimension is 4 - interpreted as RGBA image - * :obj:`torch.tensor` with values in the [0, 1] range. - :obj:`torch.tensor` is converted to :obj:`numpy.array` via `.numpy()` method and logged. - * :obj:`tensorflow.tensor` with values in [0, 1] range. - :obj:`tensorflow.tensor` is converted to :obj:`numpy.array` via `.numpy()` method and logged. - - image_name (:obj:`str`, optional, default is ``None``): Image name - description (:obj:`str`, optional, default is ``None``): Image description - timestamp (:obj:`time`, optional, default is ``None``): - Timestamp to be associated with log entry. Must be Unix time. - If ``None`` is passed, `time.time() `_ - (Python 3.6 example) is invoked to obtain timestamp. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - # path to image file - experiment.log_image('bbox_images', 'pictures/image.png') - experiment.log_image('bbox_images', x=5, 'pictures/image.png') - experiment.log_image('bbox_images', 'pictures/image.png', image_name='difficult_case') - - # PIL image - img = PIL.Image.new('RGB', (60, 30), color = 'red') - experiment.log_image('fig', img) - - # 2d numpy array - array = numpy.random.rand(300, 200)*255 - experiment.log_image('fig', array) - - # 3d grayscale array - array = numpy.random.rand(300, 200, 1)*255 - experiment.log_image('fig', array) - - # 3d RGB array - array = numpy.random.rand(300, 200, 3)*255 - experiment.log_image('fig', array) - - # 3d RGBA array - array = numpy.random.rand(300, 200, 4)*255 - experiment.log_image('fig', array) - - # torch tensor - tensor = torch.rand(10, 20) - experiment.log_image('fig', tensor) - - # tensorflow tensor - tensor = tensorflow.random.uniform(shape=[10, 20]) - experiment.log_image('fig', tensor) - - # matplotlib figure example 1 - from matplotlib import pyplot - pyplot.plot([1, 2, 3, 4]) - pyplot.ylabel('some numbers') - experiment.log_image('plots', plt.gcf()) - - # matplotlib figure example 2 - from matplotlib import pyplot - import numpy - - numpy.random.seed(19680801) - data = numpy.random.randn(2, 100) - - figure, axs = pyplot.subplots(2, 2, figsize=(5, 5)) - axs[0, 0].hist(data[0]) - axs[1, 0].scatter(data[0], data[1]) - axs[0, 1].plot(data[0], data[1]) - axs[1, 1].hist2d(data[0], data[1]) - - experiment.log_image('diagrams', figure) - - Note: - For efficiency, logs are uploaded in batches via a queue. - Hence, if you log a lot of data, you may experience slight delays in Neptune web application. - Note: - Passing ``x`` coordinate as NaN or +/-inf causes this log entry to be ignored. - Warning is printed to ``stdout``. - Warning: - Only images up to 15MB are supported. Larger files will not be logged to Neptune. - """ - x, y = self._get_valid_x_y(x, y) - - if x is not None and is_nan_or_inf(x): - x = None - - image_content = get_image_content(y) - if len(image_content) > self.IMAGE_SIZE_LIMIT_MB * 1024 * 1024: - _logger.warning( - "Your image is larger than %dMB. Neptune supports logging images smaller than %dMB. " - "Resize or increase compression of this image", - self.IMAGE_SIZE_LIMIT_MB, - self.IMAGE_SIZE_LIMIT_MB, - ) - image_content = None - - input_image = dict(name=image_name, description=description) - if image_content: - input_image["data"] = base64.b64encode(image_content).decode("utf-8") - - if x is not None and is_nan_or_inf(x): - _logger.warning( - "Invalid metric x-coordinate: %s for channel %s. " - "Metrics with nan or +/-inf x-coordinates will not be sent to server", - x, - log_name, - ) - else: - value = ChannelValue(x, dict(image_value=input_image), timestamp) - self._channels_values_sender.send(log_name, ChannelType.IMAGE.value, value) - - def send_artifact(self, artifact, destination=None): - """Save an artifact (file) in experiment storage. - - Alias for :meth:`~neptune.experiments.Experiment.log_artifact` - """ - return self.log_artifact(artifact, destination) - - def log_artifact(self, artifact, destination=None): - """Save an artifact (file) in experiment storage. - - Args: - artifact (:obj:`str` or :obj:`IO object`): - A path to the file in local filesystem or IO object. It can be open - file descriptor or in-memory buffer like `io.StringIO` or `io.BytesIO`. - destination (:obj:`str`, optional, default is ``None``): - A destination path. - If ``None`` is passed, an artifact file name will be used. - - Note: - If you use in-memory buffers like `io.StringIO` or `io.BytesIO`, remember that in typical case when you - write to such a buffer, it's current position is set to the end of the stream, so in order to read it's - content, you need to move back it's position to the beginning. - We recommend to call seek(0) on the in-memory buffers before passing it to Neptune. - Additionally, if you provide `io.StringIO`, it will be encoded in 'utf-8' before sent to Neptune. - - Raises: - `FileNotFound`: When ``artifact`` file was not found. - `StorageLimitReached`: When storage limit in the project has been reached. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - # simple use - experiment.log_artifact('images/wrong_prediction_1.png') - - # save file in other directory - experiment.log_artifact('images/wrong_prediction_1.png', 'validation/images/wrong_prediction_1.png') - - # save file under different name - experiment.log_artifact('images/wrong_prediction_1.png', 'images/my_image_1.png') - """ - self._backend.log_artifact(self, artifact, destination) - - def delete_artifacts(self, path): - """Removes an artifact(s) (file/directory) from the experiment storage. - - Args: - path (:obj:`list` or :obj:`str`): Path or list of paths to remove from the experiment's output - - Raises: - `FileNotFound`: If a path in experiment artifacts does not exist. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - experiment.delete_artifacts('forest_results.pkl') - experiment.delete_artifacts(['forest_results.pkl', 'directory']) - experiment.delete_artifacts('') - """ - self._backend.delete_artifacts(self, path) - - def download_artifact(self, path, destination_dir=None): - """Download an artifact (file) from the experiment storage. - - Download a file indicated by ``path`` from the experiment artifacts and save it in ``destination_dir``. - - Args: - path (:obj:`str`): Path to the file to be downloaded. - destination_dir (:obj:`str`): - The directory where the file will be downloaded. - If ``None`` is passed, the file will be downloaded to the current working directory. - - Raises: - `NotADirectory`: When ``destination_dir`` is not a directory. - `FileNotFound`: If a path in experiment artifacts does not exist. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - experiment.download_artifact('forest_results.pkl', '/home/user/files/') - """ - return self._backend.download_artifact(self, path, destination_dir) - - def download_sources(self, path=None, destination_dir=None): - """Download a directory or a single file from experiment's sources as a ZIP archive. - - Download a subdirectory (or file) ``path`` from the experiment sources and save it in ``destination_dir`` - as a ZIP archive. The name of an archive will be a name of downloaded directory (or file) with '.zip' extension. - - Args: - path (:obj:`str`): - Path of a directory or file in experiment sources to be downloaded. - If ``None`` is passed, all source files will be downloaded. - - destination_dir (:obj:`str`): The directory where the archive will be downloaded. - If ``None`` is passed, the archive will be downloaded to the current working directory. - - Raises: - `NotADirectory`: When ``destination_dir`` is not a directory. - `FileNotFound`: If a path in experiment sources does not exist. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - # Download all experiment sources to current working directory - experiment.download_sources() - - # Download a single directory - experiment.download_sources('src/my-module') - - # Download all experiment sources to user-defined directory - experiment.download_sources(destination_dir='/tmp/sources/') - - # Download a single directory to user-defined directory - experiment.download_sources('src/my-module', 'sources/') - """ - return self._backend.download_sources(self, path, destination_dir) - - def download_artifacts(self, path=None, destination_dir=None): - """Download a directory or a single file from experiment's artifacts as a ZIP archive. - - Download a subdirectory (or file) ``path`` from the experiment artifacts and save it in ``destination_dir`` - as a ZIP archive. The name of an archive will be a name of downloaded directory (or file) with '.zip' extension. - - Args: - path (:obj:`str`): - Path of a directory or file in experiment artifacts to be downloaded. - If ``None`` is passed, all artifacts will be downloaded. - - destination_dir (:obj:`str`): The directory where the archive will be downloaded. - If ``None`` is passed, the archive will be downloaded to the current working directory. - - Raises: - `NotADirectory`: When ``destination_dir`` is not a directory. - `FileNotFound`: If a path in experiment artifacts does not exist. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - # Download all experiment artifacts to current working directory - experiment.download_artifacts() - - # Download a single directory - experiment.download_artifacts('data/images') - - # Download all experiment artifacts to user-defined directory - experiment.download_artifacts(destination_dir='/tmp/artifacts/') - - # Download a single directory to user-defined directory - experiment.download_artifacts('data/images', 'artifacts/') - """ - return self._backend.download_artifacts(self, path, destination_dir) - - def reset_log(self, log_name): - """Resets the log. - - Removes all data from the log and enables it to be reused from scratch. - - Args: - log_name (:obj:`str`): The name of log to reset. - - Raises: - `ChannelDoesNotExist`: When the log with name ``log_name`` does not exist on the server. - - Example: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - experiment.reset_log('my_metric') - - Note: - Check Neptune web application to see that reset charts have no data. - """ - channel = self._find_channel(log_name, ChannelNamespace.USER) - if channel is None: - raise ChannelDoesNotExist(self.id, log_name) - self._backend.reset_channel(self, channel.id, log_name, channel.channelType) - - def get_parameters(self): - """Retrieve parameters for this experiment. - - Returns: - :obj:`dict` - dictionary mapping a parameter name to value. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - exp_params = experiment.get_parameters() - """ - experiment = self._backend.get_experiment(self.internal_id) - return dict((p.name, self._convert_parameter_value(p.value, p.parameterType)) for p in experiment.parameters) - - def get_properties(self): - """Retrieve User-defined properties for this experiment. - - Returns: - :obj:`dict` - dictionary mapping a property key to value. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`. - - .. code:: python3 - - exp_properties = experiment.get_properties() - """ - experiment = self._backend.get_experiment(self.internal_id) - return dict((p.key, p.value) for p in experiment.properties) - - def set_property(self, key, value): - """Set `key-value` pair as an experiment property. - - If property with given ``key`` does not exist, it adds a new one. - - Args: - key (:obj:`str`): Property key. - value (:obj:`obj`): New value of a property. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - experiment.set_property('model', 'LightGBM') - experiment.set_property('magic-number', 7) - """ - return self._backend.set_property( - experiment=self, - key=key, - value=value, - ) - - def remove_property(self, key): - """Removes a property with given key. - - Args: - key (single :obj:`str`): - Key of property to remove. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - experiment.remove_property('host') - """ - return self._backend.remove_property( - experiment=self, - key=key, - ) - - def get_hardware_utilization(self): - """Retrieve GPU, CPU and memory utilization data. - - Get hardware utilization metrics for entire experiment as a single - `pandas.DataFrame `_ - object. Returned DataFrame has following columns (assuming single GPU with 0 index): - - * `x_ram` - time (in milliseconds) from the experiment start, - * `y_ram` - memory usage in GB, - * `x_cpu` - time (in milliseconds) from the experiment start, - * `y_cpu` - CPU utilization percentage (0-100), - * `x_gpu_util_0` - time (in milliseconds) from the experiment start, - * `y_gpu_util_0` - GPU utilization percentage (0-100), - * `x_gpu_mem_0` - time (in milliseconds) from the experiment start, - * `y_gpu_mem_0` - GPU memory usage in GB. - - | If more GPUs are available they have their separate columns with appropriate indices (0, 1, 2, ...), - for example: `x_gpu_util_1`, `y_gpu_util_1`. - | The returned DataFrame may contain ``NaN`` s if one of the metrics has more values than others. - - Returns: - :obj:`pandas.DataFrame` - DataFrame containing the hardware utilization metrics. - - Examples: - The following values denote that after 3 seconds, the experiment used 16.7 GB of RAM - - * `x_ram` = 3000 - * `y_ram` = 16.7 - - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - hardware_df = experiment.get_hardware_utilization() - """ - metrics_csv = self._backend.get_metrics_csv(self) - try: - return pd.read_csv(metrics_csv) - except EmptyDataError: - return pd.DataFrame() - - def get_numeric_channels_values(self, *channel_names): - """Retrieve values of specified metrics (numeric logs). - - The returned - `pandas.DataFrame `_ - contains 1 additional column `x` along with the requested metrics. - - Args: - *channel_names (one or more :obj:`str`): comma-separated metric names. - - Returns: - :obj:`pandas.DataFrame` - DataFrame containing values for the requested metrics. - - | The returned DataFrame may contain ``NaN`` s if one of the metrics has more values than others. - - Example: - Invoking ``get_numeric_channels_values('loss', 'auc')`` returns DataFrame with columns - `x`, `loss`, `auc`. - - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - batch_channels = experiment.get_numeric_channels_values('batch-1-loss', 'batch-2-metric') - epoch_channels = experiment.get_numeric_channels_values('epoch-1-loss', 'epoch-2-metric') - - Note: - It's good idea to get metrics with common temporal pattern (like iteration or batch/epoch number). - Thanks to this each row of returned DataFrame has metrics from the same moment in experiment. - For example, combine epoch metrics to one DataFrame and batch metrics to the other. - """ - - channels_data = {} - channels_by_name = self._backend.get_channels(self) - for channel_name in channel_names: - channel_id = channels_by_name[channel_name].id - try: - channels_data[channel_name] = pd.read_csv( - self._backend.get_channel_points_csv(self, channel_id, channel_name), - header=None, - names=["x_{}".format(channel_name), "y_{}".format(channel_name)], - dtype=float, - ) - except EmptyDataError: - channels_data[channel_name] = pd.DataFrame( - columns=["x_{}".format(channel_name), "y_{}".format(channel_name)], - dtype=float, - ) - - return align_channels_on_x(pd.concat(channels_data.values(), axis=1, sort=False)) - - def _start( - self, - abort_callback=None, - logger=None, - upload_stdout=True, - upload_stderr=True, - send_hardware_metrics=True, - run_monitoring_thread=True, - handle_uncaught_exceptions=True, - ): - - self._execution_context.start( - abort_callback=abort_callback, - logger=logger, - upload_stdout=upload_stdout, - upload_stderr=upload_stderr, - send_hardware_metrics=send_hardware_metrics, - run_monitoring_thread=run_monitoring_thread, - handle_uncaught_exceptions=handle_uncaught_exceptions, - ) - - def stop(self, exc_tb=None): - """Marks experiment as finished (succeeded or failed). - - Args: - exc_tb (:obj:`str`, optional, default is ``None``): Additional traceback information - to be stored in experiment details in case of failure (stacktrace, etc). - If this argument is ``None`` the experiment will be marked as succeeded. - Otherwise, experiment will be marked as failed. - - Examples: - Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: - - .. code:: python3 - - # Marks experiment as succeeded - experiment.stop() - - # Assuming 'ex' is some exception, - # it marks experiment as failed with exception info in experiment details. - experiment.stop(str(ex)) - """ - - self._channels_values_sender.join() - - try: - if exc_tb is not None: - self._backend.mark_failed(self, exc_tb) - except ExperimentAlreadyFinished: - pass - - self._execution_context.stop() - - self._project._remove_stopped_experiment(self) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_tb is None: - self.stop() - else: - self.stop("\n".join(traceback.format_tb(exc_tb)) + "\n" + repr(exc_val)) - - def __str__(self): - return "Experiment({})".format(self.id) - - def __repr__(self): - return str(self) - - def __eq__(self, o): - return self._id == o._id and self._internal_id == o._internal_id and self._project == o._project - - def __ne__(self, o): - return not self.__eq__(o) - - @staticmethod - def _convert_parameter_value(value, parameter_type): - if parameter_type == "double": - return float(value) - else: - return value - - @staticmethod - def _get_valid_x_y(x, y): - """ - The goal of this function is to allow user to call experiment.log_* with any of: - - single parameter treated as y value - - both parameters (named/unnamed) - - single named y parameter - If intended X-coordinate is provided, it is validated to be a float value - """ - if x is None and y is None: - raise NoChannelValue() - - if x is None and y is not None: - return None, y - - if x is not None and y is None: - return None, x - - if x is not None and y is not None: - if not is_float(x): - raise InvalidChannelValue(expected_type="float", actual_type=type(x).__name__) - return x, y - - def _send_channels_values(self, channels_with_values): - self._backend.send_channels_values(self, channels_with_values) - - def _get_channels(self, channels_names_with_types): - existing_channels = self._backend.get_channels(self) - channels_by_name = {} - for (channel_name, channel_type) in channels_names_with_types: - channel = existing_channels.get(channel_name, None) - if channel is None: - channel = self._create_channel(channel_name, channel_type) - channels_by_name[channel.name] = channel - return channels_by_name - - def _get_channel(self, channel_name, channel_type, channel_namespace=ChannelNamespace.USER): - channel = self._find_channel(channel_name, channel_namespace) - if channel is None: - channel = self._create_channel(channel_name, channel_type, channel_namespace) - return channel - - def _find_channel(self, channel_name, channel_namespace): - if channel_namespace == ChannelNamespace.USER: - return self._backend.get_channels(self).get(channel_name, None) - elif channel_namespace == ChannelNamespace.SYSTEM: - return self._get_system_channels().get(channel_name, None) - else: - raise RuntimeError("Unknown channel namespace {}".format(channel_namespace)) - - def _create_channel(self, channel_name, channel_type, channel_namespace=ChannelNamespace.USER): - if channel_namespace == ChannelNamespace.USER: - return self._backend.create_channel(self, channel_name, channel_type) - elif channel_namespace == ChannelNamespace.SYSTEM: - return self._backend.create_system_channel(self, channel_name, channel_type) - else: - raise RuntimeError("Unknown channel namespace {}".format(channel_namespace)) diff --git a/src/neptune/legacy/git_info.py b/src/neptune/legacy/git_info.py deleted file mode 100644 index b6800f135..000000000 --- a/src/neptune/legacy/git_info.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -__all__ = ["GitInfo"] - -from neptune.common.git_info import GitInfo diff --git a/src/neptune/legacy/internal/__init__.py b/src/neptune/legacy/internal/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/abort.py b/src/neptune/legacy/internal/abort.py deleted file mode 100644 index db5928484..000000000 --- a/src/neptune/legacy/internal/abort.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -try: - import psutil - - PSUTIL_INSTALLED = True -except ImportError: - PSUTIL_INSTALLED = False - - -class CustomAbortImpl(object): - def __init__(self, runnable): - self.__runnable = runnable - - def abort(self): - self.__runnable() - - -class DefaultAbortImpl(object): - KILL_TIMEOUT = 5 - - def __init__(self, pid): - self._pid = pid - - @staticmethod - def requirements_installed(): - return PSUTIL_INSTALLED - - def abort(self): - try: - processes = self._get_process_with_children(psutil.Process(self._pid)) - except psutil.NoSuchProcess: - processes = [] - - for p in processes: - self._abort(p) - _, alive = psutil.wait_procs(processes, timeout=self.KILL_TIMEOUT) - for p in alive: - self._kill(p) - - @staticmethod - def _get_process_with_children(process): - try: - return [process] + process.children(recursive=True) - except psutil.NoSuchProcess: - return [] - - @staticmethod - def _abort(process): - try: - process.terminate() - except psutil.NoSuchProcess: - pass - - def _kill(self, process): - for process in self._get_process_with_children(process): - try: - if process.is_running(): - process.kill() - except psutil.NoSuchProcess: - pass diff --git a/src/neptune/legacy/internal/api_clients/__init__.py b/src/neptune/legacy/internal/api_clients/__init__.py deleted file mode 100644 index b7290500b..000000000 --- a/src/neptune/legacy/internal/api_clients/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -__all__ = ["HostedNeptuneBackendApiClient", "OfflineBackendApiClient"] - -from neptune.legacy.internal.api_clients.hosted_api_clients.hosted_backend_api_client import ( - HostedNeptuneBackendApiClient, -) -from neptune.legacy.internal.api_clients.offline_backend import OfflineBackendApiClient diff --git a/src/neptune/legacy/internal/api_clients/backend_factory.py b/src/neptune/legacy/internal/api_clients/backend_factory.py deleted file mode 100644 index d2e63bfc9..000000000 --- a/src/neptune/legacy/internal/api_clients/backend_factory.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from neptune.legacy.backend import BackendApiClient -from neptune.legacy.exceptions import InvalidNeptuneBackend -from neptune.legacy.internal.api_clients import ( - HostedNeptuneBackendApiClient, - OfflineBackendApiClient, -) - - -def backend_factory(*, backend_name, api_token=None, proxies=None) -> BackendApiClient: - if backend_name == "offline": - return OfflineBackendApiClient() - - elif backend_name is None: - return HostedNeptuneBackendApiClient(api_token, proxies) - - else: - raise InvalidNeptuneBackend(backend_name) diff --git a/src/neptune/legacy/internal/api_clients/client_config.py b/src/neptune/legacy/internal/api_clients/client_config.py deleted file mode 100644 index d685fee9a..000000000 --- a/src/neptune/legacy/internal/api_clients/client_config.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import dataclasses - - -@dataclasses.dataclass -class MultipartConfig: - min_chunk_size: int - max_chunk_size: int - max_chunk_count: int - max_single_part_size: int - - -class ClientConfig(object): - def __init__( - self, - api_url, - display_url, - min_recommended_version, - min_compatible_version, - max_compatible_version, - multipart_config, - ): - self._api_url = api_url - self._display_url = display_url - self._min_recommended_version = min_recommended_version - self._min_compatible_version = min_compatible_version - self._max_compatible_version = max_compatible_version - self._multipart_config = multipart_config - - @property - def api_url(self): - return self._api_url - - @property - def display_url(self): - return self._display_url - - @property - def min_recommended_version(self): - return self._min_recommended_version - - @property - def min_compatible_version(self): - return self._min_compatible_version - - @property - def max_compatible_version(self): - return self._max_compatible_version - - @property - def multipart_config(self): - return self._multipart_config diff --git a/src/neptune/legacy/internal/api_clients/credentials.py b/src/neptune/legacy/internal/api_clients/credentials.py deleted file mode 100644 index dddd13da7..000000000 --- a/src/neptune/legacy/internal/api_clients/credentials.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import base64 -import json -import logging -import os - -from neptune.legacy import envs -from neptune.legacy.api_exceptions import InvalidApiKey -from neptune.legacy.constants import ( - ANONYMOUS, - ANONYMOUS_API_TOKEN, -) -from neptune.legacy.exceptions import NeptuneMissingApiTokenException - -_logger = logging.getLogger(__name__) - - -class Credentials(object): - """It formats your Neptune api token to the format that can be understood by the Neptune Client. - - A constructor allowing you to pass the Neptune API token. - - Args: - api_token(str): This is a secret API key that you can retrieve by running - `$ neptune account api-token get` - - Attributes: - api_token: This is a secret API key that was passed at instantiation. - - Examples: - - >>> from neptune.internal.backends.credentials import Credentials - >>> credentials=Credentials('YOUR_NEPTUNE_API_KEY') - - Alternatively you can create an environment variable by running: - - $ export NEPTUNE_API_TOKEN=YOUR_API_TOKEN - - which will allow you to use the same method without `api_token` parameter provided. - - >>> credentials=Credentials() - - Note: - For security reasons it is recommended to provide api_token through environment variable `NEPTUNE_API_TOKEN`. - You can do that by going to your console and running: - - $ export NEPTUNE_API_TOKEN=YOUR_API_TOKEN` - - Token provided through environment variable takes precedence over `api_token` parameter. - """ - - def __init__(self, api_token=None): - if api_token is None: - api_token = os.getenv(envs.API_TOKEN_ENV_NAME) - - if api_token == ANONYMOUS: - api_token = ANONYMOUS_API_TOKEN - - self._api_token = api_token - if self.api_token is None: - raise NeptuneMissingApiTokenException() - - token_dict = self._api_token_to_dict(self.api_token) - self._token_origin_address = token_dict["api_address"] - self._api_url = token_dict["api_url"] if "api_url" in token_dict else None - - @property - def api_token(self): - return self._api_token - - @property - def token_origin_address(self): - return self._token_origin_address - - @property - def api_url_opt(self): - return self._api_url - - @staticmethod - def _api_token_to_dict(api_token): - try: - return json.loads(base64.b64decode(api_token.encode()).decode("utf-8")) - except Exception: - raise InvalidApiKey() diff --git a/src/neptune/legacy/internal/api_clients/hosted_api_clients/__init__.py b/src/neptune/legacy/internal/api_clients/hosted_api_clients/__init__.py deleted file mode 100644 index d71b3273e..000000000 --- a/src/neptune/legacy/internal/api_clients/hosted_api_clients/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py b/src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py deleted file mode 100644 index 1c8761a33..000000000 --- a/src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py +++ /dev/null @@ -1,1173 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import logging -import math -import os -import re -import sys -import time -from collections import namedtuple -from http.client import NOT_FOUND -from io import StringIO -from itertools import groupby -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Dict, - List, -) - -import requests -import six -from bravado.exception import HTTPNotFound - -import neptune.exceptions as alpha_exceptions -from neptune.attributes import constants as alpha_consts -from neptune.attributes.constants import ( - MONITORING_TRACEBACK_ATTRIBUTE_PATH, - SYSTEM_FAILED_ATTRIBUTE_PATH, -) -from neptune.common import exceptions as common_exceptions -from neptune.common.exceptions import ClientHttpError -from neptune.common.storage.storage_utils import normalize_file_name -from neptune.common.utils import ( - NoopObject, - assure_directory_exists, -) -from neptune.internal import operation as alpha_operation -from neptune.internal.backends import hosted_file_operations as alpha_hosted_file_operations -from neptune.internal.backends.api_model import AttributeType -from neptune.internal.backends.operation_api_name_visitor import OperationApiNameVisitor as AlphaOperationApiNameVisitor -from neptune.internal.backends.operation_api_object_converter import ( - OperationApiObjectConverter as AlphaOperationApiObjectConverter, -) -from neptune.internal.backends.utils import handle_server_raw_response_messages -from neptune.internal.operation import ( - AssignBool, - AssignString, - ConfigFloatSeries, - LogFloats, - LogStrings, -) -from neptune.internal.utils import ( - base64_decode, - base64_encode, -) -from neptune.internal.utils import paths as alpha_path_utils -from neptune.internal.utils.paths import parse_path -from neptune.legacy.api_exceptions import ( - ExperimentNotFound, - ExperimentOperationErrors, - NotebookNotFound, - PathInExperimentNotFound, - ProjectNotFound, -) -from neptune.legacy.backend import LeaderboardApiClient -from neptune.legacy.checkpoint import Checkpoint -from neptune.legacy.exceptions import ( - DeleteArtifactUnsupportedInAlphaException, - DownloadArtifactsUnsupportedException, - DownloadArtifactUnsupportedException, - DownloadSourcesException, - FileNotFound, - NeptuneException, -) -from neptune.legacy.experiments import Experiment -from neptune.legacy.internal.api_clients.hosted_api_clients.mixins import HostedNeptuneMixin -from neptune.legacy.internal.api_clients.hosted_api_clients.utils import legacy_with_api_exceptions_handler -from neptune.legacy.internal.channels.channels import ( - ChannelNamespace, - ChannelType, - ChannelValueType, -) -from neptune.legacy.internal.utils.alpha_integration import ( - AlphaChannelDTO, - AlphaChannelWithValueDTO, - AlphaParameterDTO, - AlphaPropertyDTO, - channel_type_to_clear_operation, - channel_type_to_operation, - channel_value_type_to_operation, - deprecated_img_to_alpha_image, -) -from neptune.legacy.internal.websockets.reconnecting_websocket_factory import ReconnectingWebsocketFactory -from neptune.legacy.model import ( - ChannelWithLastValue, - LeaderboardEntry, -) -from neptune.legacy.notebook import Notebook - -_logger = logging.getLogger(__name__) - -LegacyExperiment = namedtuple( - "LegacyExperiment", - "shortId " - "name " - "timeOfCreation " - "timeOfCompletion " - "runningTime " - "owner " - "storageSize " - "channelsSize " - "tags " - "description " - "hostname " - "state " - "properties " - "parameters", -) - -LegacyLeaderboardEntry = namedtuple( - "LegacyExperiment", - "id " - "organizationName " - "projectName " - "shortId " - "name " - "state " - "timeOfCreation " - "timeOfCompletion " - "runningTime " - "owner " - "size " - "tags " - "description " - "channelsLastValues " - "parameters " - "properties", -) - -if TYPE_CHECKING: - from neptune.legacy.internal.api_clients import HostedNeptuneBackendApiClient - - -class HostedAlphaLeaderboardApiClient(HostedNeptuneMixin, LeaderboardApiClient): - @legacy_with_api_exceptions_handler - def __init__(self, backend_api_client: "HostedNeptuneBackendApiClient"): - self._backend_api_client = backend_api_client - - self._client_config = self._create_client_config( - api_token=self.credentials.api_token, backend_client=self.backend_client - ) - - self.leaderboard_swagger_client = self._get_swagger_client( - "{}/api/leaderboard/swagger.json".format(self._client_config.api_url), - self._backend_api_client.http_client, - ) - - if sys.version_info >= (3, 7): - try: - os.register_at_fork(after_in_child=self._handle_fork_in_child) - except AttributeError: - pass - - def _handle_fork_in_child(self): - self.leaderboard_swagger_client = NoopObject() - - @property - def http_client(self): - return self._backend_api_client.http_client - - @property - def backend_client(self): - return self._backend_api_client.backend_client - - @property - def authenticator(self): - return self._backend_api_client.authenticator - - @property - def credentials(self): - return self._backend_api_client.credentials - - @property - def backend_swagger_client(self): - return self._backend_api_client.backend_swagger_client - - @property - def client_lib_version(self): - return self._backend_api_client.client_lib_version - - @property - def api_address(self): - return self._client_config.api_url - - @property - def display_address(self): - return self._backend_api_client.display_address - - @property - def proxies(self): - return self._backend_api_client.proxies - - @legacy_with_api_exceptions_handler - def get_project_members(self, project_identifier): - try: - r = self.backend_swagger_client.api.listProjectMembers(projectIdentifier=project_identifier).response() - return r.result - except HTTPNotFound: - raise ProjectNotFound(project_identifier) - - @legacy_with_api_exceptions_handler - def create_experiment( - self, - project, - name, - description, - params, - properties, - tags, - abortable, # deprecated in alpha - monitored, # deprecated in alpha - git_info, - hostname, - entrypoint, - notebook_id, - checkpoint_id, - ): - if not isinstance(name, six.string_types): - raise ValueError("Invalid name {}, should be a string.".format(name)) - if not isinstance(description, six.string_types): - raise ValueError("Invalid description {}, should be a string.".format(description)) - if not isinstance(params, dict): - raise ValueError("Invalid params {}, should be a dict.".format(params)) - if not isinstance(properties, dict): - raise ValueError("Invalid properties {}, should be a dict.".format(properties)) - if hostname is not None and not isinstance(hostname, six.string_types): - raise ValueError("Invalid hostname {}, should be a string.".format(hostname)) - if entrypoint is not None and not isinstance(entrypoint, six.string_types): - raise ValueError("Invalid entrypoint {}, should be a string.".format(entrypoint)) - - git_info = ( - { - "commit": { - "commitId": git_info.commit_id, - "message": git_info.message, - "authorName": git_info.author_name, - "authorEmail": git_info.author_email, - "commitDate": git_info.commit_date, - }, - "repositoryDirty": git_info.repository_dirty, - "currentBranch": git_info.active_branch, - "remotes": git_info.remote_urls, - } - if git_info - else None - ) - - api_params = { - "notebookId": notebook_id, - "checkpointId": checkpoint_id, - "projectIdentifier": str(project.internal_id), - "cliVersion": self.client_lib_version, - "gitInfo": git_info, - "customId": None, - } - - kwargs = { - "experimentCreationParams": api_params, - "X-Neptune-CliVersion": self.client_lib_version, - "_request_options": {"headers": {"X-Neptune-LegacyClient": "true"}}, - } - - try: - api_experiment = self.leaderboard_swagger_client.api.createExperiment(**kwargs).response().result - except HTTPNotFound: - raise ProjectNotFound(project_identifier=project.full_id) - - experiment = self._convert_to_experiment(api_experiment, project) - # Initialize new experiment - init_experiment_operations = self._get_init_experiment_operations( - name, description, params, properties, tags, hostname, entrypoint - ) - self._execute_operations( - experiment=experiment, - operations=init_experiment_operations, - ) - return experiment - - def upload_source_code(self, experiment, source_target_pairs): - dest_path = alpha_path_utils.parse_path(alpha_consts.SOURCE_CODE_FILES_ATTRIBUTE_PATH) - file_globs = [source_path for source_path, target_path in source_target_pairs] - upload_files_operation = alpha_operation.UploadFileSet( - path=dest_path, - file_globs=file_globs, - reset=True, - ) - self._execute_upload_operations_with_400_retry(experiment, upload_files_operation) - - @legacy_with_api_exceptions_handler - def get_notebook(self, project, notebook_id): - try: - api_notebook_list = ( - self.leaderboard_swagger_client.api.listNotebooks( - projectIdentifier=project.internal_id, id=[notebook_id] - ) - .response() - .result - ) - - if not api_notebook_list.entries: - raise NotebookNotFound(notebook_id=notebook_id, project=project.full_id) - - api_notebook = api_notebook_list.entries[0] - - return Notebook( - backend=self, - project=project, - _id=api_notebook.id, - owner=api_notebook.owner, - ) - except HTTPNotFound: - raise NotebookNotFound(notebook_id=notebook_id, project=project.full_id) - - @legacy_with_api_exceptions_handler - def get_last_checkpoint(self, project, notebook_id): - try: - api_checkpoint_list = ( - self.leaderboard_swagger_client.api.listCheckpoints(notebookId=notebook_id, offset=0, limit=1) - .response() - .result - ) - - if not api_checkpoint_list.entries: - raise NotebookNotFound(notebook_id=notebook_id, project=project.full_id) - - checkpoint = api_checkpoint_list.entries[0] - return Checkpoint(checkpoint.id, checkpoint.name, checkpoint.path) - except HTTPNotFound: - raise NotebookNotFound(notebook_id=notebook_id, project=project.full_id) - - @legacy_with_api_exceptions_handler - def create_notebook(self, project): - try: - api_notebook = ( - self.leaderboard_swagger_client.api.createNotebook(projectIdentifier=project.internal_id) - .response() - .result - ) - - return Notebook( - backend=self, - project=project, - _id=api_notebook.id, - owner=api_notebook.owner, - ) - except HTTPNotFound: - raise ProjectNotFound(project_identifier=project.full_id) - - @legacy_with_api_exceptions_handler - def create_checkpoint(self, notebook_id, jupyter_path, _file=None): - if _file is not None: - with self._upload_raw_data( - api_method=self.leaderboard_swagger_client.api.createCheckpoint, - data=_file, - headers={"Content-Type": "application/octet-stream"}, - path_params={"notebookId": notebook_id}, - query_params={"jupyterPath": jupyter_path}, - ) as response: - if response.status_code == NOT_FOUND: - raise NotebookNotFound(notebook_id=notebook_id) - else: - response.raise_for_status() - CheckpointDTO = self.leaderboard_swagger_client.get_model("CheckpointDTO") - return CheckpointDTO.unmarshal(response.json()) - else: - try: - NewCheckpointDTO = self.leaderboard_swagger_client.get_model("NewCheckpointDTO") - return ( - self.leaderboard_swagger_client.api.createEmptyCheckpoint( - notebookId=notebook_id, - checkpoint=NewCheckpointDTO(path=jupyter_path), - ) - .response() - .result - ) - except HTTPNotFound: - return None - - @legacy_with_api_exceptions_handler - def get_experiment(self, experiment_id): - api_attributes = self._get_api_experiment_attributes(experiment_id) - attributes = api_attributes.attributes - system_attributes = api_attributes.systemAttributes - - return LegacyExperiment( - shortId=system_attributes.shortId.value, - name=system_attributes.name.value, - timeOfCreation=system_attributes.creationTime.value, - timeOfCompletion=None, - runningTime=system_attributes.runningTime.value, - owner=system_attributes.owner.value, - storageSize=system_attributes.size.value, - channelsSize=0, - tags=system_attributes.tags.values, - description=system_attributes.description.value, - hostname=system_attributes.hostname.value if system_attributes.hostname else None, - state="running" if system_attributes.state.value == "running" else "succeeded", - properties=[AlphaPropertyDTO(attr) for attr in attributes if AlphaPropertyDTO.is_valid_attribute(attr)], - parameters=[AlphaParameterDTO(attr) for attr in attributes if AlphaParameterDTO.is_valid_attribute(attr)], - ) - - @legacy_with_api_exceptions_handler - def set_property(self, experiment, key, value): - """Save attribute casted to string under `alpha_consts.PROPERTIES_ATTRIBUTE_SPACE` namespace""" - self._execute_operations( - experiment=experiment, - operations=[ - alpha_operation.AssignString( - path=alpha_path_utils.parse_path(f"{alpha_consts.PROPERTIES_ATTRIBUTE_SPACE}{key}"), - value=str(value), - ) - ], - ) - - @legacy_with_api_exceptions_handler - def remove_property(self, experiment, key): - self._remove_attribute(experiment, str_path=f"{alpha_consts.PROPERTIES_ATTRIBUTE_SPACE}{key}") - - @legacy_with_api_exceptions_handler - def update_tags(self, experiment, tags_to_add, tags_to_delete): - operations = [ - alpha_operation.AddStrings( - path=alpha_path_utils.parse_path(alpha_consts.SYSTEM_TAGS_ATTRIBUTE_PATH), - values=tags_to_add, - ), - alpha_operation.RemoveStrings( - path=alpha_path_utils.parse_path(alpha_consts.SYSTEM_TAGS_ATTRIBUTE_PATH), - values=tags_to_delete, - ), - ] - self._execute_operations( - experiment=experiment, - operations=operations, - ) - - @staticmethod - def _get_channel_attribute_path(channel_name: str, channel_namespace: ChannelNamespace) -> str: - if channel_namespace == ChannelNamespace.USER: - prefix = alpha_consts.LOG_ATTRIBUTE_SPACE - else: - prefix = alpha_consts.MONITORING_ATTRIBUTE_SPACE - return f"{prefix}{channel_name}" - - def _create_channel( - self, - experiment: Experiment, - channel_id: str, - channel_name: str, - channel_type: ChannelType, - channel_namespace: ChannelNamespace, - ): - """This function is responsible for creating 'fake' channels in alpha projects. - - Since channels are abandoned in alpha api, we're mocking them using empty logging operation.""" - - operation = channel_type_to_operation(channel_type) - - log_empty_operation = operation( - path=alpha_path_utils.parse_path(self._get_channel_attribute_path(channel_name, channel_namespace)), - values=[], - ) # this operation is used to create empty attribute - self._execute_operations( - experiment=experiment, - operations=[log_empty_operation], - ) - return ChannelWithLastValue( - AlphaChannelWithValueDTO( - channelId=channel_id, - channelName=channel_name, - channelType=channel_type.value, - x=None, - y=None, - ) - ) - - @legacy_with_api_exceptions_handler - def create_channel(self, experiment, name, channel_type) -> ChannelWithLastValue: - channel_id = f"{alpha_consts.LOG_ATTRIBUTE_SPACE}{name}" - return self._create_channel( - experiment, - channel_id, - channel_name=name, - channel_type=ChannelType(channel_type), - channel_namespace=ChannelNamespace.USER, - ) - - def _get_channels(self, experiment) -> List[AlphaChannelDTO]: - try: - return [ - AlphaChannelDTO(attr) - for attr in self._get_attributes(experiment.internal_id) - if AlphaChannelDTO.is_valid_attribute(attr) - ] - except HTTPNotFound: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) - - @legacy_with_api_exceptions_handler - def get_channels(self, experiment) -> Dict[str, AlphaChannelDTO]: - api_channels = [ - channel - for channel in self._get_channels(experiment) - # return channels from LOG_ATTRIBUTE_SPACE namespace only - if channel.id.startswith(alpha_consts.LOG_ATTRIBUTE_SPACE) - ] - return {ch.name: ch for ch in api_channels} - - @legacy_with_api_exceptions_handler - def create_system_channel(self, experiment, name, channel_type) -> ChannelWithLastValue: - channel_id = f"{alpha_consts.MONITORING_ATTRIBUTE_SPACE}{name}" - return self._create_channel( - experiment, - channel_id, - channel_name=name, - channel_type=ChannelType(channel_type), - channel_namespace=ChannelNamespace.SYSTEM, - ) - - @legacy_with_api_exceptions_handler - def get_system_channels(self, experiment) -> Dict[str, AlphaChannelDTO]: - return { - channel.name: channel - for channel in self._get_channels(experiment) - if ( - channel.channelType == ChannelType.TEXT.value - and channel.id.startswith(alpha_consts.MONITORING_ATTRIBUTE_SPACE) - ) - } - - @legacy_with_api_exceptions_handler - def send_channels_values(self, experiment, channels_with_values): - send_operations = [] - for channel_with_values in channels_with_values: - channel_value_type = channel_with_values.channel_type - operation = channel_value_type_to_operation(channel_value_type) - - if channel_value_type == ChannelValueType.IMAGE_VALUE: - # IMAGE_VALUE requires minor data modification before it's sent - data_transformer = deprecated_img_to_alpha_image - else: - # otherwise use identity function as transformer - data_transformer = lambda e: e # noqa: E731 - - ch_values = [ - alpha_operation.LogSeriesValue( - value=data_transformer(ch_value.value), - step=ch_value.x, - ts=ch_value.ts, - ) - for ch_value in channel_with_values.channel_values - ] - send_operations.append( - operation( - path=alpha_path_utils.parse_path( - self._get_channel_attribute_path( - channel_with_values.channel_name, - channel_with_values.channel_namespace, - ) - ), - values=ch_values, - ) - ) - - self._execute_operations(experiment, send_operations) - - def mark_failed(self, experiment, traceback): - operations = [] - path = parse_path(SYSTEM_FAILED_ATTRIBUTE_PATH) - traceback_values = [LogStrings.ValueType(val, step=None, ts=time.time()) for val in traceback.split("\n")] - operations.append(AssignBool(path=path, value=True)) - operations.append( - LogStrings( - values=traceback_values, - path=parse_path(MONITORING_TRACEBACK_ATTRIBUTE_PATH), - ) - ) - self._execute_operations(experiment, operations) - - def ping_experiment(self, experiment): - try: - self.leaderboard_swagger_client.api.ping(experimentId=str(experiment.internal_id)).response().result - except HTTPNotFound: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) - - @staticmethod - def _get_attribute_name_for_metric(resource_type, gauge_name, gauges_count) -> str: - if gauges_count > 1: - return "monitoring/{}_{}".format(resource_type, gauge_name).lower() - return "monitoring/{}".format(resource_type).lower() - - @legacy_with_api_exceptions_handler - def create_hardware_metric(self, experiment, metric): - operations = [] - gauges_count = len(metric.gauges) - for gauge in metric.gauges: - path = parse_path(self._get_attribute_name_for_metric(metric.resource_type, gauge.name(), gauges_count)) - operations.append(ConfigFloatSeries(path, min=metric.min_value, max=metric.max_value, unit=metric.unit)) - self._execute_operations(experiment, operations) - - @legacy_with_api_exceptions_handler - def send_hardware_metric_reports(self, experiment, metrics, metric_reports): - operations = [] - metrics_by_name = {metric.name: metric for metric in metrics} - for report in metric_reports: - metric = metrics_by_name.get(report.metric.name) - gauges_count = len(metric.gauges) - for gauge_name, metric_values in groupby(report.values, lambda value: value.gauge_name): - metric_values = list(metric_values) - path = parse_path(self._get_attribute_name_for_metric(metric.resource_type, gauge_name, gauges_count)) - operations.append( - LogFloats( - path, - [LogFloats.ValueType(value.value, step=None, ts=value.timestamp) for value in metric_values], - ) - ) - self._execute_operations(experiment, operations) - - def log_artifact(self, experiment, artifact, destination=None): - if isinstance(artifact, str): - if os.path.isfile(artifact): - target_name = os.path.basename(artifact) if destination is None else destination - dest_path = self._get_dest_and_ext(target_name) - operation = alpha_operation.UploadFile( - path=dest_path, - ext="", - file_path=os.path.abspath(artifact), - ) - elif os.path.isdir(artifact): - for path, file_destination in self._log_dir_artifacts(artifact, destination): - self.log_artifact(experiment, path, file_destination) - return - else: - raise FileNotFound(artifact) - elif hasattr(artifact, "read"): - if not destination: - raise ValueError("destination is required for IO streams") - dest_path = self._get_dest_and_ext(destination) - data = artifact.read() - content = data.encode("utf-8") if isinstance(data, str) else data - operation = alpha_operation.UploadFileContent(path=dest_path, ext="", file_content=base64_encode(content)) - else: - raise ValueError("Artifact must be a local path or an IO object") - - self._execute_upload_operations_with_400_retry(experiment, operation) - - @staticmethod - def _get_dest_and_ext(target_name): - qualified_target_name = f"{alpha_consts.ARTIFACT_ATTRIBUTE_SPACE}{target_name}" - return alpha_path_utils.parse_path(normalize_file_name(qualified_target_name)) - - def _log_dir_artifacts(self, directory_path, destination): - directory_path = Path(directory_path) - prefix = directory_path.name if destination is None else destination - for path in directory_path.glob("**/*"): - if path.is_file(): - relative_path = path.relative_to(directory_path) - file_destination = prefix + "/" + str(relative_path) - yield str(path), file_destination - - def delete_artifacts(self, experiment, path): - try: - self._remove_attribute(experiment, str_path=f"{alpha_consts.ARTIFACT_ATTRIBUTE_SPACE}{path}") - except ExperimentOperationErrors as e: - if all(isinstance(err, alpha_exceptions.MetadataInconsistency) for err in e.errors): - raise DeleteArtifactUnsupportedInAlphaException(path, experiment) from None - raise - - @legacy_with_api_exceptions_handler - def download_data(self, experiment: Experiment, path: str, destination): - project_storage_path = f"artifacts/{path}" - with self._download_raw_data( - api_method=self.leaderboard_swagger_client.api.downloadAttribute, - headers={"Accept": "application/octet-stream"}, - path_params={}, - query_params={ - "experimentId": experiment.internal_id, - "attribute": project_storage_path, - }, - ) as response: - if response.status_code == NOT_FOUND: - raise PathInExperimentNotFound(path=path, exp_identifier=experiment.internal_id) - else: - response.raise_for_status() - - with open(destination, "wb") as f: - for chunk in response.iter_content(chunk_size=10 * 1024 * 1024): - if chunk: - f.write(chunk) - - def download_sources(self, experiment: Experiment, path=None, destination_dir=None): - if path is not None: - # in alpha all source files stored as single FileSet must be downloaded at once - raise DownloadSourcesException(experiment) - path = alpha_consts.SOURCE_CODE_FILES_ATTRIBUTE_PATH - - destination_dir = assure_directory_exists(destination_dir) - - download_request = self._get_file_set_download_request(experiment.internal_id, path) - alpha_hosted_file_operations.download_file_set_attribute( - swagger_client=self.leaderboard_swagger_client, - download_id=download_request.id, - destination=destination_dir, - ) - - @legacy_with_api_exceptions_handler - def _get_file_set_download_request(self, run_id: str, path: str): - params = { - "experimentId": run_id, - "attribute": path, - } - return self.leaderboard_swagger_client.api.prepareForDownloadFileSetAttributeZip(**params).response().result - - def download_artifacts(self, experiment: Experiment, path=None, destination_dir=None): - raise DownloadArtifactsUnsupportedException(experiment) - - def download_artifact(self, experiment: Experiment, path=None, destination_dir=None): - destination_dir = assure_directory_exists(destination_dir) - destination_path = os.path.join(destination_dir, os.path.basename(path)) - - try: - self.download_data(experiment, path, destination_path) - except PathInExperimentNotFound: - raise DownloadArtifactUnsupportedException(path, experiment) from None - - def _get_attributes(self, experiment_id) -> list: - return self._get_api_experiment_attributes(experiment_id).attributes - - def _get_api_experiment_attributes(self, experiment_id): - params = { - "experimentId": experiment_id, - } - return self.leaderboard_swagger_client.api.getExperimentAttributes(**params).response().result - - def _remove_attribute(self, experiment, str_path: str): - """Removes given attribute""" - self._execute_operations( - experiment=experiment, - operations=[ - alpha_operation.DeleteAttribute( - path=alpha_path_utils.parse_path(str_path), - ) - ], - ) - - @staticmethod - def _get_client_config_args(api_token): - return dict( - X_Neptune_Api_Token=api_token, - alpha="true", - ) - - def _execute_upload_operation(self, experiment: Experiment, upload_operation: alpha_operation.Operation): - experiment_id = experiment.internal_id - try: - if isinstance(upload_operation, alpha_operation.UploadFile): - alpha_hosted_file_operations.upload_file_attribute( - swagger_client=self.leaderboard_swagger_client, - container_id=experiment_id, - attribute=alpha_path_utils.path_to_str(upload_operation.path), - source=upload_operation.file_path, - ext=upload_operation.ext, - multipart_config=self._client_config.multipart_config, - ) - elif isinstance(upload_operation, alpha_operation.UploadFileContent): - alpha_hosted_file_operations.upload_file_attribute( - swagger_client=self.leaderboard_swagger_client, - container_id=experiment_id, - attribute=alpha_path_utils.path_to_str(upload_operation.path), - source=base64_decode(upload_operation.file_content), - ext=upload_operation.ext, - multipart_config=self._client_config.multipart_config, - ) - elif isinstance(upload_operation, alpha_operation.UploadFileSet): - alpha_hosted_file_operations.upload_file_set_attribute( - swagger_client=self.leaderboard_swagger_client, - container_id=experiment_id, - attribute=alpha_path_utils.path_to_str(upload_operation.path), - file_globs=upload_operation.file_globs, - reset=upload_operation.reset, - multipart_config=self._client_config.multipart_config, - ) - else: - raise NeptuneException("Upload operation in neither File or FileSet") - except common_exceptions.NeptuneException as e: - raise NeptuneException(e) from e - - return None - - def _execute_upload_operations_with_400_retry( - self, experiment: Experiment, upload_operation: alpha_operation.Operation - ): - while True: - try: - return self._execute_upload_operation(experiment, upload_operation) - except ClientHttpError as ex: - if "Length of stream does not match given range" not in ex.response: - raise ex - - @legacy_with_api_exceptions_handler - def _execute_operations(self, experiment: Experiment, operations: List[alpha_operation.Operation]): - experiment_id = experiment.internal_id - file_operations = ( - alpha_operation.UploadFile, - alpha_operation.UploadFileContent, - alpha_operation.UploadFileSet, - ) - if any(isinstance(op, file_operations) for op in operations): - raise NeptuneException( - "File operations must be handled directly by `_execute_upload_operation`," - " not by `_execute_operations` function call." - ) - - kwargs = { - "experimentId": experiment_id, - "operations": [ - { - "path": alpha_path_utils.path_to_str(op.path), - AlphaOperationApiNameVisitor().visit(op): AlphaOperationApiObjectConverter().convert(op), - } - for op in operations - ], - } - try: - result = self.leaderboard_swagger_client.api.executeOperations(**kwargs).response().result - errors = [alpha_exceptions.MetadataInconsistency(err.errorDescription) for err in result] - if errors: - raise ExperimentOperationErrors(errors=errors) - return None - except HTTPNotFound as e: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) from e - - def _get_init_experiment_operations( - self, name, description, params, properties, tags, hostname, entrypoint - ) -> List[alpha_operation.Operation]: - """Returns operations required to initialize newly created experiment""" - init_operations = list() - - # Assign experiment name - init_operations.append( - alpha_operation.AssignString( - path=alpha_path_utils.parse_path(alpha_consts.SYSTEM_NAME_ATTRIBUTE_PATH), - value=name, - ) - ) - # Assign experiment description - init_operations.append( - alpha_operation.AssignString( - path=alpha_path_utils.parse_path(alpha_consts.SYSTEM_DESCRIPTION_ATTRIBUTE_PATH), - value=description, - ) - ) - # Assign experiment parameters - for p_name, p_val in params.items(): - parameter_type, string_value = self._get_parameter_with_type(p_val) - operation_cls = alpha_operation.AssignFloat if parameter_type == "double" else alpha_operation.AssignString - init_operations.append( - operation_cls( - path=alpha_path_utils.parse_path(f"{alpha_consts.PARAMETERS_ATTRIBUTE_SPACE}{p_name}"), - value=string_value, - ) - ) - # Assign experiment properties - for p_key, p_val in properties.items(): - init_operations.append( - AssignString( - path=alpha_path_utils.parse_path(f"{alpha_consts.PROPERTIES_ATTRIBUTE_SPACE}{p_key}"), - value=str(p_val), - ) - ) - # Assign tags - if tags: - init_operations.append( - alpha_operation.AddStrings( - path=alpha_path_utils.parse_path(alpha_consts.SYSTEM_TAGS_ATTRIBUTE_PATH), - values=set(tags), - ) - ) - # Assign source hostname - if hostname: - init_operations.append( - alpha_operation.AssignString( - path=alpha_path_utils.parse_path(alpha_consts.SYSTEM_HOSTNAME_ATTRIBUTE_PATH), - value=hostname, - ) - ) - # Assign source entrypoint - if entrypoint: - init_operations.append( - alpha_operation.AssignString( - path=alpha_path_utils.parse_path(alpha_consts.SOURCE_CODE_ENTRYPOINT_ATTRIBUTE_PATH), - value=entrypoint, - ) - ) - - return init_operations - - @legacy_with_api_exceptions_handler - def reset_channel(self, experiment, channel_id, channel_name, channel_type): - op = channel_type_to_clear_operation(ChannelType(channel_type)) - attr_path = self._get_channel_attribute_path(channel_name, ChannelNamespace.USER) - self._execute_operations( - experiment=experiment, - operations=[op(path=alpha_path_utils.parse_path(attr_path))], - ) - - @legacy_with_api_exceptions_handler - def _get_channel_tuples_from_csv(self, experiment, channel_attribute_path): - try: - csv = ( - self.leaderboard_swagger_client.api.getFloatSeriesValuesCSV( - experimentId=experiment.internal_id, - attribute=channel_attribute_path, - ) - .response() - .incoming_response.text - ) - lines = csv.split("\n")[:-1] - return [line.split(",") for line in lines] - except HTTPNotFound: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) - - @legacy_with_api_exceptions_handler - def get_channel_points_csv(self, experiment, channel_internal_id, channel_name): - try: - channel_attr_path = self._get_channel_attribute_path(channel_name, ChannelNamespace.USER) - values = self._get_channel_tuples_from_csv(experiment, channel_attr_path) - step_and_value = [val[0] + "," + val[2] for val in values] - csv = StringIO() - for line in step_and_value: - csv.write(line + "\n") - csv.seek(0) - return csv - except HTTPNotFound: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) - - @legacy_with_api_exceptions_handler - def get_metrics_csv(self, experiment): - metric_channels = [ - channel - for channel in self._get_channels(experiment) - if ( - channel.channelType == ChannelType.NUMERIC.value - and channel.id.startswith(alpha_consts.MONITORING_ATTRIBUTE_SPACE) - ) - ] - data = { - # val[1] + ',' + val[2] is timestamp,value - ch.name: [val[1] + "," + val[2] for val in self._get_channel_tuples_from_csv(experiment, ch.id)] - for ch in metric_channels - } - values_count = max(len(values) for values in data.values()) - csv = StringIO() - csv.write(",".join(["x_{name},y_{name}".format(name=ch.name) for ch in metric_channels])) - csv.write("\n") - for i in range(0, values_count): - csv.write(",".join([data[ch.name][i] if i < len(data[ch.name]) else "," for ch in metric_channels])) - csv.write("\n") - csv.seek(0) - return csv - - @legacy_with_api_exceptions_handler - def get_leaderboard_entries( - self, - project, - entry_types=None, # deprecated - ids=None, - states=None, - owners=None, - tags=None, - min_running_time=None, - ): - if states is not None: - states = [state if state == "running" else "idle" for state in states] - try: - - def get_portion(limit, offset): - return ( - self.leaderboard_swagger_client.api.getLeaderboard( - projectIdentifier=project.full_id, - shortId=ids, - state=states, - owner=owners, - tags=tags, - tagsMode="and", - minRunningTimeSeconds=min_running_time, - sortBy=["sys/id"], - sortFieldType=["string"], - sortDirection=["ascending"], - limit=limit, - offset=offset, - ) - .response() - .result.entries - ) - - return [ - LeaderboardEntry(self._to_leaderboard_entry_dto(e)) for e in self._get_all_items(get_portion, step=100) - ] - except HTTPNotFound: - raise ProjectNotFound(project_identifier=project.full_id) - - def websockets_factory(self, project_id, experiment_id): - base_url = re.sub(r"^http", "ws", self.api_address) + "/api/notifications/v1" - return ReconnectingWebsocketFactory(backend=self, url=base_url + f"/runs/{project_id}/{experiment_id}/signal") - - @staticmethod - def _to_leaderboard_entry_dto(experiment_attributes): - attributes = experiment_attributes.attributes - system_attributes = experiment_attributes.systemAttributes - - def is_channel_namespace(name): - return name.startswith(alpha_consts.LOG_ATTRIBUTE_SPACE) or name.startswith( - alpha_consts.MONITORING_ATTRIBUTE_SPACE - ) - - numeric_channels = [ - HostedAlphaLeaderboardApiClient._float_series_to_channel_last_value_dto(attr) - for attr in attributes - if ( - attr.type == AttributeType.FLOAT_SERIES.value - and is_channel_namespace(attr.name) - and attr.floatSeriesProperties.last is not None - ) - ] - text_channels = [ - HostedAlphaLeaderboardApiClient._string_series_to_channel_last_value_dto(attr) - for attr in attributes - if ( - attr.type == AttributeType.STRING_SERIES.value - and is_channel_namespace(attr.name) - and attr.stringSeriesProperties.last is not None - ) - ] - - return LegacyLeaderboardEntry( - id=experiment_attributes.id, - organizationName=experiment_attributes.organizationName, - projectName=experiment_attributes.projectName, - shortId=system_attributes.shortId.value, - name=system_attributes.name.value, - state="running" if system_attributes.state.value == "running" else "succeeded", - timeOfCreation=system_attributes.creationTime.value, - timeOfCompletion=None, - runningTime=system_attributes.runningTime.value, - owner=system_attributes.owner.value, - size=system_attributes.size.value, - tags=system_attributes.tags.values, - description=system_attributes.description.value, - channelsLastValues=numeric_channels + text_channels, - parameters=[AlphaParameterDTO(attr) for attr in attributes if AlphaParameterDTO.is_valid_attribute(attr)], - properties=[AlphaPropertyDTO(attr) for attr in attributes if AlphaPropertyDTO.is_valid_attribute(attr)], - ) - - @staticmethod - def _float_series_to_channel_last_value_dto(attribute): - return AlphaChannelWithValueDTO( - channelId=attribute.name, - channelName=attribute.name.split("/", 1)[-1], - channelType="numeric", - x=attribute.floatSeriesProperties.lastStep, - y=attribute.floatSeriesProperties.last, - ) - - @staticmethod - def _string_series_to_channel_last_value_dto(attribute): - return AlphaChannelWithValueDTO( - channelId=attribute.name, - channelName=attribute.name.split("/", 1)[-1], - channelType="text", - x=attribute.stringSeriesProperties.lastStep, - y=attribute.stringSeriesProperties.last, - ) - - @staticmethod - def _get_all_items(get_portion, step): - items = [] - - previous_items = None - while previous_items is None or len(previous_items) >= step: - previous_items = get_portion(limit=step, offset=len(items)) - items += previous_items - - return items - - def _upload_raw_data(self, api_method, data, headers, path_params, query_params): - url = self.api_address + api_method.operation.path_name + "?" - - for key, val in path_params.items(): - url = url.replace("{" + key + "}", val) - - for key, val in query_params.items(): - url = url + key + "=" + val + "&" - - session = self.http_client.session - - request = self.authenticator.apply(requests.Request(method="POST", url=url, data=data, headers=headers)) - - return handle_server_raw_response_messages(session.send(session.prepare_request(request))) - - def _get_parameter_with_type(self, parameter): - string_type = "string" - double_type = "double" - if isinstance(parameter, bool): - return (string_type, str(parameter)) - elif isinstance(parameter, float) or isinstance(parameter, int): - if math.isinf(parameter) or math.isnan(parameter): - return (string_type, json.dumps(parameter)) - else: - return (double_type, str(parameter)) - else: - return (string_type, str(parameter)) - - def _convert_to_experiment(self, api_experiment, project): - return Experiment( - backend=project._backend, - project=project, - _id=api_experiment.shortId, - internal_id=api_experiment.id, - ) - - def _download_raw_data(self, api_method, headers, path_params, query_params): - url = self.api_address + api_method.operation.path_name + "?" - - for key, val in path_params.items(): - url = url.replace("{" + key + "}", val) - - for key, val in query_params.items(): - url = url + key + "=" + val + "&" - - session = self.http_client.session - - request = self.authenticator.apply(requests.Request(method="GET", url=url, headers=headers)) - - return handle_server_raw_response_messages(session.send(session.prepare_request(request), stream=True)) diff --git a/src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_backend_api_client.py b/src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_backend_api_client.py deleted file mode 100644 index d9578ee2d..000000000 --- a/src/neptune/legacy/internal/api_clients/hosted_api_clients/hosted_backend_api_client.py +++ /dev/null @@ -1,216 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import os -import platform -import sys - -import click -import urllib3 -from bravado.exception import HTTPNotFound -from bravado.requests_client import RequestsClient -from packaging import version - -from neptune.common.exceptions import STYLES -from neptune.common.oauth import NeptuneAuthenticator -from neptune.common.utils import ( - NoopObject, - update_session_proxies, -) -from neptune.internal.backends.hosted_client import NeptuneResponseAdapter -from neptune.legacy.api_exceptions import ( - ProjectNotFound, - WorkspaceNotFound, -) -from neptune.legacy.backend import ( - BackendApiClient, - LeaderboardApiClient, -) -from neptune.legacy.exceptions import UnsupportedClientVersion -from neptune.legacy.internal.api_clients.credentials import Credentials -from neptune.legacy.internal.api_clients.hosted_api_clients.hosted_alpha_leaderboard_api_client import ( - HostedAlphaLeaderboardApiClient, -) -from neptune.legacy.internal.api_clients.hosted_api_clients.mixins import HostedNeptuneMixin -from neptune.legacy.internal.api_clients.hosted_api_clients.utils import legacy_with_api_exceptions_handler -from neptune.legacy.projects import Project - -_logger = logging.getLogger(__name__) - - -class HostedNeptuneBackendApiClient(HostedNeptuneMixin, BackendApiClient): - @legacy_with_api_exceptions_handler - def __init__(self, api_token=None, proxies=None): - self._old_leaderboard_client = None - self._new_leaderboard_client = None - self._api_token = api_token - self._proxies = proxies - - # This is not a top-level import because of circular dependencies - from neptune import __version__ - - self.client_lib_version = __version__ - - self.credentials = Credentials(api_token) - - ssl_verify = True - if os.getenv("NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE"): - urllib3.disable_warnings() - ssl_verify = False - - self._http_client = RequestsClient(ssl_verify=ssl_verify, response_adapter_class=NeptuneResponseAdapter) - # for session re-creation we need to keep an authenticator-free version of http client - self._http_client_for_token = RequestsClient( - ssl_verify=ssl_verify, response_adapter_class=NeptuneResponseAdapter - ) - - user_agent = "neptune-client/{lib_version} ({system}, python {python_version})".format( - lib_version=self.client_lib_version, - system=platform.platform(), - python_version=platform.python_version(), - ) - self.http_client.session.headers.update({"User-Agent": user_agent}) - self._http_client_for_token.session.headers.update({"User-Agent": user_agent}) - - update_session_proxies(self.http_client.session, proxies) - update_session_proxies(self._http_client_for_token.session, proxies) - - config_api_url = self.credentials.api_url_opt or self.credentials.token_origin_address - # We don't need to be able to resolve Neptune host if we use proxy - if proxies is None: - self._verify_host_resolution(config_api_url, self.credentials.token_origin_address) - - # this backend client is used only for initial configuration and session re-creation - self.backend_client = self._get_swagger_client( - "{}/api/backend/swagger.json".format(config_api_url), - self._http_client_for_token, - ) - self._client_config = self._create_client_config( - api_token=self.credentials.api_token, backend_client=self.backend_client - ) - - self._verify_version() - - self.backend_swagger_client = self._get_swagger_client( - "{}/api/backend/swagger.json".format(self._client_config.api_url), - self.http_client, - ) - - self.authenticator = self._create_authenticator( - api_token=self.credentials.api_token, - ssl_verify=ssl_verify, - proxies=proxies, - backend_client=self.backend_client, - ) - self.http_client.authenticator = self.authenticator - - if sys.version_info >= (3, 7): - try: - os.register_at_fork(after_in_child=self._handle_fork_in_child) - except AttributeError: - pass - - def _handle_fork_in_child(self): - self.backend_swagger_client = NoopObject() - - @property - def api_address(self): - return self._client_config.api_url - - @property - def http_client(self): - return self._http_client - - @property - def display_address(self): - return self._client_config.display_url - - @property - def proxies(self): - return self._proxies - - @legacy_with_api_exceptions_handler - def get_project(self, project_qualified_name): - try: - response = self.backend_swagger_client.api.getProject(projectIdentifier=project_qualified_name).response() - warning = response.metadata.headers.get("X-Server-Warning") - if warning: - click.echo("{warning}{content}{end}".format(content=warning, **STYLES)) - project = response.result - - return Project( - backend=self.create_leaderboard_backend(project=project), - internal_id=project.id, - namespace=project.organizationName, - name=project.name, - ) - except HTTPNotFound: - raise ProjectNotFound(project_qualified_name) - - @legacy_with_api_exceptions_handler - def get_projects(self, namespace): - try: - r = self.backend_swagger_client.api.listProjects(organizationIdentifier=namespace).response() - return r.result.entries - except HTTPNotFound: - raise WorkspaceNotFound(namespace_name=namespace) - - def create_leaderboard_backend(self, project) -> LeaderboardApiClient: - return self.get_new_leaderboard_client() - - def get_new_leaderboard_client(self) -> HostedAlphaLeaderboardApiClient: - if self._new_leaderboard_client is None: - self._new_leaderboard_client = HostedAlphaLeaderboardApiClient(backend_api_client=self) - return self._new_leaderboard_client - - @legacy_with_api_exceptions_handler - def _create_authenticator(self, api_token, ssl_verify, proxies, backend_client): - return NeptuneAuthenticator(api_token, backend_client, ssl_verify, proxies) - - def _verify_version(self): - parsed_version = version.parse(self.client_lib_version) - - if self._client_config.min_compatible_version and self._client_config.min_compatible_version > parsed_version: - click.echo( - "ERROR: Minimal supported client version is {} (installed: {}). Please upgrade neptune-client".format( - self._client_config.min_compatible_version, self.client_lib_version - ), - sys.stderr, - ) - raise UnsupportedClientVersion( - self.client_lib_version, - self._client_config.min_compatible_version, - self._client_config.max_compatible_version, - ) - if self._client_config.max_compatible_version and self._client_config.max_compatible_version < parsed_version: - click.echo( - "ERROR: Maximal supported client version is {} (installed: {}). Please downgrade neptune-client".format( - self._client_config.max_compatible_version, self.client_lib_version - ), - sys.stderr, - ) - raise UnsupportedClientVersion( - self.client_lib_version, - self._client_config.min_compatible_version, - self._client_config.max_compatible_version, - ) - if self._client_config.min_recommended_version and self._client_config.min_recommended_version > parsed_version: - click.echo( - "WARNING: We recommend an upgrade to a new version of neptune-client - {} (installed - {}).".format( - self._client_config.min_recommended_version, self.client_lib_version - ), - sys.stderr, - ) diff --git a/src/neptune/legacy/internal/api_clients/hosted_api_clients/mixins.py b/src/neptune/legacy/internal/api_clients/hosted_api_clients/mixins.py deleted file mode 100644 index 7f8f1545a..000000000 --- a/src/neptune/legacy/internal/api_clients/hosted_api_clients/mixins.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import socket -import sys - -import click -from bravado.client import SwaggerClient -from bravado_core.formatter import SwaggerFormat -from packaging import version -from six.moves import urllib - -from neptune.legacy.exceptions import ( - CannotResolveHostname, - DeprecatedApiToken, - UnsupportedClientVersion, -) -from neptune.legacy.internal.api_clients.client_config import ( - ClientConfig, - MultipartConfig, -) -from neptune.legacy.internal.api_clients.hosted_api_clients.utils import legacy_with_api_exceptions_handler - -_logger = logging.getLogger(__name__) - -uuid_format = SwaggerFormat( - format="uuid", - to_python=lambda x: x, - to_wire=lambda x: x, - validate=lambda x: None, - description="", -) - - -class HostedNeptuneMixin: - """Mixin containing operation common for both backend and leaderboard api clients""" - - @legacy_with_api_exceptions_handler - def _get_swagger_client(self, url, http_client): - return SwaggerClient.from_url( - url, - config=dict( - validate_swagger_spec=False, - validate_requests=False, - validate_responses=False, - formats=[uuid_format], - ), - http_client=http_client, - ) - - @staticmethod - def _get_client_config_args(api_token): - return dict(X_Neptune_Api_Token=api_token) - - @legacy_with_api_exceptions_handler - def _create_client_config(self, api_token, backend_client): - client_config_args = self._get_client_config_args(api_token) - config = backend_client.api.getClientConfig(**client_config_args).response().result - - if hasattr(config, "pyLibVersions"): - min_recommended = getattr(config.pyLibVersions, "minRecommendedVersion", None) - min_compatible = getattr(config.pyLibVersions, "minCompatibleVersion", None) - max_compatible = getattr(config.pyLibVersions, "maxCompatibleVersion", None) - else: - click.echo( - "ERROR: This client version is not supported by your Neptune instance. Please contant Neptune support.", - sys.stderr, - ) - raise UnsupportedClientVersion(self.client_lib_version, None, "0.4.111") - - multipart_upload_config_obj = getattr(config, "multiPartUpload", None) - has_multipart_upload = getattr(multipart_upload_config_obj, "enabled", False) - if not has_multipart_upload: - multipart_upload_config = None - else: - min_chunk_size = getattr(multipart_upload_config_obj, "minChunkSize") - max_chunk_size = getattr(multipart_upload_config_obj, "maxChunkSize") - max_chunk_count = getattr(multipart_upload_config_obj, "maxChunkCount") - max_single_part_size = getattr(multipart_upload_config_obj, "maxSinglePartSize") - multipart_upload_config = MultipartConfig( - min_chunk_size, max_chunk_size, max_chunk_count, max_single_part_size - ) - - return ClientConfig( - api_url=config.apiUrl, - display_url=config.applicationUrl, - min_recommended_version=version.parse(min_recommended) if min_recommended else None, - min_compatible_version=version.parse(min_compatible) if min_compatible else None, - max_compatible_version=version.parse(max_compatible) if max_compatible else None, - multipart_config=multipart_upload_config, - ) - - def _verify_host_resolution(self, api_url, app_url): - host = urllib.parse.urlparse(api_url).netloc.split(":")[0] - try: - socket.gethostbyname(host) - except socket.gaierror: - if self.credentials.api_url_opt is None: - raise DeprecatedApiToken(urllib.parse.urlparse(app_url).netloc) - else: - raise CannotResolveHostname(host) diff --git a/src/neptune/legacy/internal/api_clients/hosted_api_clients/utils.py b/src/neptune/legacy/internal/api_clients/hosted_api_clients/utils.py deleted file mode 100644 index b59a6d188..000000000 --- a/src/neptune/legacy/internal/api_clients/hosted_api_clients/utils.py +++ /dev/null @@ -1,120 +0,0 @@ -# -# Copyright (c) 2023, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import time - -import requests -from bravado.exception import ( - BravadoConnectionError, - BravadoTimeoutError, - HTTPBadGateway, - HTTPForbidden, - HTTPGatewayTimeout, - HTTPInternalServerError, - HTTPRequestTimeout, - HTTPServiceUnavailable, - HTTPTooManyRequests, - HTTPUnauthorized, -) -from urllib3.exceptions import NewConnectionError - -from neptune.common.backends.utils import get_retry_from_headers_or_default -from neptune.legacy.api_exceptions import ( - ConnectionLost, - Forbidden, - NeptuneSSLVerificationError, - ServerError, - Unauthorized, -) - -_logger = logging.getLogger(__name__) - - -def legacy_with_api_exceptions_handler(func): - def wrapper(*args, **kwargs): - retries = 11 - retry = 0 - while retry < retries: - try: - return func(*args, **kwargs) - except requests.exceptions.SSLError: - raise NeptuneSSLVerificationError() - except HTTPServiceUnavailable: - if retry >= 6: - _logger.warning("Experiencing connection interruptions. Reestablishing communication with Neptune.") - time.sleep(2**retry) - retry += 1 - continue - except ( - BravadoConnectionError, - BravadoTimeoutError, - requests.exceptions.ConnectionError, - requests.exceptions.Timeout, - HTTPRequestTimeout, - HTTPGatewayTimeout, - HTTPBadGateway, - HTTPInternalServerError, - NewConnectionError, - ): - if retry >= 6: - _logger.warning("Experiencing connection interruptions. Reestablishing communication with Neptune.") - time.sleep(2**retry) - retry += 1 - continue - except HTTPTooManyRequests as e: - wait_time = get_retry_from_headers_or_default(e.response.headers, retry) - time.sleep(wait_time) - retry += 1 - continue - except HTTPUnauthorized: - raise Unauthorized() - except HTTPForbidden: - raise Forbidden() - except requests.exceptions.RequestException as e: - if e.response is None: - raise - status_code = e.response.status_code - if status_code in ( - HTTPRequestTimeout.status_code, - HTTPBadGateway.status_code, - HTTPServiceUnavailable.status_code, - HTTPGatewayTimeout.status_code, - HTTPInternalServerError.status_code, - ): - if retry >= 6: - _logger.warning( - "Experiencing connection interruptions. Reestablishing communication with Neptune." - ) - time.sleep(2**retry) - retry += 1 - continue - elif status_code == HTTPTooManyRequests.status_code: - wait_time = get_retry_from_headers_or_default(e.response.headers, retry) - time.sleep(wait_time) - retry += 1 - continue - elif status_code >= HTTPInternalServerError.status_code: - raise ServerError() - elif status_code == HTTPUnauthorized.status_code: - raise Unauthorized() - elif status_code == HTTPForbidden.status_code: - raise Forbidden() - else: - raise - raise ConnectionLost() - - return wrapper diff --git a/src/neptune/legacy/internal/api_clients/offline_backend.py b/src/neptune/legacy/internal/api_clients/offline_backend.py deleted file mode 100644 index 76510da43..000000000 --- a/src/neptune/legacy/internal/api_clients/offline_backend.py +++ /dev/null @@ -1,184 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -from io import StringIO - -from neptune.common.utils import NoopObject -from neptune.legacy.backend import ( - BackendApiClient, - LeaderboardApiClient, -) - -_logger = logging.getLogger(__name__) - - -class OfflineBackendApiClient(BackendApiClient): - def __init__(self): - _logger.warning("Neptune is running in offline mode. No data is being logged to Neptune.") - _logger.warning("Disable offline mode to log your experiments.") - - @property - def api_address(self): - return "OFFLINE" - - @property - def display_address(self): - return "OFFLINE" - - @property - def proxies(self): - return None - - def get_project(self, project_qualified_name): - return NoopObject() - - def get_projects(self, namespace): - return [] - - def create_leaderboard_backend(self, project) -> "OfflineLeaderboardApiClient": - return OfflineLeaderboardApiClient() - - -class OfflineLeaderboardApiClient(LeaderboardApiClient): - @property - def api_address(self): - return "OFFLINE" - - @property - def display_address(self): - return "OFFLINE" - - @property - def proxies(self): - return None - - def get_project_members(self, project_identifier): - return [] - - def get_leaderboard_entries( - self, - project, - entry_types=None, - ids=None, - states=None, - owners=None, - tags=None, - min_running_time=None, - ): - return [] - - def get_channel_points_csv(self, experiment, channel_internal_id, channel_name): - return StringIO() - - def get_metrics_csv(self, experiment): - return StringIO() - - def create_experiment( - self, - project, - name, - description, - params, - properties, - tags, - abortable, - monitored, - git_info, - hostname, - entrypoint, - notebook_id, - checkpoint_id, - ): - return NoopObject() - - def upload_source_code(self, experiment, source_target_pairs): - pass - - def get_notebook(self, project, notebook_id): - return NoopObject() - - def get_last_checkpoint(self, project, notebook_id): - return NoopObject() - - def create_notebook(self, project): - return NoopObject() - - def create_checkpoint(self, notebook_id, jupyter_path, _file=None): - pass - - def get_experiment(self, experiment_id): - return NoopObject() - - def set_property(self, experiment, key, value): - pass - - def remove_property(self, experiment, key): - pass - - def update_tags(self, experiment, tags_to_add, tags_to_delete): - pass - - def create_channel(self, experiment, name, channel_type): - return NoopObject() - - def reset_channel(self, experiment, channel_id, channel_name, channel_type): - pass - - def get_channels(self, experiment): - return {} - - def create_system_channel(self, experiment, name, channel_type): - return NoopObject() - - def get_system_channels(self, experiment): - return {} - - def send_channels_values(self, experiment, channels_with_values): - pass - - def mark_failed(self, experiment, traceback): - pass - - def ping_experiment(self, experiment): - pass - - def create_hardware_metric(self, experiment, metric): - return NoopObject() - - def send_hardware_metric_reports(self, experiment, metrics, metric_reports): - pass - - def log_artifact(self, experiment, artifact, destination=None): - pass - - def delete_artifacts(self, experiment, path): - pass - - def download_data(self, experiment, path, destination): - pass - - def download_sources(self, experiment, path=None, destination_dir=None): - pass - - def download_artifacts(self, experiment, path=None, destination_dir=None): - pass - - def download_artifact(self, experiment, path=None, destination_dir=None): - pass - - -# define deprecated OfflineBackend class -OfflineBackend = OfflineBackendApiClient diff --git a/src/neptune/legacy/internal/backends/__init__.py b/src/neptune/legacy/internal/backends/__init__.py deleted file mode 100644 index d71b3273e..000000000 --- a/src/neptune/legacy/internal/backends/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/backends/hosted_neptune_backend.py b/src/neptune/legacy/internal/backends/hosted_neptune_backend.py deleted file mode 100644 index f4fbbd5db..000000000 --- a/src/neptune/legacy/internal/backends/hosted_neptune_backend.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from neptune.legacy.internal.api_clients import HostedNeptuneBackendApiClient - -# define deprecated HostedNeptuneBackend class -HostedNeptuneBackend = HostedNeptuneBackendApiClient diff --git a/src/neptune/legacy/internal/channels/__init__.py b/src/neptune/legacy/internal/channels/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/channels/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/channels/channels.py b/src/neptune/legacy/internal/channels/channels.py deleted file mode 100644 index 3209efd2b..000000000 --- a/src/neptune/legacy/internal/channels/channels.py +++ /dev/null @@ -1,132 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import time -from collections import namedtuple -from enum import Enum -from typing import List - -from neptune.legacy.exceptions import NeptuneException - -ChannelNameWithTypeAndNamespace = namedtuple( - "ChannelNameWithType", - ["channel_id", "channel_name", "channel_type", "channel_namespace"], -) - - -class ChannelType(Enum): - TEXT = "text" - NUMERIC = "numeric" - IMAGE = "image" - - -class ChannelValueType(Enum): - TEXT_VALUE = "text_value" - NUMERIC_VALUE = "numeric_value" - IMAGE_VALUE = "image_value" - - -class ChannelNamespace(Enum): - USER = "user" - SYSTEM = "system" - - -class ChannelValue(object): - def __init__(self, x, y, ts): - self._x = x - self._y = y - if ts is None: - ts = time.time() - self._ts = ts - - @property - def ts(self): - return self._ts - - @property - def x(self): - return self._x - - @property - def y(self): - return self._y - - @property - def value(self): - return self.y.get(self.value_type.value) - - @property - def value_type(self) -> ChannelValueType: - """We expect that exactly one of `y` values is not None, and according to that we try to determine value type""" - unique_channel_value_types = set( - [ch_value_type for ch_value_type in ChannelValueType if self.y.get(ch_value_type.value) is not None] - ) - if len(unique_channel_value_types) > 1: - raise NeptuneException(f"There are mixed value types in {self}") - if not unique_channel_value_types: - raise NeptuneException(f"Can't determine type of {self}") - - return next(iter(unique_channel_value_types)) - - def __str__(self): - return "ChannelValue(x={},y={},ts={})".format(self.x, self.y, self.ts) - - def __repr__(self): - return str(self) - - def __eq__(self, o): - return self.__dict__ == o.__dict__ - - -class ChannelIdWithValues: - def __init__(self, channel_id, channel_name, channel_type, channel_namespace, channel_values): - self._channel_id = channel_id - self._channel_name = channel_name - self._channel_type = channel_type - self._channel_namespace = channel_namespace - self._channel_values = channel_values - - @property - def channel_id(self) -> str: - return self._channel_id - - @property - def channel_name(self) -> str: - return self._channel_name - - @property - def channel_values(self) -> List[ChannelValue]: - return self._channel_values - - @property - def channel_type(self) -> ChannelValueType: - if self._channel_type == ChannelType.NUMERIC.value: - return ChannelValueType.NUMERIC_VALUE - elif self._channel_type == ChannelType.TEXT.value: - return ChannelValueType.TEXT_VALUE - elif self._channel_type == ChannelType.IMAGE.value: - return ChannelValueType.IMAGE_VALUE - else: - raise NeptuneException(f"Unknown channel type: {self._channel_type}") - - @property - def channel_namespace(self) -> ChannelNamespace: - return self._channel_namespace - - def __eq__(self, other): - return self.channel_id == other.channel_id and self.channel_values == other.channel_values - - def __gt__(self, other): - return hash(self.channel_id) < hash(other.channel_id) diff --git a/src/neptune/legacy/internal/channels/channels_values_sender.py b/src/neptune/legacy/internal/channels/channels_values_sender.py deleted file mode 100644 index 0daeb3f28..000000000 --- a/src/neptune/legacy/internal/channels/channels_values_sender.py +++ /dev/null @@ -1,201 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import threading -import time -from collections import namedtuple -from itertools import groupby -from queue import ( - Empty, - Queue, -) - -from bravado.exception import HTTPUnprocessableEntity - -from neptune.legacy.exceptions import NeptuneException -from neptune.legacy.internal.channels.channels import ( - ChannelIdWithValues, - ChannelNamespace, - ChannelType, - ChannelValue, -) -from neptune.legacy.internal.threads.neptune_thread import NeptuneThread - -_logger = logging.getLogger(__name__) - - -class ChannelsValuesSender(object): - _QUEUED_CHANNEL_VALUE = namedtuple( - "QueuedChannelValue", - [ - "channel_id", - "channel_name", - "channel_type", - "channel_value", - "channel_namespace", - ], - ) - - __LOCK = threading.RLock() - - def __init__(self, experiment): - self._experiment = experiment - self._values_queue = None - self._sending_thread = None - self._user_channel_name_to_id_map = dict() - self._system_channel_name_to_id_map = dict() - - def send( - self, - channel_name, - channel_type, - channel_value, - channel_namespace=ChannelNamespace.USER, - ): - # Before taking the lock here, we need to check if the sending thread is not running yet. - # Otherwise, the sending thread could call send() while being join()-ed, which would result - # in a deadlock. - if not self._is_running(): - with self.__LOCK: - if not self._is_running(): - self._start() - - if channel_namespace == ChannelNamespace.USER: - namespaced_channel_map = self._user_channel_name_to_id_map - else: - namespaced_channel_map = self._system_channel_name_to_id_map - - if channel_name in namespaced_channel_map: - channel_id = namespaced_channel_map[channel_name] - else: - response = self._experiment._create_channel(channel_name, channel_type, channel_namespace) - channel_id = response.id - namespaced_channel_map[channel_name] = channel_id - - self._values_queue.put( - self._QUEUED_CHANNEL_VALUE( - channel_id=channel_id, - channel_name=channel_name, - channel_type=channel_type, - channel_value=channel_value, - channel_namespace=channel_namespace, - ) - ) - - def join(self): - with self.__LOCK: - if self._is_running(): - self._sending_thread.interrupt() - self._sending_thread.join() - self._sending_thread = None - self._values_queue = None - - def _is_running(self): - return self._values_queue is not None and self._sending_thread is not None and self._sending_thread.is_alive() - - def _start(self): - self._values_queue = Queue() - self._sending_thread = ChannelsValuesSendingThread(self._experiment, self._values_queue) - self._sending_thread.start() - - -class ChannelsValuesSendingThread(NeptuneThread): - _SLEEP_TIME = 5 - _MAX_VALUES_BATCH_LENGTH = 100 - _MAX_IMAGE_VALUES_BATCH_SIZE = 10485760 # 10 MB - - def __init__(self, experiment, values_queue): - super(ChannelsValuesSendingThread, self).__init__(is_daemon=False) - self._values_queue = values_queue - self._experiment = experiment - self._sleep_time = self._SLEEP_TIME - self._values_batch = [] - - def run(self): - while self.should_continue_running() or not self._values_queue.empty(): - try: - sleep_start = time.time() - self._values_batch.append(self._values_queue.get(timeout=max(self._sleep_time, 0))) - self._values_queue.task_done() - self._sleep_time -= time.time() - sleep_start - except Empty: - self._sleep_time = 0 - - image_values_batch_size = sum( - [ - len(v.channel_value.y["image_value"]["data"] or []) - for v in self._values_batch - if v.channel_type == ChannelType.IMAGE.value - ] - ) - if ( - self._sleep_time <= 0 - or len(self._values_batch) >= self._MAX_VALUES_BATCH_LENGTH - or image_values_batch_size >= self._MAX_IMAGE_VALUES_BATCH_SIZE - ): - self._process_batch() - - self._process_batch() - - def _process_batch(self): - send_start = time.time() - if self._values_batch: - try: - self._send_values(self._values_batch) - self._values_batch = [] - except (NeptuneException, IOError): - _logger.exception("Failed to send channel value.") - self._sleep_time = self._SLEEP_TIME - (time.time() - send_start) - - def _send_values(self, queued_channels_values): - def get_channel_metadata(value): - return ( - value.channel_id, - value.channel_name, - value.channel_type, - value.channel_namespace, - ) - - queued_grouped_by_channel = { - channel_metadata: list(values) - for channel_metadata, values in groupby( - sorted(queued_channels_values, key=get_channel_metadata), - get_channel_metadata, - ) - } - channels_with_values = [] - for channel_metadata in queued_grouped_by_channel: - channel_values = [] - for queued_value in queued_grouped_by_channel[channel_metadata]: - channel_values.append( - ChannelValue( - ts=queued_value.channel_value.ts, - x=queued_value.channel_value.x, - y=queued_value.channel_value.y, - ) - ) - channels_with_values.append(ChannelIdWithValues(*channel_metadata, channel_values)) - - try: - self._experiment._send_channels_values(channels_with_values) - except HTTPUnprocessableEntity as e: - message = "Maximum storage limit reached" - try: - message = e.response.json()["message"] - finally: - _logger.warning("Failed to send channel value: %s", message) - except (NeptuneException, IOError): - _logger.exception("Failed to send channel value.") diff --git a/src/neptune/legacy/internal/execution/__init__.py b/src/neptune/legacy/internal/execution/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/execution/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/execution/execution_context.py b/src/neptune/legacy/internal/execution/execution_context.py deleted file mode 100644 index 0c3786ebe..000000000 --- a/src/neptune/legacy/internal/execution/execution_context.py +++ /dev/null @@ -1,173 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import os -import sys -import time -import traceback -from logging import StreamHandler - -from neptune.common.hardware.gauges.gauge_mode import GaugeMode -from neptune.common.hardware.metrics.service.metric_service_factory import MetricServiceFactory -from neptune.common.utils import ( - in_docker, - is_ipython, - is_notebook, -) -from neptune.legacy.internal.abort import ( - CustomAbortImpl, - DefaultAbortImpl, -) -from neptune.legacy.internal.channels.channels import ChannelNamespace -from neptune.legacy.internal.streams.channel_writer import ChannelWriter -from neptune.legacy.internal.streams.stdstream_uploader import ( - StdErrWithUpload, - StdOutWithUpload, -) -from neptune.legacy.internal.threads.aborting_thread import AbortingThread -from neptune.legacy.internal.threads.hardware_metric_reporting_thread import HardwareMetricReportingThread -from neptune.legacy.internal.threads.ping_thread import PingThread - -_logger = logging.getLogger(__name__) - - -class ExecutionContext(object): - def __init__(self, backend, experiment): - self._backend = backend - self._experiment = experiment - self._ping_thread = None - self._hardware_metric_thread = None - self._aborting_thread = None - self._logger = None - self._logger_handler = None - self._stdout_uploader = None - self._stderr_uploader = None - self._uncaught_exception_handler = sys.__excepthook__ - - self._previous_uncaught_exception_handler = None - - def start( - self, - abort_callback=None, - logger=None, - upload_stdout=True, - upload_stderr=True, - send_hardware_metrics=True, - run_monitoring_thread=True, - handle_uncaught_exceptions=True, - ): - - if handle_uncaught_exceptions: - self._set_uncaught_exception_handler() - - if logger: - channel = self._experiment._get_channel("logger", "text", ChannelNamespace.SYSTEM) - channel_writer = ChannelWriter(self._experiment, channel.name, ChannelNamespace.SYSTEM) - self._logger_handler = StreamHandler(channel_writer) - self._logger = logger - logger.addHandler(self._logger_handler) - - if upload_stdout and not is_notebook(): - self._stdout_uploader = StdOutWithUpload(self._experiment) - - if upload_stderr and not is_notebook(): - self._stderr_uploader = StdErrWithUpload(self._experiment) - - abortable = abort_callback is not None or DefaultAbortImpl.requirements_installed() - if abortable: - self._run_aborting_thread(abort_callback) - else: - _logger.warning("psutil is not installed. You will not be able to abort this experiment from the UI.") - - if run_monitoring_thread: - self._run_monitoring_thread() - - if send_hardware_metrics: - self._run_hardware_metrics_reporting_thread() - - def stop(self): - if self._ping_thread: - self._ping_thread.interrupt() - self._ping_thread = None - - if self._hardware_metric_thread: - self._hardware_metric_thread.interrupt() - self._hardware_metric_thread = None - - if self._aborting_thread: - self._aborting_thread.shutdown() - self._aborting_thread = None - - if self._stdout_uploader: - self._stdout_uploader.close() - - if self._stderr_uploader: - self._stderr_uploader.close() - - if self._logger and self._logger_handler: - self._logger.removeHandler(self._logger_handler) - - sys.excepthook = self._previous_uncaught_exception_handler - - def _set_uncaught_exception_handler(self): - def exception_handler(exc_type, exc_val, exc_tb): - self._experiment.stop("\n".join(traceback.format_tb(exc_tb)) + "\n" + repr(exc_val)) - - sys.__excepthook__(exc_type, exc_val, exc_tb) - - self._uncaught_exception_handler = exception_handler - - self._previous_uncaught_exception_handler = sys.excepthook - sys.excepthook = exception_handler - - def _run_aborting_thread(self, abort_callback): - if abort_callback is not None: - abort_impl = CustomAbortImpl(abort_callback) - elif not is_ipython(): - abort_impl = DefaultAbortImpl(pid=os.getpid()) - else: - return - - websocket_factory = self._backend.websockets_factory( - project_id=self._experiment._project.internal_id, - experiment_id=self._experiment.internal_id, - ) - if not websocket_factory: - return - - self._aborting_thread = AbortingThread( - websocket_factory=websocket_factory, - abort_impl=abort_impl, - experiment=self._experiment, - ) - self._aborting_thread.start() - - def _run_monitoring_thread(self): - self._ping_thread = PingThread(backend=self._backend, experiment=self._experiment) - self._ping_thread.start() - - def _run_hardware_metrics_reporting_thread(self): - gauge_mode = GaugeMode.CGROUP if in_docker() else GaugeMode.SYSTEM - metric_service = MetricServiceFactory(self._backend, os.environ).create( - gauge_mode=gauge_mode, - experiment=self._experiment, - reference_timestamp=time.time(), - ) - - self._hardware_metric_thread = HardwareMetricReportingThread( - metric_service=metric_service, metric_sending_interval_seconds=10 - ) - self._hardware_metric_thread.start() diff --git a/src/neptune/legacy/internal/experiments/__init__.py b/src/neptune/legacy/internal/experiments/__init__.py deleted file mode 100644 index d71b3273e..000000000 --- a/src/neptune/legacy/internal/experiments/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/notebooks/__init__.py b/src/neptune/legacy/internal/notebooks/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/notebooks/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/notebooks/comm.py b/src/neptune/legacy/internal/notebooks/comm.py deleted file mode 100644 index 7ed91c611..000000000 --- a/src/neptune/legacy/internal/notebooks/comm.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging - -_logger = logging.getLogger(__name__) - - -class MessageType(object): - CHECKPOINT_CREATED = "CHECKPOINT_CREATED" - - -def send_checkpoint_created(notebook_id, notebook_path, checkpoint_id): - """Send checkpoint created message. - - Args: - notebook_id (:obj:`str`): The notebook's id. - notebook_path (:obj:`str`): The notebook's path. - checkpoint_id (:obj:`str`): The checkpoint's path. - - - Raises: - `ImportError`: If ipykernel is not available. - """ - neptune_comm = _get_comm() - neptune_comm.send( - data=dict( - message_type=MessageType.CHECKPOINT_CREATED, - data=dict( - checkpoint_id=checkpoint_id, - notebook_id=notebook_id, - notebook_path=notebook_path, - ), - ) - ) - - -def _get_comm(): - from ipykernel.comm import Comm - - return Comm(target_name="neptune_comm") diff --git a/src/neptune/legacy/internal/notebooks/notebooks.py b/src/neptune/legacy/internal/notebooks/notebooks.py deleted file mode 100644 index 4b23b780f..000000000 --- a/src/neptune/legacy/internal/notebooks/notebooks.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import threading - -from neptune.common.utils import is_ipython -from neptune.legacy.internal.notebooks.comm import send_checkpoint_created - -_logger = logging.getLogger(__name__) - -_checkpoints_lock = threading.Lock() -_checkpoints = dict() - - -def create_checkpoint(backend, notebook_id, notebook_path): - if is_ipython(): - import IPython - - ipython = IPython.core.getipython.get_ipython() - execution_count = -1 - if ipython.kernel is not None: - execution_count = ipython.kernel.execution_count - with _checkpoints_lock: - - if execution_count in _checkpoints: - return _checkpoints[execution_count] - - checkpoint = backend.create_checkpoint(notebook_id, notebook_path) - if ipython is not None and ipython.kernel is not None: - send_checkpoint_created( - notebook_id=notebook_id, - notebook_path=notebook_path, - checkpoint_id=checkpoint.id, - ) - _checkpoints[execution_count] = checkpoint - return checkpoint diff --git a/src/neptune/legacy/internal/streams/__init__.py b/src/neptune/legacy/internal/streams/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/streams/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/streams/channel_writer.py b/src/neptune/legacy/internal/streams/channel_writer.py deleted file mode 100644 index bdaf4a80b..000000000 --- a/src/neptune/legacy/internal/streams/channel_writer.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import unicode_literals - -import re -from datetime import datetime - -from neptune.legacy.internal.channels.channels import ( - ChannelNamespace, - ChannelType, - ChannelValue, -) - - -class ChannelWriter(object): - __SPLIT_PATTERN = re.compile(r"[\n\r]{1,2}") - - def __init__(self, experiment, channel_name, channel_namespace=ChannelNamespace.USER): - self._experiment = experiment - self._channel_name = channel_name - self._channel_namespace = channel_namespace - self._data = None - self._x_offset = TimeOffsetGenerator(self._experiment.get_system_properties()["created"]) - - def write(self, data): - if self._data is None: - self._data = data - else: - self._data += data - lines = self.__SPLIT_PATTERN.split(self._data) - for line in lines[:-1]: - value = ChannelValue(x=self._x_offset.next(), y=dict(text_value=str(line)), ts=None) - self._experiment._channels_values_sender.send( - channel_name=self._channel_name, - channel_type=ChannelType.TEXT.value, - channel_value=value, - channel_namespace=self._channel_namespace, - ) - - self._data = lines[-1] - - -class TimeOffsetGenerator(object): - def __init__(self, start): - self._start = start - self._previous_millis_from_start = None - - def next(self): - """ - This method returns the number of milliseconds from start. - It returns a float, with microsecond granularity. - - Since on Windows, datetime.now() has actually a millisecond granularity, - we remember the last returned value and in case of a collision, we add a microsecond. - """ - millis_from_start = (datetime.now(tz=self._start.tzinfo) - self._start).total_seconds() * 1000 - if self._previous_millis_from_start is not None and self._previous_millis_from_start >= millis_from_start: - microsecond = 0.001 - self._previous_millis_from_start = self._previous_millis_from_start + microsecond - else: - self._previous_millis_from_start = millis_from_start - - return self._previous_millis_from_start diff --git a/src/neptune/legacy/internal/streams/stdstream_uploader.py b/src/neptune/legacy/internal/streams/stdstream_uploader.py deleted file mode 100644 index 57e0ee13f..000000000 --- a/src/neptune/legacy/internal/streams/stdstream_uploader.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys - -from neptune.legacy.internal.channels.channels import ChannelNamespace -from neptune.legacy.internal.streams.channel_writer import ChannelWriter - - -class StdStreamWithUpload(object): - def __init__(self, experiment, channel_name, stream): - self._channel = experiment._get_channel(channel_name, "text", ChannelNamespace.SYSTEM) - self._channel_writer = ChannelWriter(experiment, channel_name, ChannelNamespace.SYSTEM) - self._stream = stream - - def write(self, data): - self._stream.write(data) - try: - self._channel_writer.write(data) - except: # noqa: E722 - pass - - def isatty(self): - return hasattr(self._stream, "isatty") and self._stream.isatty() - - def flush(self): - self._stream.flush() - - def fileno(self): - return self._stream.fileno() - - -class StdOutWithUpload(StdStreamWithUpload): - def __init__(self, experiment): - super(StdOutWithUpload, self).__init__(experiment, "stdout", sys.__stdout__) - sys.stdout = self - - def close(self): - sys.stdout = sys.__stdout__ - - -class StdErrWithUpload(StdStreamWithUpload): - def __init__(self, experiment): - super(StdErrWithUpload, self).__init__(experiment, "stderr", sys.__stderr__) - sys.stderr = self - - def close(self): - sys.stderr = sys.__stderr__ diff --git a/src/neptune/legacy/internal/threads/__init__.py b/src/neptune/legacy/internal/threads/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/threads/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/threads/aborting_thread.py b/src/neptune/legacy/internal/threads/aborting_thread.py deleted file mode 100644 index 0520388f5..000000000 --- a/src/neptune/legacy/internal/threads/aborting_thread.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import threading - -from websocket import WebSocketConnectionClosedException - -from neptune.legacy.internal.threads.neptune_thread import NeptuneThread -from neptune.legacy.internal.websockets.message import MessageType -from neptune.legacy.internal.websockets.websocket_message_processor import WebsocketMessageProcessor - - -class AbortingThread(NeptuneThread): - def __init__(self, websocket_factory, abort_impl, experiment): - super(AbortingThread, self).__init__(is_daemon=True) - self._abort_message_processor = AbortMessageProcessor(abort_impl, experiment) - self._ws_client = websocket_factory.create(shutdown_condition=threading.Event()) - - def run(self): - try: - while self.should_continue_running(): - raw_message = self._ws_client.recv() - self._abort_message_processor.run(raw_message) - except WebSocketConnectionClosedException: - pass - - def shutdown(self): - self.interrupt() - self._ws_client.shutdown() - - @staticmethod - def _is_heartbeat(message): - return message.strip() == "" - - -class AbortMessageProcessor(WebsocketMessageProcessor): - def __init__(self, abort_impl, experiment): - super(AbortMessageProcessor, self).__init__() - self._abort_impl = abort_impl - self._experiment = experiment - self.received_abort_message = False - - def _process_message(self, message): - if message.get_type() == MessageType.STOP: - self._experiment.stop() - self._abort() - elif message.get_type() == MessageType.ABORT: - self._experiment.stop("Remotely aborted") - self._abort() - - def _abort(self): - self.received_abort_message = True - self._abort_impl.abort() diff --git a/src/neptune/legacy/internal/threads/hardware_metric_reporting_thread.py b/src/neptune/legacy/internal/threads/hardware_metric_reporting_thread.py deleted file mode 100644 index 45fbc2126..000000000 --- a/src/neptune/legacy/internal/threads/hardware_metric_reporting_thread.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import time - -from bravado.exception import HTTPError - -from neptune.legacy.exceptions import NeptuneException -from neptune.legacy.internal.threads.neptune_thread import NeptuneThread - -_logger = logging.getLogger(__name__) - - -class HardwareMetricReportingThread(NeptuneThread): - def __init__(self, metric_service, metric_sending_interval_seconds): - super(HardwareMetricReportingThread, self).__init__(is_daemon=True) - self.__metric_service = metric_service - self.__metric_sending_interval_seconds = metric_sending_interval_seconds - - def run(self): - try: - while self.should_continue_running(): - before = time.time() - - try: - self.__metric_service.report_and_send(timestamp=time.time()) - except (NeptuneException, HTTPError): - _logger.exception("Unexpected HTTP error in hardware metric reporting thread.") - - reporting_duration = time.time() - before - - time.sleep(max(0, self.__metric_sending_interval_seconds - reporting_duration)) - except Exception as e: - _logger.debug("Unexpected error in hardware metric reporting thread: %s", e) diff --git a/src/neptune/legacy/internal/threads/neptune_thread.py b/src/neptune/legacy/internal/threads/neptune_thread.py deleted file mode 100644 index fc6afe63d..000000000 --- a/src/neptune/legacy/internal/threads/neptune_thread.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import threading - -import six - - -class NeptuneThread(threading.Thread): - def __init__(self, is_daemon): - super(NeptuneThread, self).__init__(target=self.run) - self.daemon = is_daemon - self._interrupted = threading.Event() - - def should_continue_running(self): - if six.PY2: - all_threads = threading.enumerate() - - main_thread_is_alive = any(t.__class__ is threading._MainThread and t.is_alive() for t in all_threads) - else: - main_thread_is_alive = threading.main_thread().is_alive() - - return not self._interrupted.is_set() and main_thread_is_alive - - def interrupt(self): - self._interrupted.set() - - def run(self): - raise NotImplementedError() diff --git a/src/neptune/legacy/internal/threads/ping_thread.py b/src/neptune/legacy/internal/threads/ping_thread.py deleted file mode 100644 index a59a51ab6..000000000 --- a/src/neptune/legacy/internal/threads/ping_thread.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging - -from bravado.exception import HTTPUnprocessableEntity - -from neptune.legacy.internal.threads.neptune_thread import NeptuneThread - -_logger = logging.getLogger(__name__) - - -class PingThread(NeptuneThread): - PING_INTERVAL_SECS = 5 - - def __init__(self, backend, experiment): - super(PingThread, self).__init__(is_daemon=True) - - self.__backend = backend - self.__experiment = experiment - - def run(self): - while self.should_continue_running(): - try: - self.__backend.ping_experiment(self.__experiment) - except HTTPUnprocessableEntity: - # A 422 error means that we tried to ping the job after marking it as completed. - # In this case, this thread is not needed anymore. - break - except Exception: - _logger.exception("Unexpected error in ping thread.") - self._interrupted.wait(self.PING_INTERVAL_SECS) diff --git a/src/neptune/legacy/internal/utils/__init__.py b/src/neptune/legacy/internal/utils/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/utils/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/utils/alpha_integration.py b/src/neptune/legacy/internal/utils/alpha_integration.py deleted file mode 100644 index 2b9056924..000000000 --- a/src/neptune/legacy/internal/utils/alpha_integration.py +++ /dev/null @@ -1,237 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc -from collections import namedtuple - -from neptune.attributes import constants as alpha_consts -from neptune.internal import operation as alpha_operation -from neptune.internal.backends.api_model import AttributeType as AlphaAttributeType - -# Alpha equivalent of old api's `KeyValueProperty` used in `Experiment.properties` -from neptune.internal.operation import ImageValue -from neptune.legacy.exceptions import NeptuneException -from neptune.legacy.internal.channels.channels import ( - ChannelType, - ChannelValueType, -) - -AlphaKeyValueProperty = namedtuple("AlphaKeyValueProperty", ["key", "value"]) - - -class AlphaAttributeWrapper(abc.ABC): - """It's simple wrapper for `AttributeDTO`.""" - - _allowed_atribute_types = list() - - def __init__(self, attribute): - """Expects `AttributeDTO`""" - assert self._allowed_atribute_types is not None - if not self.is_valid_attribute(attribute): - raise NeptuneException(f"Invalid channel attribute type: {attribute.type}") - - self._attribute = attribute - - @classmethod - def is_valid_attribute(cls, attribute): - """Checks if attribute can be wrapped by particular descendant of this class""" - return attribute.type in cls._allowed_atribute_types - - @property - def _properties(self): - """Returns proper attribute property according to type""" - return getattr(self._attribute, f"{self._attribute.type}Properties") - - -class AlphaPropertyDTO(AlphaAttributeWrapper): - """It's simple wrapper for `AttributeDTO` objects which uses alpha variables attributes to fake properties. - - Alpha leaderboard doesn't have `KeyValueProperty` since it doesn't support properties at all, - so we do need fake `KeyValueProperty` class for backward compatibility with old client's code.""" - - _allowed_atribute_types = [ - AlphaAttributeType.STRING.value, - ] - - @classmethod - def is_valid_attribute(cls, attribute): - """Checks if attribute can be used as property""" - has_valid_type = super().is_valid_attribute(attribute) - is_in_properties_space = attribute.name.startswith(alpha_consts.PROPERTIES_ATTRIBUTE_SPACE) - return has_valid_type and is_in_properties_space - - @property - def key(self): - return self._properties.attributeName.split("/", 1)[-1] - - @property - def value(self): - return self._properties.value - - -class AlphaParameterDTO(AlphaAttributeWrapper): - """It's simple wrapper for `AttributeDTO` objects which uses alpha variables attributes to fake properties. - - Alpha leaderboard doesn't have `KeyValueProperty` since it doesn't support properties at all, - so we do need fake `KeyValueProperty` class for backward compatibility with old client's code.""" - - _allowed_atribute_types = [ - AlphaAttributeType.FLOAT.value, - AlphaAttributeType.STRING.value, - AlphaAttributeType.DATETIME.value, - ] - - @classmethod - def is_valid_attribute(cls, attribute): - """Checks if attribute can be used as property""" - has_valid_type = super().is_valid_attribute(attribute) - is_in_parameters_space = attribute.name.startswith(alpha_consts.PARAMETERS_ATTRIBUTE_SPACE) - return has_valid_type and is_in_parameters_space - - @property - def name(self): - return self._properties.attributeName.split("/", 1)[-1] - - @property - def value(self): - return self._properties.value - - @property - def parameterType(self): - return "double" if self._properties.attributeType == AlphaAttributeType.FLOAT.value else "string" - - -class AlphaChannelDTO(AlphaAttributeWrapper): - """It's simple wrapper for `AttributeDTO` objects which uses alpha series attributes to fake channels. - - Alpha leaderboard doesn't have `ChannelDTO` since it doesn't support channels at all, - so we do need fake `ChannelDTO` class for backward compatibility with old client's code.""" - - _allowed_atribute_types = [ - AlphaAttributeType.FLOAT_SERIES.value, - AlphaAttributeType.STRING_SERIES.value, - AlphaAttributeType.IMAGE_SERIES.value, - ] - - @property - def id(self): - return self._properties.attributeName - - @property - def name(self): - return self._properties.attributeName.split("/", 1)[-1] - - @property - def channelType(self): - attr_type = self._properties.attributeType - if attr_type == AlphaAttributeType.FLOAT_SERIES.value: - return ChannelType.NUMERIC.value - elif attr_type == AlphaAttributeType.STRING_SERIES.value: - return ChannelType.TEXT.value - elif attr_type == AlphaAttributeType.IMAGE_SERIES.value: - return ChannelType.IMAGE.value - - @property - def x(self): - return self._properties.lastStep - - @property - def y(self): - if self.channelType == ChannelType.IMAGE.value: - # We do not store last value for image series - return None - return self._properties.last - - -class AlphaChannelWithValueDTO: - """Alpha leaderboard doesn't have `ChannelWithValueDTO` since it doesn't support channels at all, - so we do need fake `ChannelWithValueDTO` class for backward compatibility with old client's code""" - - def __init__(self, channelId: str, channelName: str, channelType: str, x, y): - self._ch_id = channelId - self._ch_name = channelName - self._ch_type = channelType - self._x = x - self._y = y - - @property - def channelId(self): - return self._ch_id - - @property - def channelName(self): - return self._ch_name - - @property - def channelType(self): - return self._ch_type - - @property - def x(self): - return self._x - - @x.setter - def x(self, x): - self._x = x - - @property - def y(self): - return self._y - - @y.setter - def y(self, y): - self._y = y - - -def _map_using_dict(el, el_name, source_dict) -> alpha_operation.Operation: - try: - return source_dict[el] - except KeyError as e: - raise NeptuneException(f"We're not supporting {el} {el_name}.") from e - - -def channel_type_to_operation(channel_type: ChannelType) -> alpha_operation.Operation: - _channel_type_to_operation = { - ChannelType.TEXT: alpha_operation.LogStrings, - ChannelType.NUMERIC: alpha_operation.LogFloats, - ChannelType.IMAGE: alpha_operation.LogImages, - } - return _map_using_dict(channel_type, "channel type", _channel_type_to_operation) - - -def channel_type_to_clear_operation( - channel_type: ChannelType, -) -> alpha_operation.Operation: - _channel_type_to_operation = { - ChannelType.TEXT: alpha_operation.ClearStringLog, - ChannelType.NUMERIC: alpha_operation.ClearFloatLog, - ChannelType.IMAGE: alpha_operation.ClearImageLog, - } - return _map_using_dict(channel_type, "channel type", _channel_type_to_operation) - - -def channel_value_type_to_operation( - channel_value_type: ChannelValueType, -) -> alpha_operation.Operation: - _channel_value_type_to_operation = { - ChannelValueType.TEXT_VALUE: alpha_operation.LogStrings, - ChannelValueType.NUMERIC_VALUE: alpha_operation.LogFloats, - ChannelValueType.IMAGE_VALUE: alpha_operation.LogImages, - } - return _map_using_dict(channel_value_type, "channel value type", _channel_value_type_to_operation) - - -def deprecated_img_to_alpha_image(img: dict) -> ImageValue: - return ImageValue(data=img["data"], name=img["name"], description=img["description"]) diff --git a/src/neptune/legacy/internal/utils/deprecation.py b/src/neptune/legacy/internal/utils/deprecation.py deleted file mode 100644 index e082c71fe..000000000 --- a/src/neptune/legacy/internal/utils/deprecation.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from functools import wraps - -from neptune.common.warnings import warn_once - - -def legacy_client_deprecation(func): - @wraps(func) - def inner(*args, **kwargs): - warn_once( - message="You're using a legacy version of Neptune client." - " It will be moved to `neptune.legacy` as of `neptune-client==1.0.0`." - ) - return func(*args, **kwargs) - - return inner diff --git a/src/neptune/legacy/internal/utils/http.py b/src/neptune/legacy/internal/utils/http.py deleted file mode 100644 index 09a04a17e..000000000 --- a/src/neptune/legacy/internal/utils/http.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -from functools import wraps -from http.client import ( - NOT_FOUND, - UNPROCESSABLE_ENTITY, -) - -from requests.exceptions import HTTPError - -from neptune.legacy.api_exceptions import ( - ExperimentNotFound, - StorageLimitReached, -) -from neptune.legacy.exceptions import NeptuneException - -_logger = logging.getLogger(__name__) - - -def extract_response_field(response, field_name): - if response is None: - return None - - try: - response_json = response.json() - if isinstance(response_json, dict): - return response_json.get(field_name) - else: - _logger.debug("HTTP response is not a dict: %s", str(response_json)) - return None - except ValueError as e: - _logger.debug("Failed to parse HTTP response: %s", e) - return None - - -def handle_quota_limits(f): - """Wrapper for functions which may request for non existing experiment or cause quota limit breach - - Limitations: - Decorated function must be called with experiment argument like this fun(..., experiment=, ...)""" - - @wraps(f) - def handler(*args, **kwargs): - experiment = kwargs.get("experiment") - if experiment is None: - raise NeptuneException( - "This function must be called with experiment passed by name," - " like this fun(..., experiment=, ...)" - ) - try: - return f(*args, **kwargs) - except HTTPError as e: - if e.response.status_code == NOT_FOUND: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) - if e.response.status_code == UNPROCESSABLE_ENTITY and extract_response_field( - e.response, "title" - ).startswith("Storage limit reached in organization: "): - raise StorageLimitReached() - raise - - return handler diff --git a/src/neptune/legacy/internal/utils/http_utils.py b/src/neptune/legacy/internal/utils/http_utils.py deleted file mode 100644 index 09a04a17e..000000000 --- a/src/neptune/legacy/internal/utils/http_utils.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -from functools import wraps -from http.client import ( - NOT_FOUND, - UNPROCESSABLE_ENTITY, -) - -from requests.exceptions import HTTPError - -from neptune.legacy.api_exceptions import ( - ExperimentNotFound, - StorageLimitReached, -) -from neptune.legacy.exceptions import NeptuneException - -_logger = logging.getLogger(__name__) - - -def extract_response_field(response, field_name): - if response is None: - return None - - try: - response_json = response.json() - if isinstance(response_json, dict): - return response_json.get(field_name) - else: - _logger.debug("HTTP response is not a dict: %s", str(response_json)) - return None - except ValueError as e: - _logger.debug("Failed to parse HTTP response: %s", e) - return None - - -def handle_quota_limits(f): - """Wrapper for functions which may request for non existing experiment or cause quota limit breach - - Limitations: - Decorated function must be called with experiment argument like this fun(..., experiment=, ...)""" - - @wraps(f) - def handler(*args, **kwargs): - experiment = kwargs.get("experiment") - if experiment is None: - raise NeptuneException( - "This function must be called with experiment passed by name," - " like this fun(..., experiment=, ...)" - ) - try: - return f(*args, **kwargs) - except HTTPError as e: - if e.response.status_code == NOT_FOUND: - raise ExperimentNotFound( - experiment_short_id=experiment.id, - project_qualified_name=experiment._project.full_id, - ) - if e.response.status_code == UNPROCESSABLE_ENTITY and extract_response_field( - e.response, "title" - ).startswith("Storage limit reached in organization: "): - raise StorageLimitReached() - raise - - return handler diff --git a/src/neptune/legacy/internal/utils/image.py b/src/neptune/legacy/internal/utils/image.py deleted file mode 100644 index 4faf520d8..000000000 --- a/src/neptune/legacy/internal/utils/image.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import io -import os - -import numpy -import six -from PIL import Image - -from neptune.legacy.exceptions import ( - FileNotFound, - InvalidChannelValue, -) - - -def get_image_content(image): - if isinstance(image, six.string_types): - if not os.path.exists(image): - raise FileNotFound(image) - with open(image, "rb") as image_file: - return image_file.read() - - elif isinstance(image, numpy.ndarray): - return _get_numpy_as_image(image) - - elif isinstance(image, Image.Image): - return _get_pil_image_data(image) - - else: - try: - from matplotlib import figure - - if isinstance(image, figure.Figure): - return _get_figure_as_image(image) - except ImportError: - pass - - try: - from torch import Tensor as TorchTensor - - if isinstance(image, TorchTensor): - return _get_numpy_as_image(image.detach().numpy()) - except ImportError: - pass - - try: - from tensorflow import Tensor as TensorflowTensor - - if isinstance(image, TensorflowTensor): - return _get_numpy_as_image(image.numpy()) - except ImportError: - pass - - raise InvalidChannelValue(expected_type="image", actual_type=type(image).__name__) - - -def _get_figure_as_image(figure): - with io.BytesIO() as image_buffer: - figure.savefig(image_buffer, format="png", bbox_inches="tight") - return image_buffer.getvalue() - - -def _get_pil_image_data(image): - with io.BytesIO() as image_buffer: - image.save(image_buffer, format="PNG") - return image_buffer.getvalue() - - -def _get_numpy_as_image(array): - array = array.copy() # prevent original array from modifying - - array *= 255 - shape = array.shape - if len(shape) == 2: - return _get_pil_image_data(Image.fromarray(array.astype(numpy.uint8))) - if len(shape) == 3: - if shape[2] == 1: - array2d = numpy.array([[col[0] for col in row] for row in array]) - return _get_pil_image_data(Image.fromarray(array2d.astype(numpy.uint8))) - if shape[2] in (3, 4): - return _get_pil_image_data(Image.fromarray(array.astype(numpy.uint8))) - raise ValueError( - "Incorrect size of numpy.ndarray. Should be 2-dimensional or" - " 3-dimensional with 3rd dimension of size 1, 3 or 4." - ) diff --git a/src/neptune/legacy/internal/utils/source_code.py b/src/neptune/legacy/internal/utils/source_code.py deleted file mode 100644 index 67acdc2ae..000000000 --- a/src/neptune/legacy/internal/utils/source_code.py +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import os.path -import sys -from typing import ( - List, - Optional, - Tuple, -) - -from neptune.common.storage.storage_utils import normalize_file_name -from neptune.common.utils import ( - glob, - is_ipython, -) - - -def get_source_code_to_upload( - upload_source_files: Optional[List[str]], -) -> Tuple[str, List[Tuple[str, str]]]: - source_target_pairs = [] - if is_ipython(): - main_file = None - entrypoint = None - else: - main_file = sys.argv[0] - entrypoint = main_file or None - if upload_source_files is None: - if main_file is not None and os.path.isfile(main_file): - entrypoint = normalize_file_name(os.path.basename(main_file)) - source_target_pairs = [ - ( - os.path.abspath(main_file), - normalize_file_name(os.path.basename(main_file)), - ) - ] - else: - expanded_source_files = set() - for filepath in upload_source_files: - expanded_source_files |= set(glob(filepath)) - if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 5): - for filepath in expanded_source_files: - if filepath.startswith(".."): - raise ValueError("You need to have Python 3.5 or later to use paths outside current directory.") - source_target_pairs.append((os.path.abspath(filepath), normalize_file_name(filepath))) - else: - absolute_paths = [] - for filepath in expanded_source_files: - absolute_paths.append(os.path.abspath(filepath)) - try: - common_source_root = os.path.commonpath(absolute_paths) - except ValueError: - for absolute_path in absolute_paths: - source_target_pairs.append((absolute_path, normalize_file_name(absolute_path))) - else: - if os.path.isfile(common_source_root): - common_source_root = os.path.dirname(common_source_root) - if common_source_root.startswith(os.getcwd() + os.sep): - common_source_root = os.getcwd() - for absolute_path in absolute_paths: - source_target_pairs.append( - ( - absolute_path, - normalize_file_name(os.path.relpath(absolute_path, common_source_root)), - ) - ) - return entrypoint, source_target_pairs diff --git a/src/neptune/legacy/internal/websockets/__init__.py b/src/neptune/legacy/internal/websockets/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/src/neptune/legacy/internal/websockets/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/src/neptune/legacy/internal/websockets/message.py b/src/neptune/legacy/internal/websockets/message.py deleted file mode 100644 index 91f4f0baa..000000000 --- a/src/neptune/legacy/internal/websockets/message.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from neptune.attributes.constants import ( - SIGNAL_TYPE_ABORT, - SIGNAL_TYPE_STOP, -) - - -class Message(object): - def __init__(self): - pass - - MESSAGE_TYPE = "messageType" - MESSAGE_NEW_TYPE = "type" - MESSAGE_BODY = "messageBody" - MESSAGE_NEW_BODY = "body" - - @classmethod - def from_json(cls, json_value): - message_type = json_value.get(Message.MESSAGE_TYPE) or json_value.get(Message.MESSAGE_NEW_TYPE) - message_body = json_value.get(Message.MESSAGE_BODY) or json_value.get(Message.MESSAGE_NEW_BODY) - - if message_type == SIGNAL_TYPE_STOP: - message_type = MessageType.STOP - elif message_type == SIGNAL_TYPE_ABORT: - message_type = MessageType.ABORT - - if message_type in MessageClassRegistry.MESSAGE_CLASSES: - return MessageClassRegistry.MESSAGE_CLASSES[message_type].from_json(message_body) - else: - raise ValueError("Unknown message type '{}'!".format(message_type)) - - @classmethod - def get_type(cls): - raise NotImplementedError() - - def body_to_json(self): - raise NotImplementedError() - - -class AbortMessage(Message): - @classmethod - def get_type(cls): - return MessageType.ABORT - - @classmethod - def from_json(cls, json_value): - return AbortMessage() - - def body_to_json(self): - return None - - -class StopMessage(Message): - @classmethod - def get_type(cls): - return MessageType.STOP - - @classmethod - def from_json(cls, json_value): - return StopMessage() - - def body_to_json(self): - return None - - -class ActionInvocationMessage(Message): - _ACTION_ID_JSON_KEY = "actionId" - _ACTION_INVOCATION_ID_JSON_KEY = "actionInvocationId" - _ARGUMENT_JSON_KEY = "argument" - - def __init__(self, action_id, action_invocation_id, argument): - super(ActionInvocationMessage, self).__init__() - self.action_id = action_id - self.action_invocation_id = action_invocation_id - self.argument = argument - - @classmethod - def get_type(cls): - return MessageType.ACTION_INVOCATION - - @classmethod - def from_json(cls, json_value): - field_names = [ - cls._ACTION_ID_JSON_KEY, - cls._ACTION_INVOCATION_ID_JSON_KEY, - cls._ARGUMENT_JSON_KEY, - ] - return ActionInvocationMessage(*[json_value[field] for field in field_names]) - - def body_to_json(self): - return { - self._ACTION_ID_JSON_KEY: self.action_id, - self._ACTION_INVOCATION_ID_JSON_KEY: self.action_invocation_id, - self._ARGUMENT_JSON_KEY: self.argument, - } - - -class MessageType(object): - NEW_CHANNEL_VALUES = "NewChannelValues" - ABORT = "Abort" - STOP = "Stop" - ACTION_INVOCATION = "InvokeAction" - - -class MessageClassRegistry(object): - def __init__(self): - pass - - MESSAGE_CLASSES = dict([(cls.get_type(), cls) for cls in Message.__subclasses__()]) diff --git a/src/neptune/legacy/internal/websockets/reconnecting_websocket_factory.py b/src/neptune/legacy/internal/websockets/reconnecting_websocket_factory.py deleted file mode 100644 index 98e3b8583..000000000 --- a/src/neptune/legacy/internal/websockets/reconnecting_websocket_factory.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from neptune.common.websockets.reconnecting_websocket import ReconnectingWebsocket - - -class ReconnectingWebsocketFactory(object): - def __init__(self, backend, url): - self._backend = backend - self._url = url - - def create(self, shutdown_condition): - return ReconnectingWebsocket( - url=self._url, - oauth2_session=self._backend.authenticator.auth.session, - shutdown_event=shutdown_condition, - proxies=self._backend.proxies, - ) diff --git a/src/neptune/legacy/internal/websockets/websocket_message_processor.py b/src/neptune/legacy/internal/websockets/websocket_message_processor.py deleted file mode 100644 index 8567ca698..000000000 --- a/src/neptune/legacy/internal/websockets/websocket_message_processor.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json - -from neptune.legacy.internal.websockets.message import Message - - -class WebsocketMessageProcessor(object): - def __init__(self): - pass - - def run(self, raw_message): - # Atmosphere framework sends heartbeat messages every minute, we have to ignore them - if raw_message is not None and not self._is_heartbeat(raw_message): - message = Message.from_json(json.loads(raw_message)) - self._process_message(message) - - def _process_message(self, message): - raise NotImplementedError() - - @staticmethod - def _is_heartbeat(message): - return message.strip() == "" diff --git a/src/neptune/legacy/model.py b/src/neptune/legacy/model.py deleted file mode 100644 index e8d40c744..000000000 --- a/src/neptune/legacy/model.py +++ /dev/null @@ -1,144 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class ChannelWithLastValue: - def __init__(self, channel_with_value_dto): - self.channel_with_value_dto = channel_with_value_dto - - @property - def id(self): - return self.channel_with_value_dto.channelId - - @property - def name(self): - return self.channel_with_value_dto.channelName - - @property - def type(self): - return self.channel_with_value_dto.channelType - - @property - def x(self): - return self.channel_with_value_dto.x - - @x.setter - def x(self, x): - self.channel_with_value_dto.x = x - - @property - def trimmed_y(self): - return self.y[:255] if self.type == "text" else self.y - - @property - def y(self): - return self.channel_with_value_dto.y - - @y.setter - def y(self, y): - self.channel_with_value_dto.y = y - - -class LeaderboardEntry(object): - def __init__(self, project_leaderboard_entry_dto): - self.project_leaderboard_entry_dto = project_leaderboard_entry_dto - - @property - def id(self): - return self.project_leaderboard_entry_dto.shortId - - @property - def name(self): - return self.project_leaderboard_entry_dto.name - - @property - def state(self): - return self.project_leaderboard_entry_dto.state - - @property - def internal_id(self): - return self.project_leaderboard_entry_dto.id - - @property - def project_full_id(self): - return "{org_name}/{project_name}".format( - org_name=self.project_leaderboard_entry_dto.organizationName, - project_name=self.project_leaderboard_entry_dto.projectName, - ) - - @property - def system_properties(self): - entry = self.project_leaderboard_entry_dto - return { - "id": entry.shortId, - "name": entry.name, - "created": entry.timeOfCreation, - "finished": entry.timeOfCompletion, - "running_time": entry.runningTime, - "owner": entry.owner, - "size": entry.size, - "tags": entry.tags, - "notes": entry.description, - } - - @property - def channels(self): - return [ChannelWithLastValue(ch) for ch in self.project_leaderboard_entry_dto.channelsLastValues] - - def add_channel(self, channel): - self.project_leaderboard_entry_dto.channelsLastValues.append(channel.channel_with_value_dto) - - @property - def channels_dict_by_name(self): - return dict((ch.name, ch) for ch in self.channels) - - @property - def parameters(self): - return dict((p.name, p.value) for p in self.project_leaderboard_entry_dto.parameters) - - @property - def properties(self): - return dict((p.key, p.value) for p in self.project_leaderboard_entry_dto.properties) - - @property - def tags(self): - return self.project_leaderboard_entry_dto.tags - - -class Point(object): - def __init__(self, point_dto): - self.point_dto = point_dto - - @property - def x(self): - return self.point_dto.x - - @property - def numeric_y(self): - return self.point_dto.y.numericValue - - -class Points(object): - def __init__(self, point_dtos): - self.point_dtos = point_dtos - - @property - def xs(self): - return [p.x for p in self.point_dtos] - - @property - def numeric_ys(self): - return [p.y.numericValue for p in self.point_dtos] diff --git a/src/neptune/legacy/notebook.py b/src/neptune/legacy/notebook.py deleted file mode 100644 index 76029342e..000000000 --- a/src/neptune/legacy/notebook.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -from neptune.common.utils import validate_notebook_path -from neptune.legacy.internal.utils.deprecation import legacy_client_deprecation - - -class Notebook(object): - """It contains all the information about a Neptune Notebook - - Args: - backend (:class:`~neptune.ApiClient`): A ApiClient object - project (:class:`~neptune.projects.Project`): Project object - _id (:obj:`str`): Notebook id - owner (:obj:`str`): Creator of the notebook is the Notebook owner - - Examples: - .. code:: python3 - - # Create a notebook in Neptune. - notebook = project.create_notebook('data_exploration.ipynb') - - """ - - @legacy_client_deprecation - def __init__(self, backend, project, _id, owner): - self._backend = backend - self._project = project - self._id = _id - self._owner = owner - - @property - def id(self): - return self._id - - @property - def owner(self): - return self._owner - - def add_checkpoint(self, file_path): - """Uploads new checkpoint of the notebook to Neptune - - Args: - file_path (:obj:`str`): File path containing notebook contents - - Example: - - .. code:: python3 - - # Create a notebook. - notebook = project.create_notebook('file.ipynb') - - # Change content in your notebook & save it - - # Upload new checkpoint - notebook.add_checkpoint('file.ipynb') - """ - validate_notebook_path(file_path) - - with open(file_path) as f: - return self._backend.create_checkpoint(self.id, os.path.abspath(file_path), f) - - def get_path(self): - """Returns the path used to upload the current checkpoint of this notebook - - Returns: - :obj:`str`: path of the current checkpoint - """ - return self._backend.get_last_checkpoint(self._project, self._id).path - - def get_name(self): - """Returns the name used to upload the current checkpoint of this notebook - - Returns: - :obj:`str`: the name of current checkpoint - """ - return self._backend.get_last_checkpoint(self._project, self._id).name diff --git a/src/neptune/legacy/oauth.py b/src/neptune/legacy/oauth.py deleted file mode 100644 index a600f2067..000000000 --- a/src/neptune/legacy/oauth.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# flake8: noqa -from neptune.common.oauth import ( - NeptuneAuth, - NeptuneAuthenticator, -) diff --git a/src/neptune/legacy/patterns.py b/src/neptune/legacy/patterns.py deleted file mode 100644 index 3b59f9b5e..000000000 --- a/src/neptune/legacy/patterns.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# flake8: noqa -from neptune.common.patterns import PROJECT_QUALIFIED_NAME_PATTERN diff --git a/src/neptune/legacy/projects.py b/src/neptune/legacy/projects.py deleted file mode 100644 index 6f80811bd..000000000 --- a/src/neptune/legacy/projects.py +++ /dev/null @@ -1,596 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import atexit -import logging -import os -import os.path -import threading -from platform import node as get_hostname - -import click -import pandas as pd -import six - -from neptune.common.utils import ( - as_list, - discover_git_repo_location, - get_git_info, - map_keys, -) -from neptune.legacy.envs import ( - NOTEBOOK_ID_ENV_NAME, - NOTEBOOK_PATH_ENV_NAME, -) -from neptune.legacy.exceptions import NeptuneNoExperimentContextException -from neptune.legacy.experiments import Experiment -from neptune.legacy.internal.abort import DefaultAbortImpl -from neptune.legacy.internal.notebooks.notebooks import create_checkpoint -from neptune.legacy.internal.utils.deprecation import legacy_client_deprecation -from neptune.legacy.internal.utils.source_code import get_source_code_to_upload - -_logger = logging.getLogger(__name__) - - -class Project(object): - """A class for storing information and managing Neptune project. - - Args: - backend (:class:`~neptune.ApiClient`, required): A ApiClient object. - internal_id (:obj:`str`, required): ID of the project. - namespace (:obj:`str`, required): It can either be your workspace or user name. - name (:obj:`str`, required): project name. - - Note: - ``namespace`` and ``name`` joined together with ``/`` form ``project_qualified_name``. - """ - - @legacy_client_deprecation - def __init__(self, backend, internal_id, namespace, name): - self._backend = backend - self.internal_id = internal_id - self.namespace = namespace - self.name = name - - self._experiments_stack = [] - self.__lock = threading.RLock() - atexit.register(self._shutdown_hook) - - def get_members(self): - """Retrieve a list of project members. - - Returns: - :obj:`list` of :obj:`str` - A list of usernames of project members. - - Examples: - - .. code:: python3 - - project = session.get_projects('neptune-ai')['neptune-ai/Salt-Detection'] - project.get_members() - - """ - project_members = self._backend.get_project_members(self.internal_id) - return [member.registeredMemberInfo.username for member in project_members if member.registeredMemberInfo] - - def get_experiments(self, id=None, state=None, owner=None, tag=None, min_running_time=None): - """Retrieve list of experiments matching the specified criteria. - - All parameters are optional, each of them specifies a single criterion. - Only experiments matching all of the criteria will be returned. - - Args: - id (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | An experiment id like ``'SAN-1'`` or list of ids like ``['SAN-1', 'SAN-2']``. - | Matching any element of the list is sufficient to pass criterion. - state (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | An experiment state like ``'succeeded'`` or list of states like ``['succeeded', 'running']``. - | Possible values: ``'running'``, ``'succeeded'``, ``'failed'``, ``'aborted'``. - | Matching any element of the list is sufficient to pass criterion. - owner (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | *Username* of the experiment owner (User who created experiment is an owner) like ``'josh'`` - or list of owners like ``['frederic', 'josh']``. - | Matching any element of the list is sufficient to pass criterion. - tag (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | An experiment tag like ``'lightGBM'`` or list of tags like ``['pytorch', 'cycleLR']``. - | Only experiments that have all specified tags will match this criterion. - min_running_time (:obj:`int`, optional, default is ``None``): - Minimum running time of an experiment in seconds, like ``2000``. - - Returns: - :obj:`list` of :class:`~neptune.experiments.Experiment` objects. - - Examples: - - .. code:: python3 - - # Fetch a project - project = session.get_projects('neptune-ai')['neptune-ai/Salt-Detection'] - - # Get list of experiments - project.get_experiments(state=['aborted'], owner=['neyo'], min_running_time=100000) - - # Example output: - # [Experiment(SAL-1609), - # Experiment(SAL-1765), - # Experiment(SAL-1941), - # Experiment(SAL-1960), - # Experiment(SAL-2025)] - """ - leaderboard_entries = self._fetch_leaderboard(id, state, owner, tag, min_running_time) - return [Experiment(self._backend, self, entry.id, entry.internal_id) for entry in leaderboard_entries] - - def get_leaderboard(self, id=None, state=None, owner=None, tag=None, min_running_time=None): - """Fetch Neptune experiments view as pandas ``DataFrame``. - - **returned DataFrame** - - | In the returned ``DataFrame`` each *row* is an experiment and *columns* represent all system properties, - numeric and text logs, parameters and properties in these experiments. - | Note that, returned ``DataFrame`` does not contain all columns across the entire project. - | Some columns may be empty, since experiments may define various logs, properties, etc. - | For each log at most one (the last one) value is returned per experiment. - | Text values are trimmed to 255 characters. - - **about parameters** - - All parameters are optional, each of them specifies a single criterion. - Only experiments matching all of the criteria will be returned. - - Args: - id (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | An experiment id like ``'SAN-1'`` or list of ids like ``['SAN-1', 'SAN-2']``. - | Matching any element of the list is sufficient to pass criterion. - state (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | An experiment state like ``'succeeded'`` or list of states like ``['succeeded', 'running']``. - | Possible values: ``'running'``, ``'succeeded'``, ``'failed'``, ``'aborted'``. - | Matching any element of the list is sufficient to pass criterion. - owner (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | *Username* of the experiment owner (User who created experiment is an owner) like ``'josh'`` - or list of owners like ``['frederic', 'josh']``. - | Matching any element of the list is sufficient to pass criterion. - tag (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``): - | An experiment tag like ``'lightGBM'`` or list of tags like ``['pytorch', 'cycleLR']``. - | Only experiments that have all specified tags will match this criterion. - min_running_time (:obj:`int`, optional, default is ``None``): - Minimum running time of an experiment in seconds, like ``2000``. - - Returns: - :obj:`pandas.DataFrame` - Fetched Neptune experiments view. - - Examples: - - .. code:: python3 - - # Fetch a project. - project = session.get_projects('neptune-ai')['neptune-ai/Salt-Detection'] - - # Get DataFrame that resembles experiment view. - project.get_leaderboard(state=['aborted'], owner=['neyo'], min_running_time=100000) - """ - - leaderboard_entries = self._fetch_leaderboard(id, state, owner, tag, min_running_time) - - def make_row(entry): - channels = dict(("channel_{}".format(ch.name), ch.trimmed_y) for ch in entry.channels) - - parameters = map_keys("parameter_{}".format, entry.parameters) - properties = map_keys("property_{}".format, entry.properties) - - r = {} - r.update(entry.system_properties) - r.update(channels) - r.update(parameters) - r.update(properties) - return r - - rows = ((n, make_row(e)) for (n, e) in enumerate(leaderboard_entries)) - - df = pd.DataFrame.from_dict(data=dict(rows), orient="index") - df = df.reindex(self._sort_leaderboard_columns(df.columns), axis="columns") - return df - - def create_experiment( - self, - name=None, - description=None, - params=None, - properties=None, - tags=None, - upload_source_files=None, - abort_callback=None, - logger=None, - upload_stdout=True, - upload_stderr=True, - send_hardware_metrics=True, - run_monitoring_thread=True, - handle_uncaught_exceptions=True, - git_info=None, - hostname=None, - notebook_id=None, - notebook_path=None, - ): - """Create and start Neptune experiment. - - Create experiment, set its status to `running` and append it to the top of the experiments view. - All parameters are optional, hence minimal invocation: ``neptune.create_experiment()``. - - Args: - name (:obj:`str`, optional, default is ``'Untitled'``): - Editable name of the experiment. - Name is displayed in the experiment's `Details` (`Metadata` section) - and in `experiments view` as a column. - - description (:obj:`str`, optional, default is ``''``): - Editable description of the experiment. - Description is displayed in the experiment's `Details` (`Metadata` section) - and can be displayed in the `experiments view` as a column. - - params (:obj:`dict`, optional, default is ``{}``): - Parameters of the experiment. - After experiment creation ``params`` are read-only - (see: :meth:`~neptune.experiments.Experiment.get_parameters`). - Parameters are displayed in the experiment's `Details` (`Parameters` section) - and each key-value pair can be viewed in `experiments view` as a column. - - properties (:obj:`dict`, optional, default is ``{}``): - Properties of the experiment. - They are editable after experiment is created. - Properties are displayed in the experiment's `Details` (`Properties` section) - and each key-value pair can be viewed in `experiments view` as a column. - - tags (:obj:`list`, optional, default is ``[]``): - Must be list of :obj:`str`. Tags of the experiment. - They are editable after experiment is created - (see: :meth:`~neptune.experiments.Experiment.append_tag` - and :meth:`~neptune.experiments.Experiment.remove_tag`). - Tags are displayed in the experiment's `Details` (`Metadata` section) - and can be viewed in `experiments view` as a column. - - upload_source_files (:obj:`list` or :obj:`str`, optional, default is ``None``): - List of source files to be uploaded. Must be list of :obj:`str` or single :obj:`str`. - Uploaded sources are displayed in the experiment's `Source code` tab. - - | If ``None`` is passed, Python file from which experiment was created will be uploaded. - | Pass empty list (``[]``) to upload no files. - | Unix style pathname pattern expansion is supported. For example, you can pass ``'*.py'`` to upload - all python source files from the current directory. - For Python 3.5 or later, paths of uploaded files on server are resolved as relative to the - | calculated common root of all uploaded source files. For older Python versions, paths on server are - | resolved always as relative to the current directory. - For recursion lookup use ``'**/*.py'`` (for Python 3.5 and later). - For more information see `glob library `_. - - abort_callback (:obj:`callable`, optional, default is ``None``): - Callback that defines how `abort experiment` action in the Web application should work. - Actual behavior depends on your setup: - - * (default) If ``abort_callback=None`` and `psutil `_ - is installed, then current process and it's children are aborted by sending `SIGTERM`. - If, after grace period, processes are not terminated, `SIGKILL` is sent. - * If ``abort_callback=None`` and `psutil `_ - is **not** installed, then `abort experiment` action just marks experiment as *aborted* - in the Web application. No action is performed on the current process. - * If ``abort_callback=callable``, then ``callable`` is executed when `abort experiment` action - in the Web application is triggered. - - logger (:obj:`logging.Logger` or `None`, optional, default is ``None``): - If Python's `Logger `_ - is passed, new experiment's `text log` - (see: :meth:`~neptune.experiments.Experiment.log_text`) with name `"logger"` is created. - Each time `Python logger` logs new data, it is automatically sent to the `"logger"` in experiment. - As a results all data from `Python logger` are in the `Logs` tab in the experiment. - - upload_stdout (:obj:`Boolean`, optional, default is ``True``): - Whether to send stdout to experiment's *Monitoring*. - - upload_stderr (:obj:`Boolean`, optional, default is ``True``): - Whether to send stderr to experiment's *Monitoring*. - - send_hardware_metrics (:obj:`Boolean`, optional, default is ``True``): - Whether to send hardware monitoring logs (CPU, GPU, Memory utilization) to experiment's *Monitoring*. - - run_monitoring_thread (:obj:`Boolean`, optional, default is ``True``): - Whether to run thread that pings Neptune server in order to determine if experiment is responsive. - - handle_uncaught_exceptions (:obj:`Boolean`, optional, default is ``True``): - Two options ``True`` and ``False`` are possible: - - * If set to ``True`` and uncaught exception occurs, then Neptune automatically place - `Traceback` in the experiment's `Details` and change experiment status to `Failed`. - * If set to ``False`` and uncaught exception occurs, then no action is performed - in the Web application. As a consequence, experiment's status is `running` or `not responding`. - - git_info (:class:`~neptune.git_info.GitInfo`, optional, default is ``None``): - - | Instance of the class :class:`~neptune.git_info.GitInfo` that provides information about - the git repository from which experiment was started. - | If ``None`` is passed, - system attempts to automatically extract information about git repository in the following way: - - * System looks for `.git` file in the current directory and, if not found, - goes up recursively until `.git` file will be found - (see: :meth:`~neptune.utils.get_git_info`). - * If there is no git repository, - then no information about git is displayed in experiment details in Neptune web application. - - hostname (:obj:`str`, optional, default is ``None``): - If ``None``, neptune.legacy automatically get `hostname` information. - User can also set `hostname` directly by passing :obj:`str`. - - Returns: - :class:`~neptune.experiments.Experiment` object that is used to manage experiment and log data to it. - - Raises: - `ExperimentValidationError`: When provided arguments are invalid. - `ExperimentLimitReached`: When experiment limit in the project has been reached. - - Examples: - - .. code:: python3 - - # minimal invoke - neptune.create_experiment() - - # explicitly return experiment object - experiment = neptune.create_experiment() - - # create experiment with name and two parameters - neptune.create_experiment(name='first-pytorch-ever', - params={'lr': 0.0005, - 'dropout': 0.2}) - - # create experiment with name and description, and no sources files uploaded - neptune.create_experiment(name='neural-net-mnist', - description='neural net trained on MNIST', - upload_source_files=[]) - - # Send all py files in cwd (excluding hidden files with names beginning with a dot) - neptune.create_experiment(upload_source_files='*.py') - - # Send all py files from all subdirectories (excluding hidden files with names beginning with a dot) - # Supported on Python 3.5 and later. - neptune.create_experiment(upload_source_files='**/*.py') - - # Send all files and directories in cwd (excluding hidden files with names beginning with a dot) - neptune.create_experiment(upload_source_files='*') - - # Send all files and directories in cwd including hidden files - neptune.create_experiment(upload_source_files=['*', '.*']) - - # Send files with names being a single character followed by '.py' extension. - neptune.create_experiment(upload_source_files='?.py') - - # larger example - neptune.create_experiment(name='first-pytorch-ever', - params={'lr': 0.0005, - 'dropout': 0.2}, - properties={'key1': 'value1', - 'key2': 17, - 'key3': 'other-value'}, - description='write longer description here', - tags=['list-of', 'tags', 'goes-here', 'as-list-of-strings'], - upload_source_files=['training_with_pytorch.py', 'net.py']) - """ - - if name is None: - name = "Untitled" - - if description is None: - description = "" - - if params is None: - params = {} - - if properties is None: - properties = {} - - if tags is None: - tags = [] - - if git_info is None: - git_info = get_git_info(discover_git_repo_location()) - - if hostname is None: - hostname = get_hostname() - - if notebook_id is None and os.getenv(NOTEBOOK_ID_ENV_NAME, None) is not None: - notebook_id = os.environ[NOTEBOOK_ID_ENV_NAME] - - if isinstance(upload_source_files, six.string_types): - upload_source_files = [upload_source_files] - - entrypoint, source_target_pairs = get_source_code_to_upload(upload_source_files=upload_source_files) - - if notebook_path is None and os.getenv(NOTEBOOK_PATH_ENV_NAME, None) is not None: - notebook_path = os.environ[NOTEBOOK_PATH_ENV_NAME] - - abortable = abort_callback is not None or DefaultAbortImpl.requirements_installed() - - checkpoint_id = None - if notebook_id is not None and notebook_path is not None: - checkpoint = create_checkpoint( - backend=self._backend, - notebook_id=notebook_id, - notebook_path=notebook_path, - ) - if checkpoint is not None: - checkpoint_id = checkpoint.id - - experiment = self._backend.create_experiment( - project=self, - name=name, - description=description, - params=params, - properties=properties, - tags=tags, - abortable=abortable, - monitored=run_monitoring_thread, - git_info=git_info, - hostname=hostname, - entrypoint=entrypoint, - notebook_id=notebook_id, - checkpoint_id=checkpoint_id, - ) - - self._backend.upload_source_code(experiment, source_target_pairs) - - experiment._start( - abort_callback=abort_callback, - logger=logger, - upload_stdout=upload_stdout, - upload_stderr=upload_stderr, - send_hardware_metrics=send_hardware_metrics, - run_monitoring_thread=run_monitoring_thread, - handle_uncaught_exceptions=handle_uncaught_exceptions, - ) - - self._push_new_experiment(experiment) - - click.echo(self._get_experiment_link(experiment)) - - return experiment - - def _get_experiment_link(self, experiment): - return "{base_url}/{namespace}/{project}/e/{exp_id}".format( - base_url=self._backend.display_address, - namespace=self.namespace, - project=self.name, - exp_id=experiment.id, - ) - - def create_notebook(self): - """Create a new notebook object and return corresponding :class:`~neptune.notebook.Notebook` instance. - - Returns: - :class:`~neptune.notebook.Notebook` object. - - Examples: - - .. code:: python3 - - # Instantiate a session and fetch a project - project = neptune.init() - - # Create a notebook in Neptune - notebook = project.create_notebook() - """ - return self._backend.create_notebook(self) - - def get_notebook(self, notebook_id): - """Get a :class:`~neptune.notebook.Notebook` object with given ``notebook_id``. - - Returns: - :class:`~neptune.notebook.Notebook` object. - - Examples: - - .. code:: python3 - - # Instantiate a session and fetch a project - project = neptune.init() - - # Get a notebook object - notebook = project.get_notebook('d1c1b494-0620-4e54-93d5-29f4e848a51a') - """ - return self._backend.get_notebook(project=self, notebook_id=notebook_id) - - @property - def full_id(self): - """Project qualified name as :obj:`str`, for example `john/sandbox`.""" - return "{}/{}".format(self.namespace, self.name) - - def __str__(self): - return "Project({})".format(self.full_id) - - def __repr__(self): - return str(self) - - def __eq__(self, o): - return self.__dict__ == o.__dict__ - - def __ne__(self, o): - return not self.__eq__(o) - - def _fetch_leaderboard(self, id, state, owner, tag, min_running_time): - return self._backend.get_leaderboard_entries( - project=self, - ids=as_list(id), - states=as_list(state), - owners=as_list(owner), - tags=as_list(tag), - min_running_time=min_running_time, - ) - - @staticmethod - def _sort_leaderboard_columns(column_names): - user_defined_weights = {"channel": 1, "parameter": 2, "property": 3} - - system_properties_weights = { - "id": 0, - "name": 1, - "created": 2, - "finished": 3, - "owner": 4, - "worker_type": 5, - "environment": 6, - } - - def key(c): - """A sorting key for a column name. - - Sorts by the system properties first, then channels, parameters, user-defined properties. - - Within each group columns are sorted alphabetically, except for system properties, - where order is custom. - """ - parts = c.split("_", 1) - if parts[0] in user_defined_weights.keys(): - name = parts[1] - weight = user_defined_weights.get(parts[0], 99) - system_property_weight = None - else: - name = c - weight = 0 - system_property_weight = system_properties_weights.get(name, 99) - - return weight, system_property_weight, name - - return sorted(column_names, key=key) - - def _get_current_experiment(self): - with self.__lock: - if self._experiments_stack: - return self._experiments_stack[-1] - else: - raise NeptuneNoExperimentContextException() - - def _push_new_experiment(self, new_experiment): - with self.__lock: - self._experiments_stack.append(new_experiment) - return new_experiment - - def _remove_stopped_experiment(self, experiment): - with self.__lock: - if self._experiments_stack: - self._experiments_stack = [exp for exp in self._experiments_stack if exp != experiment] - - def _shutdown_hook(self): - if self._experiments_stack: - # stopping experiment removes it from list, co we copy it - copied_experiment_list = [exp for exp in self._experiments_stack] - for exp in copied_experiment_list: - exp.stop() diff --git a/src/neptune/legacy/sessions.py b/src/neptune/legacy/sessions.py deleted file mode 100644 index e936c4ec9..000000000 --- a/src/neptune/legacy/sessions.py +++ /dev/null @@ -1,231 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -from collections import OrderedDict - -from neptune.common.utils import assure_project_qualified_name -from neptune.legacy.internal.api_clients import HostedNeptuneBackendApiClient -from neptune.legacy.internal.utils.deprecation import legacy_client_deprecation -from neptune.legacy.projects import Project - -_logger = logging.getLogger(__name__) - - -class Session(object): - """A class for running communication with Neptune. - - In order to query Neptune experiments you need to instantiate this object first. - - Args: - backend (:class:`~neptune.backend.ApiClient`, optional, default is ``None``): - By default, Neptune client library sends logs, metrics, images, etc to Neptune servers: - either publicly available SaaS, or an on-premises installation. - - You can pass the default backend instance explicitly to specify its parameters: - - .. code :: python3 - - from neptune.legacy import Session, HostedNeptuneBackendApiClient - session = Session(backend=HostedNeptuneBackendApiClient(...)) - - Passing an instance of :class:`~neptune.OfflineApiClient` makes your code run without communicating - with Neptune servers. - - .. code :: python3 - - from neptune.legacy import Session, OfflineApiClient - session = Session(backend=OfflineApiClient()) - - api_token (:obj:`str`, optional, default is ``None``): - User's API token. If ``None``, the value of ``NEPTUNE_API_TOKEN`` environment variable will be taken. - Parameter is ignored if ``backend`` is passed. - - .. deprecated :: 0.4.4 - - Instead, use: - - .. code :: python3 - - from neptune.legacy import Session - session = Session.with_default_backend(api_token='...') - - proxies (:obj:`str`, optional, default is ``None``): - Argument passed to HTTP calls made via the `Requests `_ library. - For more information see their proxies - `section `_. - Parameter is ignored if ``backend`` is passed. - - .. deprecated :: 0.4.4 - - Instead, use: - - .. code :: python3 - - from neptune.legacy import Session, HostedNeptuneBackendApiClient - session = Session(backend=HostedNeptuneBackendApiClient(proxies=...)) - - Examples: - - Create session, assuming you have created an environment variable ``NEPTUNE_API_TOKEN`` - - .. code:: python3 - - from neptune.legacy import Session - session = Session.with_default_backend() - - Create session and pass ``api_token`` - - .. code:: python3 - - from neptune.legacy import Session - session = Session.with_default_backend(api_token='...') - - Create an offline session - - .. code:: python3 - - from neptune.legacy import Session, OfflineApiClient - session = Session(backend=OfflineApiClient()) - - """ - - @legacy_client_deprecation - def __init__(self, api_token=None, proxies=None, backend=None): - self._backend = backend - - if self._backend is None: - _logger.warning( - "WARNING: Instantiating Session without specifying a backend is deprecated " - "and will be removed in future versions. For current behaviour " - "use `neptune.init(...)` or `Session.with_default_backend(...)" - ) - - self._backend = HostedNeptuneBackendApiClient(api_token, proxies) - - @classmethod - def with_default_backend(cls, api_token=None, proxies=None): - """The simplest way to instantiate a ``Session``. - - Args: - api_token (:obj:`str`): - User's API token. - If ``None``, the value of ``NEPTUNE_API_TOKEN`` environment variable will be taken. - - proxies (:obj:`str`, optional, default is ``None``): - Argument passed to HTTP calls made via the `Requests `_ - library. - For more information see their proxies - `section `_. - - Examples: - - .. code :: python3 - - from neptune.legacy import Session - session = Session.with_default_backend() - - """ - return cls(backend=HostedNeptuneBackendApiClient(api_token=api_token, proxies=proxies)) - - def get_project(self, project_qualified_name): - """Get a project with given ``project_qualified_name``. - - In order to access experiments data one needs to get a :class:`~neptune.projects.Project` object first. - This method gives you the ability to do that. - - Args: - project_qualified_name (:obj:`str`): - Qualified name of a project in a form of ``namespace/project_name``. - If ``None``, the value of ``NEPTUNE_PROJECT`` environment variable will be taken. - - Returns: - :class:`~neptune.projects.Project` object. - - Raise: - :class:`~neptune.api_exceptions.ProjectNotFound`: When a project with given name does not exist. - - Examples: - - .. code:: python3 - - # Create a Session instance - from neptune.sessions import Session - session = Session() - - # Get a project by it's ``project_qualified_name``: - my_project = session.get_project('namespace/project_name') - - """ - project_qualified_name = assure_project_qualified_name(project_qualified_name) - - return self._backend.get_project(project_qualified_name) - - def get_projects(self, namespace): - """Get all projects that you have permissions to see in given workspace. - - | This method gets you all available projects names and their - corresponding :class:`~neptune.projects.Project` objects. - | Both private and public projects may be returned for the workspace. - If you have role in private project, it is included. - | You can retrieve all the public projects that belong to any user or workspace, - as long as you know their username or workspace name. - - Args: - namespace (:obj:`str`): It can either be name of the workspace or username. - - Returns: - :obj:`OrderedDict` - | **keys** are ``project_qualified_name`` that is: *'workspace/project_name'* - | **values** are corresponding :class:`~neptune.projects.Project` objects. - - Raises: - `WorkspaceNotFound`: When the given workspace does not exist. - - Examples: - - .. code:: python3 - - # create Session - from neptune.sessions import Session - session = Session() - - # Now, you can list all the projects available for a selected namespace. - # You can use `YOUR_NAMESPACE` which is your workspace or user name. - # You can also list public projects created in other workspaces. - # For example you can use the `neptune-ai` namespace. - - session.get_projects('neptune-ai') - - # Example output: - # OrderedDict([('neptune-ai/credit-default-prediction', - # Project(neptune-ai/credit-default-prediction)), - # ('neptune-ai/GStore-Customer-Revenue-Prediction', - # Project(neptune-ai/GStore-Customer-Revenue-Prediction)), - # ('neptune-ai/human-protein-atlas', - # Project(neptune-ai/human-protein-atlas)), - # ('neptune-ai/Ships', - # Project(neptune-ai/Ships)), - # ('neptune-ai/Mapping-Challenge', - # Project(neptune-ai/Mapping-Challenge)) - # ]) - """ - - projects = [ - Project(self._backend.create_leaderboard_backend(p), p.id, namespace, p.name) - for p in self._backend.get_projects(namespace) - ] - return OrderedDict((p.full_id, p) for p in projects) diff --git a/src/neptune/legacy/utils.py b/src/neptune/legacy/utils.py deleted file mode 100644 index 84ad17154..000000000 --- a/src/neptune/legacy/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2022, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# flake8: noqa -from neptune.common.backends.utils import with_api_exceptions_handler -from neptune.common.utils import ( - IS_MACOS, - IS_WINDOWS, - NoopObject, - align_channels_on_x, - as_list, - assure_directory_exists, - assure_project_qualified_name, - discover_git_repo_location, - file_contains, - get_channel_name_stems, - get_git_info, - glob, - in_docker, - is_float, - is_ipython, - is_nan_or_inf, - is_notebook, - map_keys, - map_values, - merge_dataframes, - update_session_proxies, - validate_notebook_path, -) diff --git a/tests/e2e/standard/test_legacy_client.py b/tests/e2e/standard/test_legacy_client.py deleted file mode 100644 index 73f46f318..000000000 --- a/tests/e2e/standard/test_legacy_client.py +++ /dev/null @@ -1,129 +0,0 @@ -from datetime import datetime - -import numpy -from PIL import Image - -from neptune.legacy import Session -from tests.e2e.base import fake -from tests.e2e.utils import tmp_context - - -class TestLegacyClient: - def test_experiment_creation(self, environment): - session = Session(api_token=environment.user_token) - project = session.get_project(project_qualified_name=environment.project) - project.create_experiment() - - def test_tags_operations(self, environment): - session = Session(api_token=environment.user_token) - project = session.get_project(project_qualified_name=environment.project) - experiment = project.create_experiment( - tags=["initial tag 1", "initial tag 2"], - ) - experiment.append_tags("tag1") - experiment.append_tag(["tag2_to_remove", "tag3"]) - - assert set(experiment.get_tags()) == { - "initial tag 1", - "initial tag 2", - "tag1", - "tag2_to_remove", - "tag3", - } - - def test_properties(self, environment): - session = Session(api_token=environment.user_token) - project = session.get_project(project_qualified_name=environment.project) - experiment = project.create_experiment( - params={ - "init_text_parameter": "some text", - "init_number parameter": 42, - "init_list": [1, 2, 3], - "init_datetime": datetime.now(), - }, - ) - experiment.set_property("prop", "some text") - experiment.set_property("prop_number", 42) - experiment.set_property("nested/prop", 42) - experiment.set_property("prop_to_del", 42) - experiment.set_property("prop_list", [1, 2, 3]) - - experiment.set_property("prop_datetime", datetime.now()) - experiment.remove_property("prop_to_del") - - properties = experiment.get_properties() - assert properties["prop"] == "some text" - assert properties["prop_number"] == "42" - assert properties["nested/prop"] == "42" - assert "prop_to_del" not in properties - - def test_log_operations(self, environment): - # given - session = Session(api_token=environment.user_token) - project = session.get_project(project_qualified_name=environment.project) - experiment = project.create_experiment() - - # when - experiment.log_metric("m1", 1) - experiment.log_metric("m1", 2) - experiment.log_metric("m1", 3) - experiment.log_metric("m1", 2) - experiment.log_metric("nested/m1", 1) - - # and - experiment.log_text("m2", "a") - experiment.log_text("m2", "b") - experiment.log_text("m2", "c") - - # and - with tmp_context(): - filename = fake.file_name(extension="png") - - tmp = numpy.random.rand(100, 100, 3) * 255 - im = Image.fromarray(tmp.astype("uint8")).convert("RGBA") - im.save(filename) - - experiment.log_image("g_img", filename, image_name="name", description="desc") - experiment.log_image("g_img", filename) - - # then - logs = experiment.get_logs() - assert "m1" in logs - assert "m2" in logs - assert "nested/m1" in logs - assert "g_img" in logs - - def test_files_operations(self, environment): - # given - session = Session(api_token=environment.user_token) - project = session.get_project(project_qualified_name=environment.project) - experiment = project.create_experiment() - - # when - # image - with tmp_context(): - filename = fake.file_name(extension="png") - - tmp = numpy.random.rand(100, 100, 3) * 255 - im = Image.fromarray(tmp.astype("uint8")).convert("RGBA") - im.save(filename) - - experiment.send_image("image", filename, name="name", description="desc") - - # artifact - with tmp_context(): - filename = fake.file_name(extension="png") - - with open(filename, "wb") as file: - file.write(fake.sentence().encode("utf-8")) - - experiment.send_artifact(filename) - experiment.log_artifact(filename, destination="dir/text file artifact") - - with open(filename, mode="r") as f: - experiment.send_artifact(f, destination="file stream.txt") - - experiment.log_artifact(filename, destination="dir to delete/art1") - experiment.log_artifact(filename, destination="dir to delete/art2") - - experiment.delete_artifacts("dir to delete/art1") diff --git a/tests/unit/neptune/legacy/__init__.py b/tests/unit/neptune/legacy/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/neptune/legacy/api_models.py b/tests/unit/neptune/legacy/api_models.py deleted file mode 100644 index 9abaee017..000000000 --- a/tests/unit/neptune/legacy/api_models.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from collections import namedtuple - -ApiParameter = namedtuple("ApiParameter", ["id", "name", "parameterType", "value"]) diff --git a/tests/unit/neptune/legacy/api_objects_factory.py b/tests/unit/neptune/legacy/api_objects_factory.py deleted file mode 100644 index 3ae27f0b9..000000000 --- a/tests/unit/neptune/legacy/api_objects_factory.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from random import ( - randint, - uniform, -) - -from mock import MagicMock - -from tests.unit.neptune.legacy.random_utils import ( - a_string, - a_timestamp, - a_uuid_string, -) - - -def a_project(): - project = MagicMock() - project.id = a_uuid_string() - project.name = a_string() - return project - - -def an_invited_project_member(): - invitation_info = MagicMock() - invitation_info.id = a_uuid_string() - invitation_info.email = a_string() + "@example.com" - - project_member = MagicMock() - project_member.invitationInfo = invitation_info - project_member.registeredMemberInfo = None - project_member.role = "member" - - return project_member - - -def a_registered_project_member(username=None): - if username is None: - username = a_string() - - registered_member_info = MagicMock() - registered_member_info.avatarSource = "default" - registered_member_info.avatarUrl = "" - registered_member_info.firstName = a_string() - registered_member_info.lastName = a_string() - registered_member_info.username = username - - project_member = MagicMock() - project_member.invitationInfo = None - project_member.registeredMemberInfo = registered_member_info - project_member.role = "manager" - - return project_member - - -def an_experiment_states( - creating=None, - waiting=None, - initializing=None, - running=None, - cleaning=None, - aborted=None, - crashed=None, - failed=None, - succeeded=None, - not_responding=None, -): - def random_state_count(): - return randint(1, 100) - - experiment_states = MagicMock() - experiment_states.creating = creating or random_state_count() - experiment_states.waiting = waiting or random_state_count() - experiment_states.initializing = initializing or random_state_count() - experiment_states.running = running or random_state_count() - experiment_states.cleaning = cleaning or random_state_count() - experiment_states.aborted = aborted or random_state_count() - experiment_states.crashed = crashed or random_state_count() - experiment_states.failed = failed or random_state_count() - experiment_states.succeeded = succeeded or random_state_count() - experiment_states.notResponding = not_responding or random_state_count() - return experiment_states - - -def a_property(): - p = MagicMock() - p.key = a_string() - p.value = a_string() - return p - - -def a_parameter(): - p = MagicMock() - p.id = a_uuid_string() - p.name = a_string() - p.parameterType = "double" - p.value = str(uniform(-100, 100)) - return p - - -def a_channel_value(): - cv = MagicMock() - cv.channelId = a_uuid_string() - cv.channelName = a_string() - cv.channelType = "numeric" - cv.x = uniform(1, 100) - cv.y = str(uniform(1, 100)) - return cv - - -def an_experiment_leaderboard_entry_dto(): - entry = MagicMock() - entry.entryType = "experiment" - entry.id = a_uuid_string() - entry.shortId = a_string() - entry.projectId = a_uuid_string() - entry.state = "succeeded" - entry.experimentStates = an_experiment_states(succeeded=1) - entry.responding = True - entry.name = a_string() - entry.organizationName = a_string() - entry.projectName = a_string() - entry.description = a_string() - entry.timeOfCreation = a_timestamp() - entry.timeOfCompletion = a_timestamp() - entry.runningTime = randint(1, 1000) - entry.owner = a_string() - entry.size = randint(1, 1000) - entry.tags = [a_string(), a_string()] - entry.environment = a_string() - entry.workerType = a_string() - entry.hostname = a_string() - entry.sourceSize = randint(1, 1000) - entry.sourceMd5 = a_string() - entry.commitId = a_string() - entry.properties = [a_property(), a_property()] - entry.parameters = [a_parameter(), a_parameter()] - entry.channelsLastValues = [a_channel_value(), a_channel_value()] - entry.trashed = False - entry.deleted = False - entry.isBestExperiment = False - return entry diff --git a/tests/unit/neptune/legacy/assertions.py b/tests/unit/neptune/legacy/assertions.py deleted file mode 100644 index e37865941..000000000 --- a/tests/unit/neptune/legacy/assertions.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class AssertionExtensions(object): - def assert_int_greater_than(self, x, y): - self.assertGreater(x, y) - self.assertNotEqual(float, type(x)) - - def assert_int_greater_or_equal(self, x, y): - self.assertGreaterEqual(x, y) - self.assertNotEqual(float, type(x)) - - def assert_float_greater_than(self, x, y): - self.assertGreater(x, y) - self.assertEqual(float, type(x)) diff --git a/tests/unit/neptune/legacy/experiments_object_factory.py b/tests/unit/neptune/legacy/experiments_object_factory.py deleted file mode 100644 index 5147194c4..000000000 --- a/tests/unit/neptune/legacy/experiments_object_factory.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import random - -from munch import Munch - -from tests.unit.neptune.legacy.random_utils import ( - a_string, - a_uuid_string, -) - - -def a_channel(): - x = random.randint(0, 100) - return Munch( - id=a_uuid_string(), - name=a_string(), - channelType="text", - lastX=x, - x=x, - y=a_string(), - ) diff --git a/tests/unit/neptune/legacy/http_objects_factory.py b/tests/unit/neptune/legacy/http_objects_factory.py deleted file mode 100644 index 34edda2ba..000000000 --- a/tests/unit/neptune/legacy/http_objects_factory.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from mock import MagicMock - -from tests.unit.neptune.legacy.random_utils import a_string - - -def a_request(): - request = MagicMock() - request.method = "post" - request.url = "http://{}.com".format(a_string()) - request.headers = {a_string(): a_string()} - request.body = a_string() - return request diff --git a/tests/unit/neptune/legacy/internal/__init__.py b/tests/unit/neptune/legacy/internal/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/backends/__init__.py b/tests/unit/neptune/legacy/internal/backends/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/backends/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/backends/test_hosted_neptune_backend.py b/tests/unit/neptune/legacy/internal/backends/test_hosted_neptune_backend.py deleted file mode 100644 index 5dcc413e2..000000000 --- a/tests/unit/neptune/legacy/internal/backends/test_hosted_neptune_backend.py +++ /dev/null @@ -1,180 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import socket -import unittest - -import mock -from mock import MagicMock - -from neptune.legacy.exceptions import ( - CannotResolveHostname, - DeprecatedApiToken, - UnsupportedClientVersion, -) -from neptune.legacy.internal.api_clients import HostedNeptuneBackendApiClient - -API_TOKEN = ( - "eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLnN0YWdlLm5lcHR1bmUuYWkiLCJ" - "hcGlfa2V5IjoiOTJhNzhiOWQtZTc3Ni00ODlhLWI5YzEtNzRkYmI1ZGVkMzAyIn0=" -) - - -@mock.patch( - "neptune.legacy.internal.api_clients.hosted_api_clients.hosted_backend_api_client.NeptuneAuthenticator", - new=MagicMock, -) -class TestHostedNeptuneBackend(unittest.TestCase): - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch("neptune.__version__", "0.5.13") - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def test_min_compatible_version_ok(self, swagger_client_factory): - # given - self._get_swagger_client_mock(swagger_client_factory, min_compatible="0.5.13") - - # expect - HostedNeptuneBackendApiClient(api_token=API_TOKEN) - - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch("neptune.__version__", "0.5.13") - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def test_min_compatible_version_fail(self, swagger_client_factory): - # given - self._get_swagger_client_mock(swagger_client_factory, min_compatible="0.5.14") - - # expect - with self.assertRaises(UnsupportedClientVersion) as ex: - HostedNeptuneBackendApiClient(api_token=API_TOKEN) - - self.assertTrue("Please install neptune-client>=0.5.14" in str(ex.exception)) - - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch("neptune.__version__", "0.5.13") - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def test_max_compatible_version_ok(self, swagger_client_factory): - # given - self._get_swagger_client_mock(swagger_client_factory, max_compatible="0.5.13") - - # expect - HostedNeptuneBackendApiClient(api_token=API_TOKEN) - - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch("neptune.__version__", "0.5.13") - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def test_max_compatible_version_fail(self, swagger_client_factory): - # given - self._get_swagger_client_mock(swagger_client_factory, max_compatible="0.5.12") - - # expect - with self.assertRaises(UnsupportedClientVersion) as ex: - HostedNeptuneBackendApiClient(api_token=API_TOKEN) - - self.assertTrue("Please install neptune-client==0.5.12" in str(ex.exception)) - - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch("neptune.legacy.internal.api_clients.credentials.os.getenv", return_value=API_TOKEN) - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def test_should_take_default_credentials_from_env(self, env, swagger_client_factory): - # given - self._get_swagger_client_mock(swagger_client_factory) - - # when - backend = HostedNeptuneBackendApiClient() - - # then - self.assertEqual(API_TOKEN, backend.credentials.api_token) - - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def test_should_accept_given_api_token(self, swagger_client_factory): - # given - self._get_swagger_client_mock(swagger_client_factory) - - # when - session = HostedNeptuneBackendApiClient(API_TOKEN) - - # then - self.assertEqual(API_TOKEN, session.credentials.api_token) - - @mock.patch("socket.gethostbyname") - def test_deprecated_token(self, gethostname_mock): - # given - token = ( - "eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkuc3RhZ2UubmVwdHVuZS5tbCIsImF" - "waV9rZXkiOiI5ODM4ZDk1NC00MDAzLTExZTktYmY1MC0yMzE5ODM1NWRhNjYifQ==" - ) - - gethostname_mock.side_effect = socket.gaierror - - # expect - with self.assertRaises(DeprecatedApiToken): - HostedNeptuneBackendApiClient(token) - - @mock.patch("socket.gethostbyname") - def test_cannot_resolve_host(self, gethostname_mock): - # given - token = ( - "eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkuc3RhZ2UubmVwdHVuZS5tbCIsImFwaV91cmwiOiJodHRwczovL3VpLn" - "N0YWdlLm5lcHR1bmUuYWkiLCJhcGlfa2V5IjoiOTgzOGQ5NTQtNDAwMy0xMWU5LWJmNTAtMjMxOTgzNTVkYTY2In0=" - ) - - gethostname_mock.side_effect = socket.gaierror - - # expect - with self.assertRaises(CannotResolveHostname): - HostedNeptuneBackendApiClient(token) - - @staticmethod - def _get_swagger_client_mock( - swagger_client_factory, - min_recommended=None, - min_compatible=None, - max_compatible=None, - ): - py_lib_versions = type("py_lib_versions", (object,), {})() - setattr(py_lib_versions, "minRecommendedVersion", min_recommended) - setattr(py_lib_versions, "minCompatibleVersion", min_compatible) - setattr(py_lib_versions, "maxCompatibleVersion", max_compatible) - - artifacts = type("artifacts", (object,), {})() - setattr(artifacts, "enabled", True) - - multipart_upload = type("multiPartUpload", (object,), {})() - setattr(multipart_upload, "enabled", True) - setattr(multipart_upload, "minChunkSize", 5242880) - setattr(multipart_upload, "maxChunkSize", 1073741824) - setattr(multipart_upload, "maxChunkCount", 1000) - setattr(multipart_upload, "maxSinglePartSize", 5242880) - - client_config = type("client_config_response_result", (object,), {})() - setattr(client_config, "pyLibVersions", py_lib_versions) - setattr(client_config, "artifacts", artifacts) - setattr(client_config, "multiPartUpload", multipart_upload) - setattr(client_config, "apiUrl", None) - setattr(client_config, "applicationUrl", None) - - swagger_client = MagicMock() - swagger_client.api.getClientConfig.return_value.response.return_value.result = client_config - swagger_client_factory.return_value = swagger_client - - return swagger_client - - -class SomeClass(object): - pass - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/backends/test_noop_object.py b/tests/unit/neptune/legacy/internal/backends/test_noop_object.py deleted file mode 100644 index 96b637755..000000000 --- a/tests/unit/neptune/legacy/internal/backends/test_noop_object.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -from neptune.common.utils import NoopObject - - -class TestHostedNeptuneObject(unittest.TestCase): - def test_builtin_fields_not_overriden(self): - # given - objectUnderTest = NoopObject() - - # then - self.assertIsInstance(objectUnderTest.__class__, type) - - def test_attributes_fall_back_on_getattr(self): - # given - objectUnderTest = NoopObject() - some_value = 42 - - # then - self.assertEqual(objectUnderTest.foo, objectUnderTest) - self.assertEqual(objectUnderTest.bar(some_value), objectUnderTest) - - def test_attributes_fall_back_on_getitem(self): - # given - objectUnderTest = NoopObject() - some_value = 42 - - # then - self.assertEqual(objectUnderTest["foo"], objectUnderTest) - self.assertEqual(objectUnderTest["bar"](some_value), objectUnderTest) - - def test_noop_object_callable(self): - # given - objectUnderTest = NoopObject() - some_value = 42 - - # then - self.assertEqual(objectUnderTest(some_value), objectUnderTest) - - def test_noop_object_context_manager(self): - # given - objectUnderTest = NoopObject() - - # when - with objectUnderTest as e: - - # then - e(42) diff --git a/tests/unit/neptune/legacy/internal/channels/__init__.py b/tests/unit/neptune/legacy/internal/channels/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/channels/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/channels/test_channels_values_sender.py b/tests/unit/neptune/legacy/internal/channels/test_channels_values_sender.py deleted file mode 100644 index 6c976d94d..000000000 --- a/tests/unit/neptune/legacy/internal/channels/test_channels_values_sender.py +++ /dev/null @@ -1,318 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import unicode_literals - -import threading -import time -import unittest - -import mock - -from neptune.legacy.internal.channels.channels import ( - ChannelIdWithValues, - ChannelNamespace, - ChannelType, - ChannelValue, -) -from neptune.legacy.internal.channels.channels_values_sender import ( - ChannelsValuesSender, - ChannelsValuesSendingThread, -) -from tests.unit.neptune.legacy.experiments_object_factory import a_channel - - -class TestChannelsValuesSender(unittest.TestCase): - _TS = time.time() - - _EXPERIMENT = mock.MagicMock() - - _NUMERIC_CHANNEL = a_channel() - _NUMERIC_CHANNEL.update(name="1numeric", channelType=ChannelType.NUMERIC.value) - - _TEXT_CHANNEL = a_channel() - _TEXT_CHANNEL.update(name="2text", channelType=ChannelType.TEXT.value) - - _IMAGE_CHANNEL = a_channel() - _IMAGE_CHANNEL.update(name="3image", channelType=ChannelType.IMAGE.value) - - _OTHER_CHANNEL = a_channel() - - _CHANNELS = {c.name: c for c in [_NUMERIC_CHANNEL, _TEXT_CHANNEL, _IMAGE_CHANNEL, _OTHER_CHANNEL]} - - _BATCH_SIZE = ChannelsValuesSendingThread._MAX_VALUES_BATCH_LENGTH - _IMAGES_BATCH_IMAGE_SIZE = ChannelsValuesSendingThread._MAX_IMAGE_VALUES_BATCH_SIZE / 3 - _IMAGES_BATCH_SIZE = 3 - - def setUp(self): - self._EXPERIMENT._get_channels.return_value = self._CHANNELS - - def create_channel_fun(channel_name, channel_type, channel_namespace=ChannelNamespace.USER): - if channel_name == self._NUMERIC_CHANNEL.name: - return self._NUMERIC_CHANNEL - if channel_name == self._TEXT_CHANNEL.name: - return self._TEXT_CHANNEL - if channel_name == self._IMAGE_CHANNEL.name: - return self._IMAGE_CHANNEL - if channel_name == self._OTHER_CHANNEL.name: - return self._OTHER_CHANNEL - raise ValueError("unexpected channel") - - self._EXPERIMENT._create_channel = mock.MagicMock(side_effect=create_channel_fun) - - def tearDown(self): - self._EXPERIMENT.reset_mock() - - def test_send_values_on_join(self): - # given - channel_value = ChannelValue(x=1, y="value", ts=self._TS) - # and - channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT) - - # when - channels_values_sender.send(self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value) - # and - channels_values_sender.join() - - # then - self._EXPERIMENT._send_channels_values.assert_called_with( - [ - ChannelIdWithValues( - channel_id=self._TEXT_CHANNEL.id, - channel_name=self._TEXT_CHANNEL.name, - channel_type=self._TEXT_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=[channel_value], - ) - ] - ) - - def test_send_values_in_multiple_batches(self): - # given - channels_values = [ - ChannelValue(x=i, y="value{}".format(i), ts=self._TS + i) for i in range(0, self._BATCH_SIZE * 3) - ] - # and - channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT) - - # when - for channel_value in channels_values: - channels_values_sender.send(self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value) - # and - channels_values_sender.join() - - # then - self.assertEqual( - self._EXPERIMENT._send_channels_values.mock_calls, - [ - mock.call._send_channels_values( - [ - ChannelIdWithValues( - channel_id=self._TEXT_CHANNEL.id, - channel_name=self._TEXT_CHANNEL.name, - channel_type=self._TEXT_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=channels_values[0 : self._BATCH_SIZE], - ) - ] - ), - mock.call._send_channels_values( - [ - ChannelIdWithValues( - channel_id=self._TEXT_CHANNEL.id, - channel_name=self._TEXT_CHANNEL.name, - channel_type=self._TEXT_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=channels_values[self._BATCH_SIZE : self._BATCH_SIZE * 2], - ) - ] - ), - mock.call._send_channels_values( - [ - ChannelIdWithValues( - channel_id=self._TEXT_CHANNEL.id, - channel_name=self._TEXT_CHANNEL.name, - channel_type=self._TEXT_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=channels_values[self._BATCH_SIZE * 2 : self._BATCH_SIZE * 3], - ) - ] - ), - ], - ) - - def test_send_images_in_smaller_batches(self): - # and - value = "base64Image==" - channels_values = [ - ChannelValue( - x=i, - y={"image_value": {"data": value + value * int(self._IMAGES_BATCH_IMAGE_SIZE / (len(value)))}}, - ts=self._TS + i, - ) - for i in range(0, self._IMAGES_BATCH_SIZE * 3) - ] - # and - channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT) - - # when - for channel_value in channels_values: - channels_values_sender.send(self._IMAGE_CHANNEL.name, self._IMAGE_CHANNEL.channelType, channel_value) - # and - channels_values_sender.join() - - # then - self.assertEqual( - self._EXPERIMENT._send_channels_values.mock_calls, - [ - mock.call._send_channels_values( - [ - ChannelIdWithValues( - channel_id=self._IMAGE_CHANNEL.id, - channel_name=self._IMAGE_CHANNEL.name, - channel_type=self._IMAGE_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=channels_values[0 : self._IMAGES_BATCH_SIZE], - ) - ] - ), - mock.call._send_channels_values( - [ - ChannelIdWithValues( - channel_id=self._IMAGE_CHANNEL.id, - channel_name=self._IMAGE_CHANNEL.name, - channel_type=self._IMAGE_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=channels_values[self._IMAGES_BATCH_SIZE : self._IMAGES_BATCH_SIZE * 2], - ) - ] - ), - mock.call._send_channels_values( - [ - ChannelIdWithValues( - channel_id=self._IMAGE_CHANNEL.id, - channel_name=self._IMAGE_CHANNEL.name, - channel_type=self._IMAGE_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=channels_values[self._IMAGES_BATCH_SIZE * 2 :], - ) - ] - ), - ], - ) - - def test_send_values_from_multiple_channels(self): - # given - numeric_values = [ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)] - - text_values = [ChannelValue(x=i, y="text", ts=self._TS + i) for i in range(0, 3)] - - image_values = [ - ChannelValue(x=i, y={"image_value": {"data": "base64Image=="}}, ts=self._TS + i) for i in range(0, 3) - ] - # and - channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT) - - # when - for channel_value in numeric_values: - channels_values_sender.send( - self._NUMERIC_CHANNEL.name, - self._NUMERIC_CHANNEL.channelType, - channel_value, - ) - - for channel_value in text_values: - channels_values_sender.send(self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value) - - for channel_value in image_values: - channels_values_sender.send(self._IMAGE_CHANNEL.name, self._IMAGE_CHANNEL.channelType, channel_value) - - # and - channels_values_sender.join() - - # then - (args, _) = self._EXPERIMENT._send_channels_values.call_args - self.assertEqual(len(args), 1) - self.assertEqual( - sorted(args[0]), - sorted( - [ - ChannelIdWithValues( - channel_id=self._NUMERIC_CHANNEL.id, - channel_name=self._NUMERIC_CHANNEL.name, - channel_type=self._NUMERIC_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=numeric_values, - ), - ChannelIdWithValues( - channel_id=self._TEXT_CHANNEL.id, - channel_name=self._NUMERIC_CHANNEL.name, - channel_type=self._NUMERIC_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=text_values, - ), - ChannelIdWithValues( - channel_id=self._IMAGE_CHANNEL.id, - channel_name=self._NUMERIC_CHANNEL.name, - channel_type=self._NUMERIC_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=image_values, - ), - ] - ), - ) - - @mock.patch( - "neptune.legacy.internal.channels.channels_values_sender.ChannelsValuesSendingThread._SLEEP_TIME", - 0.1, - ) - def test_send_when_waiting_for_next_value_timed_out(self): - # given - numeric_values = [ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)] - - # and - semaphore = threading.Semaphore(0) - self._EXPERIMENT._send_channels_values.side_effect = lambda _: semaphore.release() - - # and - channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT) - - # when - for channel_value in numeric_values: - channels_values_sender.send( - self._NUMERIC_CHANNEL.name, - self._NUMERIC_CHANNEL.channelType, - channel_value, - ) - - # then - semaphore.acquire() - self._EXPERIMENT._send_channels_values.assert_called_with( - [ - ChannelIdWithValues( - channel_id=self._NUMERIC_CHANNEL.id, - channel_name=self._NUMERIC_CHANNEL.name, - channel_type=self._NUMERIC_CHANNEL.channelType, - channel_namespace=ChannelNamespace.USER, - channel_values=numeric_values, - ) - ] - ) - - # and - self._EXPERIMENT._send_channels_values.reset_mock() - channels_values_sender.join() - # and - self._EXPERIMENT._send_channels_values.assert_not_called() diff --git a/tests/unit/neptune/legacy/internal/hardware/__init__.py b/tests/unit/neptune/legacy/internal/hardware/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/hardware/gauges/__init__.py b/tests/unit/neptune/legacy/internal/hardware/gauges/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/gauges/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/hardware/gauges/gauges_fixture.py b/tests/unit/neptune/legacy/internal/hardware/gauges/gauges_fixture.py deleted file mode 100644 index af8613e0f..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/gauges/gauges_fixture.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from mock import MagicMock - -from neptune.common.hardware.gauges.cpu import SystemCpuUsageGauge -from neptune.common.hardware.gauges.gauge_factory import ( - GaugeFactory, - SystemMemoryUsageGauge, -) -from neptune.common.hardware.gauges.gpu import ( - GpuMemoryGauge, - GpuUsageGauge, -) - - -class GaugesFixture(object): - def __init__(self): - self.gauge_factory = MagicMock(spec_set=GaugeFactory) - - self.cpu_gauge_value = 1.0 - self.memory_gauge_value = 2.0 - - self.gpu0_usage_gauge_value = 3.0 - self.gpu1_usage_gauge_value = 4.0 - - self.gpu0_memory_gauge_value = 5.0 - self.gpu1_memory_gauge_value = 6.0 - - cpu_gauge = MagicMock(wraps=SystemCpuUsageGauge()) - cpu_gauge.value.return_value = self.cpu_gauge_value - self.gauge_factory.create_cpu_usage_gauge.return_value = cpu_gauge - - ram_gauge = MagicMock(wraps=SystemMemoryUsageGauge()) - ram_gauge.value.return_value = self.memory_gauge_value - self.gauge_factory.create_memory_usage_gauge.return_value = ram_gauge - - gpu_usage_gauge_0 = MagicMock(wraps=GpuUsageGauge(card_index=0)) - gpu_usage_gauge_0.value.return_value = self.gpu0_usage_gauge_value - - gpu_usage_gauge_2 = MagicMock(wraps=GpuUsageGauge(card_index=2)) - gpu_usage_gauge_2.value.return_value = self.gpu1_usage_gauge_value - - self.gauge_factory.create_gpu_usage_gauge.side_effect = ( - lambda card_index: gpu_usage_gauge_0 if card_index == 0 else gpu_usage_gauge_2 - ) - - gpu_memory_gauge_0 = MagicMock(wraps=GpuMemoryGauge(card_index=0)) - gpu_memory_gauge_0.value.return_value = self.gpu0_memory_gauge_value - - gpu_memory_gauge_2 = MagicMock(wraps=GpuMemoryGauge(card_index=2)) - gpu_memory_gauge_2.value.return_value = self.gpu1_memory_gauge_value - self.gauge_factory.create_gpu_memory_gauge.side_effect = ( - lambda card_index: gpu_memory_gauge_0 if card_index == 0 else gpu_memory_gauge_2 - ) diff --git a/tests/unit/neptune/legacy/internal/hardware/gauges/test_cpu_gauges.py b/tests/unit/neptune/legacy/internal/hardware/gauges/test_cpu_gauges.py deleted file mode 100644 index e01be5a4e..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/gauges/test_cpu_gauges.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import time -import unittest - -from neptune.common.hardware.gauges.cpu import ( - CGroupCpuUsageGauge, - SystemCpuUsageGauge, -) - - -class TestCpuGauges(unittest.TestCase): - def test_system_cpu_gauge(self): - # given - gauge = SystemCpuUsageGauge() - - # when - cpu_usage = gauge.value() - - # then - self.assertGreater(cpu_usage, 0.0) - self.assertLessEqual(cpu_usage, 100.0) - self.assertEqual(float, type(cpu_usage)) - - @unittest.skip("We do not have docker infrastructure to test cgroupsV1") - def test_cgroup_cpu_gauge_returns_zero_on_first_measurement(self): - # given - gauge = CGroupCpuUsageGauge() - - # when - cpu_usage = gauge.value() - - # then - self.assertEqual(0.0, cpu_usage) - self.assertEqual(float, type(cpu_usage)) - - @unittest.skip("We do not have docker infrastructure to test cgroupsV1") - def test_cgroup_cpu_gauge_measurement(self): - # given - gauge = CGroupCpuUsageGauge() - # and - gauge.value() - time.sleep(0.1) - - # when - cpu_usage = gauge.value() - - # then - self.assertGreater(cpu_usage, 0.0) - self.assertLessEqual(cpu_usage, 100.0) - self.assertEqual(float, type(cpu_usage)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/hardware/gauges/test_gpu_gauges.py b/tests/unit/neptune/legacy/internal/hardware/gauges/test_gpu_gauges.py deleted file mode 100644 index c6c6533b0..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/gauges/test_gpu_gauges.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -from mock import ( - MagicMock, - patch, -) - -from neptune.common.hardware.constants import BYTES_IN_ONE_GB -from neptune.common.hardware.gauges.gpu import ( - GpuMemoryGauge, - GpuUsageGauge, -) - - -@patch("neptune.common.hardware.gpu.gpu_monitor.nvmlInit", MagicMock()) -class TestGPUGauges(unittest.TestCase): - def setUp(self): - self.card_index = 2 - self.gpu_card_handle = MagicMock() - - patcher = patch("neptune.common.hardware.gpu.gpu_monitor.nvmlDeviceGetHandleByIndex") - nvmlDeviceGetHandleByIndex = patcher.start() - nvmlDeviceGetHandleByIndex.side_effect = ( - lambda card_index: self.gpu_card_handle if card_index == self.card_index else None - ) - self.addCleanup(patcher.stop) - - @patch("neptune.common.hardware.gpu.gpu_monitor.nvmlDeviceGetUtilizationRates") - def test_gpu_usage_gauge(self, nvmlDeviceGetMemoryInfo): - # given - gauge = GpuUsageGauge(card_index=self.card_index) - # and - util_info = MagicMock() - util_info.gpu = 40 - nvmlDeviceGetMemoryInfo.side_effect = lambda handle: util_info if handle == self.gpu_card_handle else None - - # when - usage_percent = gauge.value() - - # then - self.assertEqual(40.0, usage_percent) - self.assertEqual(float, type(usage_percent)) - - @patch("neptune.common.hardware.gpu.gpu_monitor.nvmlDeviceGetMemoryInfo") - def test_gpu_memory_gauge(self, nvmlDeviceGetMemoryInfo): - # given - gauge = GpuMemoryGauge(card_index=self.card_index) - # and - memory_info = MagicMock() - memory_info.used = 3 * BYTES_IN_ONE_GB - nvmlDeviceGetMemoryInfo.side_effect = lambda handle: memory_info if handle == self.gpu_card_handle else None - - # when - memory_gb = gauge.value() - - # then - self.assertEqual(memory_gb, 3.0) - self.assertEqual(float, type(memory_gb)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/hardware/gauges/test_memory_gauges.py b/tests/unit/neptune/legacy/internal/hardware/gauges/test_memory_gauges.py deleted file mode 100644 index 41ab7dfd4..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/gauges/test_memory_gauges.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -import psutil - -from neptune.common.hardware.gauges.gauge_factory import ( - CGroupMemoryUsageGauge, - SystemMemoryUsageGauge, -) - - -class TestMemoryGauges(unittest.TestCase): - def setUp(self): - self.system_memory_gb = psutil.virtual_memory().total / float(2**30) - - def test_system_memory_gauge(self): - # given - gauge = SystemMemoryUsageGauge() - - # when - memory_gb = gauge.value() - - # then - self.assertGreater(memory_gb, 0) - self.assertLess(memory_gb, self.system_memory_gb) - self.assertEqual(float, type(memory_gb)) - - @unittest.skip("We do not have docker infrastructure to test cgroupsV1") - def test_cgroup_memory_gauge(self): - # given - gauge = CGroupMemoryUsageGauge() - - # when - memory_gb = gauge.value() - - # then - self.assertGreater(memory_gb, 0) - self.assertLess(memory_gb, self.system_memory_gb) - self.assertEqual(float, type(memory_gb)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/hardware/metrics/__init__.py b/tests/unit/neptune/legacy/internal/hardware/metrics/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/metrics/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_reporter_integration.py b/tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_reporter_integration.py deleted file mode 100644 index e4d0a5e10..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_reporter_integration.py +++ /dev/null @@ -1,121 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time -import unittest - -from neptune.common.hardware.constants import BYTES_IN_ONE_GB -from neptune.common.hardware.metrics.metrics_factory import MetricsFactory -from neptune.common.hardware.metrics.reports.metric_report import ( - MetricReport, - MetricValue, -) -from neptune.common.hardware.metrics.reports.metric_reporter_factory import MetricReporterFactory -from neptune.common.hardware.resources.system_resource_info import SystemResourceInfo -from tests.unit.neptune.legacy.internal.hardware.gauges.gauges_fixture import GaugesFixture - - -class TestMetricReporterIntegration(unittest.TestCase): - def setUp(self): - self.maxDiff = 65536 - - self.fixture = GaugesFixture() - self.metrics_container = MetricsFactory( - gauge_factory=self.fixture.gauge_factory, - system_resource_info=SystemResourceInfo( - cpu_core_count=4, - memory_amount_bytes=64 * BYTES_IN_ONE_GB, - gpu_card_indices=[0, 2], - gpu_memory_amount_bytes=32 * BYTES_IN_ONE_GB, - ), - ).create_metrics_container() - - self.reference_timestamp = time.time() - metric_reporter_factory = MetricReporterFactory(reference_timestamp=self.reference_timestamp) - self.metric_reporter = metric_reporter_factory.create(self.metrics_container.metrics()) - - def test_report_metrics(self): - # given - measurement_timestamp = self.reference_timestamp + 10 - - # when - metric_reports = self.metric_reporter.report(measurement_timestamp) - - # then - expected_time = measurement_timestamp - self.reference_timestamp - expected_reports = [ - MetricReport( - metric=self.metrics_container.cpu_usage_metric, - values=[ - MetricValue( - timestamp=measurement_timestamp, - running_time=expected_time, - gauge_name="cpu", - value=self.fixture.cpu_gauge_value, - ) - ], - ), - MetricReport( - metric=self.metrics_container.memory_metric, - values=[ - MetricValue( - timestamp=measurement_timestamp, - running_time=expected_time, - gauge_name="ram", - value=self.fixture.memory_gauge_value, - ) - ], - ), - MetricReport( - metric=self.metrics_container.gpu_usage_metric, - values=[ - MetricValue( - timestamp=measurement_timestamp, - running_time=expected_time, - gauge_name="0", - value=self.fixture.gpu0_usage_gauge_value, - ), - MetricValue( - timestamp=measurement_timestamp, - running_time=expected_time, - gauge_name="2", - value=self.fixture.gpu1_usage_gauge_value, - ), - ], - ), - MetricReport( - metric=self.metrics_container.gpu_memory_metric, - values=[ - MetricValue( - timestamp=measurement_timestamp, - running_time=expected_time, - gauge_name="0", - value=self.fixture.gpu0_memory_gauge_value, - ), - MetricValue( - timestamp=measurement_timestamp, - running_time=expected_time, - gauge_name="2", - value=self.fixture.gpu1_memory_gauge_value, - ), - ], - ), - ] - self.assertListEqual(expected_reports, metric_reports) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_service_integration.py b/tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_service_integration.py deleted file mode 100644 index 0badf27ad..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/metrics/test_metric_service_integration.py +++ /dev/null @@ -1,154 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import time -import unittest -import uuid - -import psutil -from mock import ( - ANY, - MagicMock, - call, -) - -from neptune.common.hardware.constants import BYTES_IN_ONE_GB -from neptune.common.hardware.gauges.cpu import SystemCpuUsageGauge -from neptune.common.hardware.gauges.gauge_factory import SystemMemoryUsageGauge -from neptune.common.hardware.gauges.gauge_mode import GaugeMode -from neptune.common.hardware.metrics.metric import ( - Metric, - MetricResourceType, -) -from neptune.common.hardware.metrics.reports.metric_reporter import ( - MetricReport, - MetricValue, -) -from neptune.common.hardware.metrics.service.metric_service_factory import MetricServiceFactory -from neptune.common.utils import IS_MACOS - - -class TestMetricServiceIntegration(unittest.TestCase): - def setUp(self): - self.backend = MagicMock() - self.metric_service_factory = MetricServiceFactory(backend=self.backend, os_environ=os.environ) - - @unittest.skipIf(IS_MACOS, "MacOS behaves strangely") - def test_create_system_metrics(self): - # given - memory_amount_gb = psutil.virtual_memory().total / float(BYTES_IN_ONE_GB) - - # and - experiment = MagicMock() - - # and - cpu_metric_id = str(uuid.uuid4()) - ram_metric_id = str(uuid.uuid4()) - self.backend.create_hardware_metric.side_effect = [cpu_metric_id, ram_metric_id] - - # when - self.metric_service_factory.create( - gauge_mode=GaugeMode.SYSTEM, - experiment=experiment, - reference_timestamp=time.time(), - ) - - # then - self.backend.create_hardware_metric.assert_has_calls( - [ - call( - experiment, - Metric( - internal_id=cpu_metric_id, - name="CPU - usage", - description="average of all cores", - resource_type=MetricResourceType.CPU, - unit="%", - min_value=0.0, - max_value=100.0, - gauges=[SystemCpuUsageGauge()], - ), - ), - call( - experiment, - Metric( - internal_id=ram_metric_id, - name="RAM", - description="", - resource_type=MetricResourceType.RAM, - unit="GB", - min_value=0.0, - max_value=memory_amount_gb, - gauges=[SystemMemoryUsageGauge()], - ), - ), - ] - ) - - @unittest.skipIf(IS_MACOS, "MacOS behaves strangely") - def test_report_and_send_metrics(self): - # given - experiment_start = time.time() - second_after_start = experiment_start + 1.0 - - # and - experiment = MagicMock() - - # and - metric_service = self.metric_service_factory.create( - gauge_mode=GaugeMode.SYSTEM, - experiment=experiment, - reference_timestamp=experiment_start, - ) - metrics_container = metric_service.metrics_container - - # when - metric_service.report_and_send(timestamp=second_after_start) - - # then - self.backend.send_hardware_metric_reports.assert_called_once_with( - experiment, - metrics_container.metrics(), - [ - MetricReport( - metric=metrics_container.cpu_usage_metric, - values=[ - MetricValue( - timestamp=second_after_start, - running_time=1.0, - gauge_name="cpu", - value=ANY, - ) - ], - ), - MetricReport( - metric=metrics_container.memory_metric, - values=[ - MetricValue( - timestamp=second_after_start, - running_time=1.0, - gauge_name="ram", - value=ANY, - ) - ], - ), - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/hardware/metrics/test_metrics_factory.py b/tests/unit/neptune/legacy/internal/hardware/metrics/test_metrics_factory.py deleted file mode 100644 index a09cd7893..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/metrics/test_metrics_factory.py +++ /dev/null @@ -1,163 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -from neptune.common.hardware.constants import BYTES_IN_ONE_GB -from neptune.common.hardware.gauges.cpu import SystemCpuUsageGauge -from neptune.common.hardware.gauges.gauge_factory import ( - GaugeFactory, - GpuMemoryGauge, - GpuUsageGauge, - SystemMemoryUsageGauge, -) -from neptune.common.hardware.gauges.gauge_mode import GaugeMode -from neptune.common.hardware.metrics.metric import ( - Metric, - MetricResourceType, -) -from neptune.common.hardware.metrics.metrics_factory import MetricsFactory -from neptune.common.hardware.resources.system_resource_info import SystemResourceInfo - - -class TestMetricsFactory(unittest.TestCase): - def setUp(self): - self.gauge_factory = GaugeFactory(GaugeMode.SYSTEM) - - def test_create_metrics_with_gpu(self): - # given - system_resource_info = SystemResourceInfo( - cpu_core_count=4, - memory_amount_bytes=16 * BYTES_IN_ONE_GB, - gpu_card_indices=[0, 1], - gpu_memory_amount_bytes=8 * BYTES_IN_ONE_GB, - ) - # and - metrics_factory = MetricsFactory(self.gauge_factory, system_resource_info) - - # when - metrics_container = metrics_factory.create_metrics_container() - - # then - self.assertEqual( - Metric( - name="CPU - usage", - description="average of all cores", - resource_type=MetricResourceType.CPU, - unit="%", - min_value=0.0, - max_value=100.0, - gauges=[SystemCpuUsageGauge()], - ), - metrics_container.cpu_usage_metric, - ) - # and - self.assertEqual( - Metric( - name="RAM", - description="", - resource_type=MetricResourceType.RAM, - unit="GB", - min_value=0.0, - max_value=16.0, - gauges=[SystemMemoryUsageGauge()], - ), - metrics_container.memory_metric, - ) - # and - self.assertEqual( - Metric( - name="GPU - usage", - description="2 cards", - resource_type=MetricResourceType.GPU, - unit="%", - min_value=0.0, - max_value=100.0, - gauges=[GpuUsageGauge(card_index=0), GpuUsageGauge(card_index=1)], - ), - metrics_container.gpu_usage_metric, - ) - # and - self.assertEqual( - Metric( - name="GPU - memory", - description="2 cards", - resource_type=MetricResourceType.GPU_RAM, - unit="GB", - min_value=0.0, - max_value=8.0, - gauges=[GpuMemoryGauge(card_index=0), GpuMemoryGauge(card_index=1)], - ), - metrics_container.gpu_memory_metric, - ) - - def test_create_metrics_without_gpu(self): - # given - system_resource_info = SystemResourceInfo( - cpu_core_count=4, - memory_amount_bytes=16 * BYTES_IN_ONE_GB, - gpu_card_indices=[], - gpu_memory_amount_bytes=0, - ) - # and - metrics_factory = MetricsFactory(self.gauge_factory, system_resource_info) - - # when - metrics_container = metrics_factory.create_metrics_container() - - # then - self.assertIsNotNone(metrics_container.cpu_usage_metric) - self.assertIsNotNone(metrics_container.memory_metric) - self.assertIsNone(metrics_container.gpu_usage_metric) - self.assertIsNone(metrics_container.gpu_memory_metric) - - # and - self.assertEqual( - [metrics_container.cpu_usage_metric, metrics_container.memory_metric], - metrics_container.metrics(), - ) - - def test_format_fractional_cpu_core_count(self): - # given - system_resource_info = SystemResourceInfo( - cpu_core_count=0.5, - memory_amount_bytes=2 * BYTES_IN_ONE_GB, - gpu_card_indices=[], - gpu_memory_amount_bytes=0, - ) - # and - metrics_factory = MetricsFactory(self.gauge_factory, system_resource_info) - - # when - metrics_container = metrics_factory.create_metrics_container() - - # then - self.assertEqual( - Metric( - name="CPU - usage", - description="average of all cores", - resource_type=MetricResourceType.CPU, - unit="%", - min_value=0.0, - max_value=100.0, - gauges=[SystemCpuUsageGauge()], - ), - metrics_container.cpu_usage_metric, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/hardware/resources/__init__.py b/tests/unit/neptune/legacy/internal/hardware/resources/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/resources/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/hardware/resources/test_system_resource_info_factory.py b/tests/unit/neptune/legacy/internal/hardware/resources/test_system_resource_info_factory.py deleted file mode 100644 index d94e295be..000000000 --- a/tests/unit/neptune/legacy/internal/hardware/resources/test_system_resource_info_factory.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import unittest - -from mock import MagicMock - -from neptune.common.hardware.gauges.gauge_mode import GaugeMode -from neptune.common.hardware.gpu.gpu_monitor import GPUMonitor -from neptune.common.hardware.resources.system_resource_info_factory import SystemResourceInfoFactory -from neptune.common.hardware.system.system_monitor import SystemMonitor -from tests.unit.neptune.legacy.assertions import AssertionExtensions - - -class TestSystemResourceInfoFactoryIntegration(unittest.TestCase, AssertionExtensions): - def test_whole_system_resource_info(self): - # given - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=GPUMonitor(), - os_environ=os.environ, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assert_float_greater_than(resource_info.cpu_core_count, 0) - self.assert_int_greater_than(resource_info.memory_amount_bytes, 0) - self.assert_int_greater_or_equal(resource_info.gpu_card_count, 0) - self.assert_int_greater_or_equal(resource_info.gpu_memory_amount_bytes, 0) - - @unittest.skip("We do not have docker infrastructure to test cgroupsV1") - def test_cgroup_resource_info(self): - # given - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=GPUMonitor(), - os_environ=os.environ, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.CGROUP) - - # then - self.assert_float_greater_than(resource_info.cpu_core_count, 0) - self.assert_int_greater_than(resource_info.memory_amount_bytes, 0) - self.assert_int_greater_or_equal(resource_info.gpu_card_count, 0) - self.assert_int_greater_or_equal(resource_info.gpu_memory_amount_bytes, 0) - - -class TestSystemResourceInfoFactory(unittest.TestCase): - def test_gpu_card_indices_without_cuda_env_variable(self): - # given - gpu_monitor = MagicMock(spec_set=GPUMonitor) - gpu_monitor.get_card_count.return_value = 2 - # and - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), gpu_monitor=gpu_monitor, os_environ=dict() - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assertEqual([0, 1], resource_info.gpu_card_indices) - - def test_gpu_card_indices_based_on_cuda_env_variable(self): - # given - gpu_monitor = MagicMock(spec_set=GPUMonitor) - gpu_monitor.get_card_count.return_value = 4 - # and - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=gpu_monitor, - os_environ={"CUDA_VISIBLE_DEVICES": "1,3"}, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assertEqual([1, 3], resource_info.gpu_card_indices) - - def test_empty_gpu_card_indices_on_cuda_env_variable_minus_one(self): - # given - gpu_monitor = MagicMock(spec_set=GPUMonitor) - gpu_monitor.get_card_count.return_value = 4 - # and - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=gpu_monitor, - os_environ={"CUDA_VISIBLE_DEVICES": "-1"}, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assertEqual([], resource_info.gpu_card_indices) - - def test_should_ignore_gpu_indices_after_index_out_of_range(self): - # given - gpu_monitor = MagicMock(spec_set=GPUMonitor) - gpu_monitor.get_card_count.return_value = 4 - # and - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=gpu_monitor, - os_environ={"CUDA_VISIBLE_DEVICES": "1,3,5,2"}, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assertEqual([1, 3], resource_info.gpu_card_indices) - - def test_ignore_empty_cuda_env_variable(self): - # given - gpu_monitor = MagicMock(spec_set=GPUMonitor) - gpu_monitor.get_card_count.return_value = 2 - # and - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=gpu_monitor, - os_environ={"CUDA_VISIBLE_DEVICES": ""}, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assertEqual([0, 1], resource_info.gpu_card_indices) - - def test_should_ignore_invalid_cuda_env_variable_syntax(self): - # given - gpu_monitor = MagicMock(spec_set=GPUMonitor) - gpu_monitor.get_card_count.return_value = 4 - # and - system_resource_info_factory = SystemResourceInfoFactory( - system_monitor=SystemMonitor(), - gpu_monitor=gpu_monitor, - os_environ={"CUDA_VISIBLE_DEVICES": "1,3,abc"}, - ) - - # when - resource_info = system_resource_info_factory.create(GaugeMode.SYSTEM) - - # then - self.assertEqual([0, 1, 2, 3], resource_info.gpu_card_indices) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/storage/__init__.py b/tests/unit/neptune/legacy/internal/storage/__init__.py deleted file mode 100644 index 62a86a5be..000000000 --- a/tests/unit/neptune/legacy/internal/storage/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tests/unit/neptune/legacy/internal/storage/test_datastream.py b/tests/unit/neptune/legacy/internal/storage/test_datastream.py deleted file mode 100644 index b53ea2908..000000000 --- a/tests/unit/neptune/legacy/internal/storage/test_datastream.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -import pytest -from mock import Mock - -from neptune.common.exceptions import InternalClientError -from neptune.common.storage.datastream import FileChunker -from neptune.legacy.internal.api_clients.client_config import MultipartConfig - - -class TestFileChunker: - multipart_config = MultipartConfig( - min_chunk_size=5_242_880, # 5 MB - max_chunk_size=1_073_741_824, # 1 GB - max_chunk_count=1_000, - max_single_part_size=5_242_880, # 1 GB - ) - - def get_chunk_count(self, file_size, chunk_size): - chunk_idx = 0 - while file_size > chunk_size: - chunk_idx += 1 - file_size -= chunk_size - return chunk_idx + 1 - - @pytest.mark.parametrize( - "file_size, expected_chunk_size, expected_chunk_count", - ( - (1_000_000, 5_242_880, 1), - (6_000_000, 5_242_880, 2), - (5_242_880_000, 5_242_880, 1_000), - (5_242_880_001, 5_242_881, 1_000), - (5_242_891_001, 5_242_892, 1_000), - (1_073_741_824_000, 1_073_741_824, 1_000), - ), - ) - def test_chunk_size_for_small_file(self, file_size, expected_chunk_size, expected_chunk_count): - chunker = FileChunker(Mock(), Mock(), total_size=file_size, multipart_config=self.multipart_config) - - chunk_size = chunker._get_chunk_size() - - chunk_count = self.get_chunk_count(file_size, chunk_size) - assert chunk_count == expected_chunk_count - assert chunk_size == expected_chunk_size - assert chunk_count <= self.multipart_config.max_chunk_count - assert chunk_size <= self.multipart_config.max_chunk_size - assert chunk_size >= self.multipart_config.min_chunk_size - - def test_too_large_file(self): - file_size = 1_073_741_824_001 - - chunker = FileChunker(Mock(), Mock(), total_size=file_size, multipart_config=self.multipart_config) - - with pytest.raises(InternalClientError): - chunker._get_chunk_size() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/internal/storage/test_upload_storage_utils.py b/tests/unit/neptune/legacy/internal/storage/test_upload_storage_utils.py deleted file mode 100644 index af0096ca2..000000000 --- a/tests/unit/neptune/legacy/internal/storage/test_upload_storage_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -from mock import patch - -from neptune.common.storage.storage_utils import ( - AttributeUploadConfiguration, - UploadEntry, - UploadPackage, - split_upload_files, -) - - -class TestUploadStorageUtils(unittest.TestCase): - MAX_PACKAGE_SIZE = 1024 - - @patch("os.path.isdir", new=lambda _: False) - @patch("os.path.getsize") - def test_split_upload_files_should_generate_upload_files_list_for_only_one_file(self, getsize): - # GIVEN - entry = UploadEntry("/tmp/test.gz", "test.gz") - size = 10 * self.MAX_PACKAGE_SIZE - config = AttributeUploadConfiguration(size) - getsize.return_value = size - - # EXPECT - expected = UploadPackage() - expected.update(entry, size) - self.assertEqual( - list(split_upload_files(upload_entries={entry}, upload_configuration=config)), - [expected], - ) - - @patch("os.path.isdir", new=lambda _: False) - @patch("os.path.getsize") - def test_split_upload_files_should_not_generate_empty_packages(self, getsize): - # GIVEN - entry = UploadEntry("/tmp/test.gz", "test.gz") - # AND - upload_entry = UploadEntry(entry.source, entry.target_path) - size = 10 * self.MAX_PACKAGE_SIZE - config = AttributeUploadConfiguration(size) - getsize.return_value = size - - # EXPECT - expected = UploadPackage() - expected.update(entry, size) - for package in split_upload_files(upload_entries={upload_entry}, upload_configuration=config): - self.assertFalse(package.is_empty()) diff --git a/tests/unit/neptune/legacy/internal/utils/__init__.py b/tests/unit/neptune/legacy/internal/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/neptune/legacy/internal/utils/test_image.py b/tests/unit/neptune/legacy/internal/utils/test_image.py deleted file mode 100644 index f563713a2..000000000 --- a/tests/unit/neptune/legacy/internal/utils/test_image.py +++ /dev/null @@ -1,137 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import sys -import unittest -from uuid import uuid4 - -import matplotlib -import numpy -from matplotlib import pyplot -from PIL import Image - -from neptune.common.utils import ( - IS_MACOS, - IS_WINDOWS, -) -from neptune.legacy.internal.utils.image import ( - _get_figure_as_image, - _get_pil_image_data, - get_image_content, -) - -matplotlib.use("agg") - - -class TestImage(unittest.TestCase): - TEST_DIR = "/tmp/neptune/{}".format(uuid4()) - - def setUp(self): - if not os.path.exists(self.TEST_DIR): - os.makedirs(self.TEST_DIR) - - def test_get_image_content_from_string(self): - # given - filename = "{}/image.png".format(self.TEST_DIR) - image_array = numpy.random.rand(200, 300, 3) - scaled_array = image_array * 255 - expected_image = Image.fromarray(scaled_array.astype(numpy.uint8)) - expected_image.save(filename) - - # expect - self.assertEqual(get_image_content(filename), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_pil_image(self): - # given - image_array = numpy.random.rand(200, 300, 3) - scaled_array = image_array * 255 - expected_image = Image.fromarray(scaled_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(expected_image), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_2d_grayscale_array(self): - # given - image_array = numpy.random.rand(200, 300) - scaled_array = image_array * 255 - expected_image = Image.fromarray(scaled_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(image_array), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_3d_grayscale_array(self): - # given - image_array = numpy.array([[[1], [2]], [[3], [4]], [[5], [6]]]) - expected_array = numpy.array([[1, 2], [3, 4], [5, 6]]) * 255 - expected_image = Image.fromarray(expected_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(image_array), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_rgb_array(self): - # given - image_array = numpy.random.rand(200, 300, 3) - scaled_array = image_array * 255 - expected_image = Image.fromarray(scaled_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(image_array), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_rgba_array(self): - # given - image_array = numpy.random.rand(200, 300, 4) - scaled_array = image_array * 255 - expected_image = Image.fromarray(scaled_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(image_array), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_figure(self): - # given - pyplot.plot([1, 2, 3, 4]) - pyplot.ylabel("some interesting numbers") - figure = pyplot.gcf() - figure.canvas.draw() - - # expect - self.assertEqual(get_image_content(figure), _get_figure_as_image(figure)) - - @unittest.skipIf(IS_WINDOWS, "Installing Torch on Windows takes too long") - @unittest.skipIf( - IS_MACOS and sys.version_info.major == 3 and sys.version_info.minor == 10, - "No torch for 3.10 on Mac", - ) - def test_get_image_content_from_torch_tensor(self): - import torch - - # given - image_tensor = torch.rand(200, 300, 3) - expected_array = image_tensor.numpy() * 255 - expected_image = Image.fromarray(expected_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(image_tensor), _get_pil_image_data(expected_image)) - - def test_get_image_content_from_tensorflow_tensor(self): - import tensorflow as tf - - # given - image_tensor = tf.random.uniform(shape=[200, 300, 3]) - expected_array = image_tensor.numpy() * 255 - expected_image = Image.fromarray(expected_array.astype(numpy.uint8)) - - # expect - self.assertEqual(get_image_content(image_tensor), _get_pil_image_data(expected_image)) diff --git a/tests/unit/neptune/legacy/oauth_objects_factory.py b/tests/unit/neptune/legacy/oauth_objects_factory.py deleted file mode 100644 index c6178c898..000000000 --- a/tests/unit/neptune/legacy/oauth_objects_factory.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -import jwt - -from tests.unit.neptune.legacy.random_utils import a_string - -SECRET = "secret" - - -def an_access_token(): - return jwt.encode( - { - "exp": time.time(), - "azp": a_string(), - "iss": "http://{}.com".format(a_string()), - }, - SECRET, - ) - - -def a_refresh_token(): - return jwt.encode({"exp": 0, "azp": a_string(), "iss": "http://{}.com".format(a_string())}, SECRET) diff --git a/tests/unit/neptune/legacy/project_test_fixture.py b/tests/unit/neptune/legacy/project_test_fixture.py deleted file mode 100644 index 0f51c7fb6..000000000 --- a/tests/unit/neptune/legacy/project_test_fixture.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from tests.unit.neptune.legacy.api_objects_factory import an_experiment_leaderboard_entry_dto - -some_exp_entry_dto = an_experiment_leaderboard_entry_dto() - -some_exp_entry_row = { - "id": some_exp_entry_dto.shortId, - "name": some_exp_entry_dto.name, - "created": some_exp_entry_dto.timeOfCreation, - "finished": some_exp_entry_dto.timeOfCompletion, - "owner": some_exp_entry_dto.owner, - "notes": some_exp_entry_dto.description, - "running_time": some_exp_entry_dto.runningTime, - "size": some_exp_entry_dto.size, - "tags": some_exp_entry_dto.tags, -} -some_exp_entry_row.update({"property_" + p.key: p.value for p in some_exp_entry_dto.properties}) -some_exp_entry_row.update({"parameter_" + p.name: p.value for p in some_exp_entry_dto.parameters}) -some_exp_entry_row.update({"channel_" + c.channelName: c.y for c in some_exp_entry_dto.channelsLastValues}) diff --git a/tests/unit/neptune/legacy/random_utils.py b/tests/unit/neptune/legacy/random_utils.py deleted file mode 100644 index 0f69bc4e1..000000000 --- a/tests/unit/neptune/legacy/random_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import datetime -import random -import string -import uuid - -import mock - -from neptune.legacy.projects import Project - - -def a_string(): - char_set = string.ascii_letters - return "".join(random.sample(char_set * 10, 10)) - - -def a_uuid_string(): - return str(uuid.uuid4()) - - -def a_string_list(length=2): - return [a_string() for _ in range(0, length)] - - -def a_timestamp(): - return datetime.datetime.now() - - -def sort_df_by_columns(df): - df = df.reindex(sorted(df.columns), axis=1) - return df - - -def an_experiment_id(): - prefix = "".join(random.choice(string.ascii_uppercase) for _ in range(3)) - number = random.randint(0, 100) - return "{}-{}".format(prefix, number) - - -def a_project_qualified_name(): - return "{}/{}".format(a_string(), a_string()) - - -def a_project(): - return Project( - backend=mock.MagicMock(), - internal_id=a_uuid_string(), - name=a_string(), - namespace=a_string(), - ) diff --git a/tests/unit/neptune/legacy/test_alpha_integration_backend.py b/tests/unit/neptune/legacy/test_alpha_integration_backend.py deleted file mode 100644 index eb2a8ee8d..000000000 --- a/tests/unit/neptune/legacy/test_alpha_integration_backend.py +++ /dev/null @@ -1,138 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time -import unittest -from typing import List - -import mock -from freezegun import freeze_time -from mock import MagicMock - -from neptune.legacy.internal.api_clients import HostedNeptuneBackendApiClient -from neptune.legacy.internal.api_clients.hosted_api_clients.hosted_alpha_leaderboard_api_client import ( - HostedAlphaLeaderboardApiClient, -) -from neptune.legacy.internal.channels.channels import ( - ChannelIdWithValues, - ChannelNamespace, - ChannelType, - ChannelValue, -) -from tests.unit.neptune.backend_test_mixin import BackendTestMixin as AlphaBackendTestMixin - -API_TOKEN = ( - "eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYWxwaGEuc3RhZ2UubmVwdHVuZS5haSIsImFwaV91cmwiOiJodHRwczovL2FscG" - "hhLnN0YWdlLm5lcHR1bmUuYWkiLCJhcGlfa2V5IjoiZDg5MGQ3Y2ItZGEzNi00MjRkLWJhNTQtZmVjZDJmYTdhOTQzIn0=" -) -"""base64 decoded `API_TOKEN` -{ - "api_address": "https://alpha.stage.neptune.ai", - "api_url": "https://alpha.stage.neptune.ai", - "api_key": "d890d7cb-da36-424d-ba54-fecd2fa7a943" -} -""" - - -class TestAlphaIntegrationNeptuneBackend(unittest.TestCase, AlphaBackendTestMixin): - @mock.patch("bravado.client.SwaggerClient.from_url") - @mock.patch( - "neptune.legacy.internal.api_clients.hosted_api_clients.hosted_backend_api_client.NeptuneAuthenticator", - new=MagicMock, - ) - @mock.patch( - "neptune.internal.backends.hosted_client.NeptuneAuthenticator", - new=MagicMock, - ) - @mock.patch("socket.gethostbyname", MagicMock(return_value="1.1.1.1")) - def setUp(self, swagger_client_factory) -> None: - self._get_swagger_client_mock(swagger_client_factory) - self.backend = HostedNeptuneBackendApiClient(API_TOKEN) - self.leaderboard = HostedAlphaLeaderboardApiClient(self.backend) - self.exp_mock = MagicMock(internal_id="00000000-0000-0000-0000-000000000000") - - @freeze_time() - def _test_send_channel_values( - self, - channel_y_elements: List[tuple], - expected_operation: str, - channel_type: ChannelType, - ): - # given prepared `ChannelIdWithValues` - channel_id = "channel_id" - channel_name = "channel_name" - now_ms = int(time.time() * 1000) - channel_with_values = ChannelIdWithValues( - channel_id=channel_id, - channel_name=channel_name, - channel_type=channel_type.value, - channel_namespace=ChannelNamespace.USER, - channel_values=[ - ChannelValue(x=None, y={channel_y_key: channel_y_value}, ts=None) - for channel_y_key, channel_y_value in channel_y_elements - ], - ) - - # invoke send_channels_values - self.leaderboard.send_channels_values(self.exp_mock, [channel_with_values]) - - # expect `executeOperations` was called once with properly prepared kwargs - expected_call_args = { - "experimentId": "00000000-0000-0000-0000-000000000000", - "operations": [ - { - "path": f"logs/{channel_name}", - expected_operation: { - "entries": [ - { - "value": channel_y_value, - "step": None, - "timestampMilliseconds": now_ms, - } - for _, channel_y_value in channel_y_elements - ] - }, - } - ], - } - execute_operations = self.leaderboard.leaderboard_swagger_client.api.executeOperations - self.assertEqual(len(execute_operations.call_args_list), 1) - self.assertDictEqual(execute_operations.call_args_list[0][1], expected_call_args) - - def test_send_channels_text_values(self): - channel_y_elements = [ - ("text_value", "Line of text"), - ("text_value", "Another line of text"), - ] - self._test_send_channel_values( - channel_y_elements, - expected_operation="logStrings", - channel_type=ChannelType.TEXT, - ) - - def test_send_channels_numeric_values(self): - channel_y_elements = [ - ("numeric_value", 42), - ("numeric_value", 0.07), - ] - self._test_send_channel_values( - channel_y_elements, - expected_operation="logFloats", - channel_type=ChannelType.NUMERIC, - ) - - def test_send_channels_image_values(self): - """TODO: implement in NPT-9207""" diff --git a/tests/unit/neptune/legacy/test_channel_writer.py b/tests/unit/neptune/legacy/test_channel_writer.py deleted file mode 100644 index b91a65135..000000000 --- a/tests/unit/neptune/legacy/test_channel_writer.py +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest -from datetime import datetime - -import mock - -from neptune.legacy.internal.streams.channel_writer import ChannelWriter - - -class TestChannelWriter(unittest.TestCase): - def test_write_data_to_channel_writer(self): - # given - experiment = mock.MagicMock() - experiment.get_system_properties.return_value = {"created": datetime.now()} - channel_name = "a channel name" - writer = ChannelWriter(experiment, channel_name) - - # when - writer.write("some\ndata") - - # then - experiment._channels_values_sender.send.assert_called_once() - - @mock.patch("neptune.legacy.internal.streams.channel_writer.datetime") - def test_write_data_with_low_resolution_datetime_now(self, dt): - # given - experiment = mock.MagicMock() - experiment.get_system_properties.return_value = {"created": datetime(2022, 2, 2, 2, 2, 2, 2)} - channel_name = "a channel name" - writer = ChannelWriter(experiment, channel_name) - - # and - dt.now.return_value = datetime(2022, 2, 2, 2, 2, 2, 3) - - # when - writer.write("text1\ntext2\n") - - # then - x_to_text = self._extract_x_to_text_from_calls(experiment._channels_values_sender.send.call_args_list) - self.assertEqual(x_to_text, {0.001: "text1", 0.002: "text2"}) - - @mock.patch("neptune.legacy.internal.streams.channel_writer.datetime") - def test_write_data_with_high_resolution_datetime_now(self, dt): - # given - experiment = mock.MagicMock() - experiment.get_system_properties.return_value = {"created": datetime(2022, 2, 2, 2, 2, 2, 2)} - channel_name = "a channel name" - writer = ChannelWriter(experiment, channel_name) - - # when - dt.now.return_value = datetime(2022, 2, 2, 2, 2, 2, 4) - writer.write("text1\n") - dt.now.return_value = datetime(2022, 2, 2, 2, 2, 2, 5) - writer.write("text2\n") - - # then - x_to_text = self._extract_x_to_text_from_calls(experiment._channels_values_sender.send.call_args_list) - self.assertEqual(x_to_text, {0.002: "text1", 0.003: "text2"}) - - @staticmethod - def _extract_x_to_text_from_calls(calls): - channel_values = [kwargs["channel_value"] for (_, kwargs) in calls] - return dict((v.x, v.y["text_value"]) for v in channel_values) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/test_experiment.py b/tests/unit/neptune/legacy/test_experiment.py deleted file mode 100644 index a2e9a5ad4..000000000 --- a/tests/unit/neptune/legacy/test_experiment.py +++ /dev/null @@ -1,165 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import base64 -import random -import time -import unittest -from io import StringIO - -import mock -import pandas as pd -from mock import ( - MagicMock, - call, -) -from munch import Munch -from pandas.testing import assert_frame_equal - -from neptune.legacy.experiments import Experiment -from neptune.legacy.internal.channels.channels import ( - ChannelType, - ChannelValue, -) -from tests.unit.neptune.legacy.random_utils import ( - a_project, - a_string, - a_uuid_string, - an_experiment_id, - sort_df_by_columns, -) - - -class TestExperiment(unittest.TestCase): - @mock.patch("neptune.legacy.experiments.ChannelsValuesSender", return_value=mock.MagicMock()) - @mock.patch("neptune.legacy.experiments.ExecutionContext", mock.MagicMock()) - def test_send_metric(self, ChannelsValuesSender): - # given - channels_values_sender = ChannelsValuesSender.return_value - experiment = Experiment(mock.MagicMock(), mock.MagicMock(), an_experiment_id(), a_uuid_string()) - channel_value = ChannelValue( - x=random.randint(0, 100), - y=dict(numeric_value=random.randint(0, 100)), - ts=time.time(), - ) - - # when - experiment.send_metric("loss", channel_value.x, channel_value.y["numeric_value"], channel_value.ts) - - # then - channels_values_sender.send.assert_called_with("loss", ChannelType.NUMERIC.value, channel_value) - - @mock.patch("neptune.legacy.experiments.ChannelsValuesSender", return_value=mock.MagicMock()) - @mock.patch("neptune.legacy.experiments.ExecutionContext", mock.MagicMock()) - def test_send_text(self, ChannelsValuesSender): - # given - channels_values_sender = ChannelsValuesSender.return_value - experiment = Experiment(mock.MagicMock(), a_project(), an_experiment_id(), a_uuid_string()) - channel_value = ChannelValue(x=random.randint(0, 100), y=dict(text_value=a_string()), ts=time.time()) - - # when - experiment.send_text("stdout", channel_value.x, channel_value.y["text_value"], channel_value.ts) - - # then - channels_values_sender.send.assert_called_with("stdout", ChannelType.TEXT.value, channel_value) - - @mock.patch("neptune.legacy.experiments.get_image_content", return_value=b"content") - @mock.patch("neptune.legacy.experiments.ChannelsValuesSender", return_value=mock.MagicMock()) - @mock.patch("neptune.legacy.experiments.ExecutionContext", mock.MagicMock()) - def test_send_image(self, ChannelsValuesSender, content): - # given - channels_values_sender = ChannelsValuesSender.return_value - experiment = Experiment(mock.MagicMock(), a_project(), an_experiment_id(), a_uuid_string()) - image_value = dict( - name=a_string(), - description=a_string(), - data=base64.b64encode(content()).decode("utf-8"), - ) - channel_value = ChannelValue(x=random.randint(0, 100), y=dict(image_value=image_value), ts=time.time()) - - # when - experiment.send_image( - "errors", - channel_value.x, - "/tmp/img.png", - image_value["name"], - image_value["description"], - channel_value.ts, - ) - - # then - channels_values_sender.send.assert_called_with("errors", ChannelType.IMAGE.value, channel_value) - - def test_append_tags(self): - # given - backend = mock.MagicMock() - experiment = Experiment(backend, a_project(), an_experiment_id(), a_uuid_string()) - - # and - def build_call(tags_list): - return call(experiment=experiment, tags_to_add=tags_list, tags_to_delete=[]) - - # when - experiment.append_tag("tag") - experiment.append_tag(["tag1", "tag2", "tag3"]) - experiment.append_tag("tag1", "tag2", "tag3") - experiment.append_tags("tag") - experiment.append_tags(["tag1", "tag2", "tag3"]) - experiment.append_tags("tag1", "tag2", "tag3") - - # then - backend.update_tags.assert_has_calls( - [ - build_call(["tag"]), - build_call(["tag1", "tag2", "tag3"]), - build_call(["tag1", "tag2", "tag3"]), - build_call(["tag"]), - build_call(["tag1", "tag2", "tag3"]), - build_call(["tag1", "tag2", "tag3"]), - ] - ) - - def test_get_numeric_channels_values(self): - # when - backend = MagicMock() - backend.get_channel_points_csv.return_value = StringIO("\n".join(["0.3,2.5", "1,2"])) - - experiment = MagicMock() - experiment.id = a_string() - experiment.internal_id = a_uuid_string() - experiment.channels = [Munch(id=a_uuid_string(), name="epoch_loss")] - experiment.channelsLastValues = [Munch(channelName="epoch_loss", x=2.5, y=2)] - - backend.get_experiment.return_value = experiment - - # then - experiment = Experiment( - backend=backend, - project=a_project(), - _id=a_string(), - internal_id=a_uuid_string(), - ) - result = experiment.get_numeric_channels_values("epoch_loss") - - expected_result = pd.DataFrame({"x": [0.3, 1.0], "epoch_loss": [2.5, 2.0]}, dtype=float) - - expected_result = sort_df_by_columns(expected_result) - result = sort_df_by_columns(result) - - assert_frame_equal(expected_result, result) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/test_git_info.py b/tests/unit/neptune/legacy/test_git_info.py deleted file mode 100644 index add3ce42c..000000000 --- a/tests/unit/neptune/legacy/test_git_info.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import datetime -import unittest - -import mock - -from neptune.common.utils import get_git_info -from neptune.legacy.git_info import GitInfo - - -class TestGitInfo(unittest.TestCase): - def test_getting_some_git_info_from_current_repository(self): - # when - git_info = get_git_info("") - - # then - self.assertNotEqual(git_info, None) - - @mock.patch("git.Repo", return_value=mock.MagicMock()) - def test_getting_git_info(self, repo_mock): - # given - now = datetime.datetime.now() - repo = repo_mock.return_value - repo.is_dirty.return_value = True - repo.head.commit.hexsha = "sha" - repo.head.commit.message = "message" - repo.head.commit.author.name = "author_name" - repo.head.commit.author.email = "author@email" - repo.head.commit.committed_datetime = now - repo.active_branch.name = "master" - repo.remotes = [] - - # when - git_info = get_git_info("") - - # then - self.assertEqual( - git_info, - GitInfo( - commit_id="sha", - message="message", - author_name="author_name", - author_email="author@email", - commit_date=now, - repository_dirty=True, - active_branch="master", - remote_urls=[], - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/test_imports.py b/tests/unit/neptune/legacy/test_imports.py deleted file mode 100644 index e69564553..000000000 --- a/tests/unit/neptune/legacy/test_imports.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (c) 2021, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# isort:skip -# fmt: off -# flake8: noqa -import unittest - -from neptune.legacy.api_exceptions import ( - ChannelAlreadyExists, - ChannelDoesNotExist, - ChannelNotFound, - ChannelsValuesSendBatchError, - ConnectionLost, - ExperimentAlreadyFinished, - ExperimentLimitReached, - ExperimentNotFound, - ExperimentOperationErrors, - ExperimentValidationError, - Forbidden, - InvalidApiKey, - NeptuneApiException, - NeptuneSSLVerificationError, - NotebookNotFound, - PathInExperimentNotFound, - PathInProjectNotFound, - ProjectNotFound, - ServerError, - StorageLimitReached, - Unauthorized, - WorkspaceNotFound, -) -from neptune.legacy.backend import ( - ApiClient, - BackendApiClient, - LeaderboardApiClient, -) -from neptune.legacy.checkpoint import Checkpoint -from neptune.legacy.exceptions import ( - CannotResolveHostname, - DeleteArtifactUnsupportedInAlphaException, - DeprecatedApiToken, - DownloadArtifactsUnsupportedException, - DownloadArtifactUnsupportedException, - DownloadSourcesException, - FileNotFound, - InvalidChannelValue, - InvalidNeptuneBackend, - InvalidNotebookPath, - NeptuneException, - NeptuneIncorrectImportException, - NeptuneIncorrectProjectQualifiedNameException, - NeptuneLibraryNotInstalledException, - NeptuneMissingApiTokenException, - NeptuneMissingProjectQualifiedNameException, - NeptuneNoExperimentContextException, - NeptuneUninitializedException, - NoChannelValue, - NotADirectory, - NotAFile, - UnsupportedClientVersion, - UnsupportedInAlphaException, -) -from neptune.legacy.experiments import Experiment -from neptune.legacy.git_info import GitInfo -from neptune.legacy.model import ( - ChannelWithLastValue, - LeaderboardEntry, - Point, - Points, -) -from neptune.legacy.notebook import Notebook -from neptune.legacy.oauth import ( - NeptuneAuth, - NeptuneAuthenticator, -) -from neptune.legacy.projects import Project -from neptune.legacy.sessions import Session -from neptune.management.exceptions import ( - AccessRevokedOnDeletion, - AccessRevokedOnMemberRemoval, - BadRequestException, - ConflictingWorkspaceName, - InvalidProjectName, - ManagementOperationFailure, - MissingWorkspaceName, - ProjectAlreadyExists, - ProjectNotFound, - ProjectsLimitReached, - UnsupportedValue, - UserAlreadyHasAccess, - UserNotExistsOrWithoutAccess, - WorkspaceNotFound, -) - - -class TestImports(unittest.TestCase): - def test_imports(self): - pass -# fmt: on diff --git a/tests/unit/neptune/legacy/test_oauth.py b/tests/unit/neptune/legacy/test_oauth.py deleted file mode 100644 index 7d636eced..000000000 --- a/tests/unit/neptune/legacy/test_oauth.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time -import unittest - -import jwt -from mock import ( - MagicMock, - patch, -) -from oauthlib.oauth2 import TokenExpiredError - -from neptune.common.oauth import ( - NeptuneAuth, - NeptuneAuthenticator, - _decoding_options, - _no_token_updater, -) -from tests.unit.neptune.legacy.http_objects_factory import a_request -from tests.unit.neptune.legacy.oauth_objects_factory import ( - SECRET, - a_refresh_token, - an_access_token, -) - - -class TestNeptuneAuth(unittest.TestCase): - def setUp(self): - super(TestNeptuneAuth, self).setUp() - - self.session = MagicMock() - self.session.token = dict() - self.neptune_auth = NeptuneAuth(lambda: self.session) - self.neptune_auth.token_expires_at = time.time() + 60 - self.request = a_request() - - self.url, self.method, self.body, self.headers = ( - self.request.url, - self.request.method, - self.request.body, - self.request.headers, - ) - - self.updated_url, self.updated_headers, self.updated_body = ( - a_request().url, - a_request().headers, - a_request().body, - ) - - def test_add_valid_token(self): - # given - self.session._client.add_token.return_value = ( - self.updated_url, - self.updated_headers, - self.updated_body, - ) - - # when - updated_request = self.neptune_auth(self.request) - - # then - self.session._client.add_token.assert_called_once_with( - self.url, http_method=self.method, body=self.body, headers=self.headers - ) - - # and - self.assertEqual(self.updated_url, updated_request.url) - self.assertEqual(self.updated_headers, updated_request.headers) - self.assertEqual(self.updated_body, updated_request.body) - - def test_refresh_token_and_add(self): - # given - self.session._client.add_token.side_effect = [ - TokenExpiredError, - (self.updated_url, self.updated_headers, self.updated_body), - ] - - # when - updated_request = self.neptune_auth(self.request) - - # then - self.session._client.add_token.assert_called_with( - self.url, http_method=self.method, body=self.body, headers=self.headers - ) - - # and - self.assertEqual(self.updated_url, updated_request.url) - self.assertEqual(self.updated_headers, updated_request.headers) - self.assertEqual(self.updated_body, updated_request.body) - - -class TestNeptuneAuthenticator(unittest.TestCase): - @patch("neptune.common.oauth.OAuth2Session") - @patch("neptune.common.oauth.time") - def test_apply_oauth2_session_to_request(self, time_mock, session_mock): - # given - api_token = MagicMock() - backend_client = MagicMock() - - auth_tokens = MagicMock() - auth_tokens.accessToken = an_access_token() - auth_tokens.refreshToken = a_refresh_token() - decoded_access_token = jwt.decode(auth_tokens.accessToken, SECRET, options=_decoding_options) - - backend_client.api.exchangeApiToken(X_Neptune_Api_Token=api_token).response().result = auth_tokens - - # and - now = time.time() - time_mock.time.return_value = now - - # and - session = MagicMock() - session_mock.return_value = session - session.token = dict() - - # and - neptune_authenticator = NeptuneAuthenticator(api_token, backend_client, False, None) - request = a_request() - - # when - updated_request = neptune_authenticator.apply(request) - - # then - expected_token = { - "access_token": auth_tokens.accessToken, - "refresh_token": auth_tokens.refreshToken, - "expires_in": decoded_access_token["exp"] - now, - } - - expected_auto_refresh_url = "{realm_url}/protocol/openid-connect/token".format( - realm_url=decoded_access_token["iss"] - ) - - session_mock.assert_called_once_with( - client_id=decoded_access_token["azp"], - token=expected_token, - auto_refresh_url=expected_auto_refresh_url, - auto_refresh_kwargs={"client_id": decoded_access_token["azp"]}, - token_updater=_no_token_updater, - ) - - # and - self.assertEqual(session, updated_request.auth.session) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/test_project.py b/tests/unit/neptune/legacy/test_project.py deleted file mode 100644 index 0c94bfcdb..000000000 --- a/tests/unit/neptune/legacy/test_project.py +++ /dev/null @@ -1,388 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import ntpath -import os.path -import sys -import unittest -from random import randint - -import pandas as pd -from mock import ( - MagicMock, - patch, -) -from munch import Munch - -from neptune.legacy.exceptions import NeptuneNoExperimentContextException -from neptune.legacy.experiments import Experiment -from neptune.legacy.model import LeaderboardEntry -from neptune.legacy.projects import Project -from tests.unit.neptune.legacy.api_objects_factory import ( - a_registered_project_member, - an_invited_project_member, -) -from tests.unit.neptune.legacy.project_test_fixture import ( - some_exp_entry_dto, - some_exp_entry_row, -) -from tests.unit.neptune.legacy.random_utils import ( - a_string, - a_string_list, - a_uuid_string, -) - - -class TestProject(unittest.TestCase): - def setUp(self): - super(TestProject, self).setUp() - self.backend = MagicMock() - self.project = Project( - backend=self.backend, - internal_id=a_uuid_string(), - namespace=a_string(), - name=a_string(), - ) - self.current_directory = os.getcwd() - - def tearDown(self): - # revert initial directory after changing location in tests - os.chdir(self.current_directory) - - def test_get_members(self): - # given - member_usernames = [a_string() for _ in range(0, 2)] - members = [a_registered_project_member(username) for username in member_usernames] - - # and - self.backend.get_project_members.return_value = members + [an_invited_project_member()] - - # when - fetched_member_usernames = self.project.get_members() - - # then - self.backend.get_project_members.assert_called_once_with(self.project.internal_id) - - # and - self.assertEqual(member_usernames, fetched_member_usernames) - - def test_get_experiments_with_no_params(self): - # given - leaderboard_entries = [MagicMock() for _ in range(0, 2)] - self.backend.get_leaderboard_entries.return_value = leaderboard_entries - - # when - experiments = self.project.get_experiments() - - # then - self.backend.get_leaderboard_entries.assert_called_once_with( - project=self.project, - ids=None, - states=None, - owners=None, - tags=None, - min_running_time=None, - ) - - # and - expected_experiments = [ - Experiment(self.backend, self.project, entry.id, entry.internal_id) for entry in leaderboard_entries - ] - self.assertEqual(expected_experiments, experiments) - - def test_get_experiments_with_scalar_params(self): - # given - leaderboard_entries = [MagicMock() for _ in range(0, 2)] - self.backend.get_leaderboard_entries.return_value = leaderboard_entries - - # and - params = dict( - id=a_string(), - state="succeeded", - owner=a_string(), - tag=a_string(), - min_running_time=randint(1, 100), - ) - - # when - experiments = self.project.get_experiments(**params) - - # then - expected_params = dict( - project=self.project, - ids=[params["id"]], - states=[params["state"]], - owners=[params["owner"]], - tags=[params["tag"]], - min_running_time=params["min_running_time"], - ) - self.backend.get_leaderboard_entries.assert_called_once_with(**expected_params) - - # and - expected_experiments = [ - Experiment(self.backend, self.project, entry.id, entry.internal_id) for entry in leaderboard_entries - ] - self.assertEqual(expected_experiments, experiments) - - def test_get_experiments_with_list_params(self): - # given - leaderboard_entries = [MagicMock() for _ in range(0, 2)] - self.backend.get_leaderboard_entries.return_value = leaderboard_entries - - # and - params = dict( - id=a_string_list(), - state=["succeeded", "failed"], - owner=a_string_list(), - tag=a_string_list(), - min_running_time=randint(1, 100), - ) - - # when - experiments = self.project.get_experiments(**params) - - # then - expected_params = dict( - project=self.project, - ids=params["id"], - states=params["state"], - owners=params["owner"], - tags=params["tag"], - min_running_time=params["min_running_time"], - ) - self.backend.get_leaderboard_entries.assert_called_once_with(**expected_params) - - # and - expected_experiments = [ - Experiment(self.backend, self.project, entry.id, entry.internal_id) for entry in leaderboard_entries - ] - self.assertEqual(expected_experiments, experiments) - - def test_get_leaderboard(self): - # given - self.backend.get_leaderboard_entries.return_value = [LeaderboardEntry(some_exp_entry_dto)] - - # when - leaderboard = self.project.get_leaderboard() - - # then - self.backend.get_leaderboard_entries.assert_called_once_with( - project=self.project, - ids=None, - states=None, - owners=None, - tags=None, - min_running_time=None, - ) - - # and - expected_data = {0: some_exp_entry_row} - expected_leaderboard = pd.DataFrame.from_dict(data=expected_data, orient="index") - expected_leaderboard = expected_leaderboard.reindex( - self.project._sort_leaderboard_columns(expected_leaderboard.columns), - axis="columns", - ) - - self.assertTrue(leaderboard.equals(expected_leaderboard)) - - def test_sort_leaderboard_columns(self): - # given - columns_in_expected_order = [ - "id", - "name", - "created", - "finished", - "owner", - "notes", - "size", - "tags", - "channel_abc", - "channel_def", - "parameter_abc", - "parameter_def", - "property_abc", - "property_def", - ] - - # when - sorted_columns = self.project._sort_leaderboard_columns(reversed(columns_in_expected_order)) - - # then - self.assertEqual(columns_in_expected_order, sorted_columns) - - def test_full_id(self): - # expect - self.assertEqual(self.project.namespace + "/" + self.project.name, self.project.full_id) - - def test_to_string(self): - # expect - self.assertEqual("Project({})".format(self.project.full_id), str(self.project)) - - def test_repr(self): - # expect - self.assertEqual("Project({})".format(self.project.full_id), repr(self.project)) - - def test_get_current_experiment_from_stack(self): - # given - experiment = Munch(internal_id=a_uuid_string()) - - # when - self.project._push_new_experiment(experiment) - - # then - self.assertEqual(self.project._get_current_experiment(), experiment) - - def test_pop_experiment_from_stack(self): - # given - first_experiment = Munch(internal_id=a_uuid_string()) - second_experiment = Munch(internal_id=a_uuid_string()) - # and - self.project._push_new_experiment(first_experiment) - - # when - self.project._push_new_experiment(second_experiment) - - # then - self.assertEqual(self.project._get_current_experiment(), second_experiment) - # and - self.project._remove_stopped_experiment(second_experiment) - # and - self.assertEqual(self.project._get_current_experiment(), first_experiment) - - def test_empty_stack(self): - # expect - with self.assertRaises(NeptuneNoExperimentContextException): - self.project._get_current_experiment() - - def test_create_experiment_with_relative_upload_sources(self): - # skip if - if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 5): - self.skipTest("not supported in this Python version") - - # given - os.chdir("tests/unit/neptune/legacy") - # and - anExperiment = MagicMock() - self.backend.create_experiment.return_value = anExperiment - - # when - self.project.create_experiment(upload_source_files=["test_project.*", "../../../../*.md"]) - - # then - self.backend.upload_source_code.assert_called_once() - source_target_pairs_targets = [ - target_p for source_p, target_p in self.backend.upload_source_code.call_args[0][1] - ] - self.assertTrue( - set(source_target_pairs_targets) - == { - "CODE_OF_CONDUCT.md", - "CHANGELOG.md", - "README.md", - "tests/unit/neptune/legacy/test_project.py", - } - ) - - def test_create_experiment_with_absolute_upload_sources(self): - # skip if - if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 5): - self.skipTest("not supported in this Python version") - - # given - os.chdir("tests/unit/neptune/legacy") - # and - anExperiment = MagicMock() - self.backend.create_experiment.return_value = anExperiment - - # when - self.project.create_experiment(upload_source_files=[os.path.abspath("test_project.py"), "../../../../*.md"]) - - # then - self.backend.upload_source_code.assert_called_once() - source_target_pairs_targets = [ - target_p for source_p, target_p in self.backend.upload_source_code.call_args[0][1] - ] - self.assertSetEqual( - set(source_target_pairs_targets), - { - "CODE_OF_CONDUCT.md", - "CHANGELOG.md", - "README.md", - "tests/unit/neptune/legacy/test_project.py", - }, - ) - - def test_create_experiment_with_upload_single_sources(self): - # given - os.chdir("tests/unit/neptune/legacy") - # and - anExperiment = MagicMock() - self.backend.create_experiment.return_value = anExperiment - - # when - self.project.create_experiment(upload_source_files=["test_project.py"]) - - # then - self.backend.upload_source_code.assert_called_once() - source_target_pairs_targets = [ - target_p for source_p, target_p in self.backend.upload_source_code.call_args[0][1] - ] - self.assertTrue(set(source_target_pairs_targets) == {"test_project.py"}) - - def test_create_experiment_with_common_path_below_current_directory(self): - # given - anExperiment = MagicMock() - self.backend.create_experiment.return_value = anExperiment - - # when - self.project.create_experiment(upload_source_files=["tests/unit/neptune/legacy/*.*"]) - - # then - self.backend.upload_source_code.assert_called_once() - source_target_pairs_targets = [ - target_p for source_p, target_p in self.backend.upload_source_code.call_args[0][1] - ] - self.assertTrue( - all(target_p.startswith("tests/unit/neptune/legacy") for target_p in source_target_pairs_targets) - ) - - @patch( - "neptune.legacy.internal.utils.source_code.glob", - new=lambda path: [path.replace("*", "file.txt")], - ) - @patch("neptune.legacy.projects.os.path", new=ntpath) - @patch("neptune.common.storage.storage_utils.os.sep", new=ntpath.sep) - def test_create_experiment_with_upload_sources_from_multiple_drives_on_windows( - self, - ): - # given - anExperiment = MagicMock() - # and - self.backend.create_experiment.return_value = anExperiment - - # when - self.project.create_experiment(upload_source_files=["c:\\test1\\*", "d:\\test2\\*"]) - - # then - self.backend.upload_source_code.assert_called_once() - source_target_pairs_targets = [ - target_p for source_p, target_p in self.backend.upload_source_code.call_args[0][1] - ] - self.assertTrue(set(source_target_pairs_targets) == {"c:/test1/file.txt", "d:/test2/file.txt"}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/test_session.py b/tests/unit/neptune/legacy/test_session.py deleted file mode 100644 index 1b9552cfa..000000000 --- a/tests/unit/neptune/legacy/test_session.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest -from collections import OrderedDict - -from mock import ( - MagicMock, - patch, -) - -from neptune.legacy.projects import Project -from neptune.legacy.sessions import Session -from tests.unit.neptune.legacy.api_objects_factory import a_project - - -@patch( - "neptune.legacy.internal.api_clients.hosted_api_clients.mixins.SwaggerClient.from_url", - MagicMock(), -) -@patch( - "neptune.legacy.internal.api_clients.hosted_api_clients.hosted_backend_api_client.NeptuneAuthenticator", - MagicMock(), -) -class TestSession(unittest.TestCase): - - # threading.RLock needs to be mocked, because it breaks the equality of Projects - @patch("threading.RLock") - def test_get_projects_with_given_namespace(self, _): - # given - api_projects = [a_project(), a_project()] - - # and - backend = MagicMock() - leaderboard = MagicMock() - backend.get_projects.return_value = api_projects - backend.create_leaderboard_backend.return_value = leaderboard - - # and - session = Session(backend=backend) - - # and - custom_namespace = "custom_namespace" - - # when - projects = session.get_projects(custom_namespace) - - # then - expected_projects = OrderedDict( - ( - custom_namespace + "/" + p.name, - Project(leaderboard, p.id, custom_namespace, p.name), - ) - for p in api_projects - ) - self.assertEqual(expected_projects, projects) - - # and - backend.get_projects.assert_called_with(custom_namespace) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/neptune/legacy/test_utils.py b/tests/unit/neptune/legacy/test_utils.py deleted file mode 100644 index 73b0204a3..000000000 --- a/tests/unit/neptune/legacy/test_utils.py +++ /dev/null @@ -1,316 +0,0 @@ -# -# Copyright (c) 2019, Neptune Labs Sp. z o.o. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -import numpy as np -import pandas as pd -from pandas.testing import assert_frame_equal - -from neptune.common.utils import ( - align_channels_on_x, - as_list, - get_channel_name_stems, - map_keys, - map_values, - merge_dataframes, -) -from tests.unit.neptune.legacy.random_utils import sort_df_by_columns - - -class TestMapValues(unittest.TestCase): - def test_empty_map(self): - # when - mapped_dict = map_values(times_2, {}) - - # then - self.assertEqual({}, mapped_dict) - - def test_non_empty_map(self): - # when - mapped_dict = map_values(times_2, {"a": 2, "b": 3}) - - # then - self.assertEqual({"a": 4, "b": 6}, mapped_dict) - - -class TestMapKeys(unittest.TestCase): - def test_empty_map(self): - # when - mapped_dict = map_keys(times_2, {}) - - # then - self.assertEqual({}, mapped_dict) - - def test_non_empty_map(self): - # when - mapped_dict = map_keys(times_2, {2: "a", 3: "b"}) - - # then - self.assertEqual({4: "a", 6: "b"}, mapped_dict) - - -class TestAsList(unittest.TestCase): - def test_none(self): - # expect - self.assertEqual(None, as_list(None)) - - def test_scalar(self): - # expect - self.assertEqual([1], as_list(1)) - - def test_list(self): - # expect - self.assertEqual([2], as_list([2])) - - def test_dict(self): - self.assertEqual([{"a": 1}], as_list({"a": 1})) - - -class TestAlignChannelsOnX(unittest.TestCase): - def test_ordered_x(self): - # when - np.random.seed(1234) - random_batch = np.random.random(10).tolist() - random_epoch = np.random.random(5).tolist() - random_odd = np.random.random(7).tolist() - - df = pd.DataFrame( - { - "x_batch_channel": list(range(10)), - "y_batch_channel": random_batch, - "x_epoch_channel": list(range(5)) + [np.nan] * 5, - "y_epoch_channel": random_epoch + [np.nan] * 5, - "x_odd_channel": list(range(7)) + [np.nan] * 3, - "y_odd_channel": random_odd + [np.nan] * 3, - }, - dtype=float, - ) - - expected_result = pd.DataFrame( - { - "x": list(range(10)), - "batch_channel": random_batch, - "epoch_channel": random_epoch + [np.nan] * 5, - "odd_channel": random_odd + [np.nan] * 3, - }, - dtype=float, - ) - expected_result = sort_df_by_columns(expected_result) - - # then - result = align_channels_on_x(df) - result = sort_df_by_columns(result) - - assert_frame_equal(result, expected_result) - - def test_shuffled_x(self): - # when - - batch_x = [4, 2, 10, 28] - epoch_x = [np.nan] + [1, 2, 21] - odd_x = [21, 10, 15, 4] - detached_x = [3, 5, 9] + [np.nan] - - batch_y = [7, 2, 9, 1] - epoch_y = [np.nan, 3, 5, 9] - odd_y = [21, 15, 4, 3] - detached_y = [1, 5, 12, np.nan] - - df = pd.DataFrame( - { - "x_batch_channel": batch_x, - "y_batch_channel": batch_y, - "x_epoch_channel": epoch_x, - "y_epoch_channel": epoch_y, - "x_odd_channel": odd_x, - "y_odd_channel": odd_y, - "x_detached_channel": detached_x, - "y_detached_channel": detached_y, - }, - dtype=float, - ) - - expected_result = pd.DataFrame( - { - "x": [1, 2, 3, 4, 5, 9, 10, 15, 21, 28], - "batch_channel": [ - np.nan, - 2, - np.nan, - 7, - np.nan, - np.nan, - 9, - np.nan, - np.nan, - 1, - ], - "epoch_channel": [ - 3, - 5, - np.nan, - np.nan, - np.nan, - np.nan, - np.nan, - np.nan, - 9, - np.nan, - ], - "odd_channel": [ - np.nan, - np.nan, - np.nan, - 3, - np.nan, - np.nan, - 15, - 4, - 21, - np.nan, - ], - "detached_channel": [ - np.nan, - np.nan, - 1, - np.nan, - 5, - 12, - np.nan, - np.nan, - np.nan, - np.nan, - ], - }, - dtype=float, - ) - expected_result = sort_df_by_columns(expected_result) - - # then - result = align_channels_on_x(df) - result = sort_df_by_columns(result) - - assert_frame_equal(result, expected_result) - - def test_fraction_x(self): - # when - - batch_x = [1.2, 0.3, 0.9, 123.4] - epoch_x = [np.nan] + [1.7, 2.9, 4.5] - - batch_y = [7.3, 2.1, 9.5, 1.2] - epoch_y = [np.nan, 0.35, 5.4, 0.9] - - df = pd.DataFrame( - { - "x_batch_channel": batch_x, - "y_batch_channel": batch_y, - "x_epoch_channel": epoch_x, - "y_epoch_channel": epoch_y, - }, - dtype=float, - ) - - expected_result = pd.DataFrame( - { - "x": [0.3, 0.9, 1.2, 1.7, 2.9, 4.5, 123.4], - "batch_channel": [2.1, 9.5, 7.3, np.nan, np.nan, np.nan, 1.2], - "epoch_channel": [np.nan, np.nan, np.nan, 0.35, 5.4, 0.9, np.nan], - }, - dtype=float, - ) - expected_result = sort_df_by_columns(expected_result) - - # then - result = align_channels_on_x(df) - result = sort_df_by_columns(result) - - assert_frame_equal(result, expected_result) - - -class TestGetChannelNameStems(unittest.TestCase): - def setUp(self): - np.random.seed(1234) - self.df = pd.DataFrame( - { - "x_batch_channel": list(range(10)), - "y_batch_channel": np.random.random(10), - "x_epoch_channel": list(range(5)) + [np.nan] * 5, - "y_epoch_channel": np.random.random(10), - "x_odd_channel": list(range(7)) + [np.nan] * 3, - "y_odd_channel": np.random.random(10), - } - ) - - def test_names(self): - correct_names = set(["epoch_channel", "batch_channel", "odd_channel"]) - self.assertEqual(set(get_channel_name_stems(self.df)), correct_names) - - -class TestMergeDataFrames(unittest.TestCase): - def setUp(self): - np.random.seed(1234) - random_df1 = np.random.random(10).tolist() - self.df1 = pd.DataFrame({"x": list(range(10)), "y1": random_df1}) - - random_df2 = np.random.random(3).tolist() - self.df2 = pd.DataFrame({"x": list(range(3)), "y2": random_df2}) - - random_df3 = np.random.random(6).tolist() - self.df3 = pd.DataFrame({"x": list(range(6)), "y3": random_df3}) - - df_merged_outer = pd.DataFrame( - { - "x": list(range(10)), - "y1": random_df1, - "y2": random_df2 + [np.nan] * 7, - "y3": random_df3 + [np.nan] * 4, - } - ) - - self.df_merged_outer = sort_df_by_columns(df_merged_outer) - - def test_merge_outer(self): - result = merge_dataframes([self.df1, self.df2, self.df3], on="x", how="outer") - result = sort_df_by_columns(result) - assert_frame_equal(result, self.df_merged_outer) - - -class TestSortDfByColumns(unittest.TestCase): - def test_letters_and_numbers(self): - sorted_df = pd.DataFrame( - columns=[ - "1", - "2", - "3", - "a", - "b", - "c", - "d", - ] - ) - shuffled_df = pd.DataFrame(columns=["c", "a", "1", "d", "3", "2", "b"]) - - assert_frame_equal(sort_df_by_columns(shuffled_df), sorted_df) - - -def times_2(x): - return x * 2 - - -if __name__ == "__main__": - unittest.main() From fdcef168d1c39dbdd67288350a3413f229f3b419 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 09:56:46 +0100 Subject: [PATCH 2/7] update tests --- .../internal/backends/test_neptune_backend_mock.py | 11 ++++++++--- tests/unit/neptune/new/test_imports.py | 3 --- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/neptune/new/internal/backends/test_neptune_backend_mock.py b/tests/unit/neptune/new/internal/backends/test_neptune_backend_mock.py index bf35d78bf..023964da8 100644 --- a/tests/unit/neptune/new/internal/backends/test_neptune_backend_mock.py +++ b/tests/unit/neptune/new/internal/backends/test_neptune_backend_mock.py @@ -14,10 +14,11 @@ # limitations under the License. # import datetime +import random +import string import unittest import uuid from pathlib import Path -from random import randint from time import time from neptune.core.components.operation_storage import OperationStorage @@ -47,7 +48,11 @@ LogFloats, LogStrings, ) -from tests.unit.neptune.legacy.random_utils import a_string + + +def a_string() -> str: + char_set = string.ascii_letters + return "".join(random.sample(char_set * 10, 10)) class TestNeptuneBackendMock(unittest.TestCase): @@ -72,7 +77,7 @@ def test_get_float_attribute(self): for container_id, container_type in self.ids_with_types: with self.subTest(f"For containerType: {container_type}"): # given - digit = randint(1, 10**4) + digit = random.randint(1, 10**4) self.backend.execute_operations( container_id, container_type, diff --git a/tests/unit/neptune/new/test_imports.py b/tests/unit/neptune/new/test_imports.py index e7a6f0cf0..bb5653b4a 100644 --- a/tests/unit/neptune/new/test_imports.py +++ b/tests/unit/neptune/new/test_imports.py @@ -70,15 +70,12 @@ NeptuneException, NeptuneFeatureNotAvailableException, NeptuneInvalidApiTokenException, - NeptuneLegacyIncompatibilityException, - NeptuneLegacyProjectException, NeptuneLimitExceedException, NeptuneLocalStorageAccessException, NeptuneMissingApiTokenException, NeptuneMissingProjectNameException, NeptuneMissingRequirementException, NeptuneOfflineModeFetchException, - NeptunePossibleLegacyUsageException, NeptuneRemoteStorageAccessException, NeptuneRemoteStorageCredentialsException, NeptuneRunResumeAndCustomIdCollision, From 9f3d2a37f01481f65f1984cfa975b76728497fd7 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 09:57:17 +0100 Subject: [PATCH 3/7] update after deleting legacy --- src/neptune/common/storage/storage_utils.py | 5 +- .../websockets/websocket_client_adapter.py | 5 +- src/neptune/exceptions.py | 69 ------------------- src/neptune/integrations/utils.py | 10 +-- .../backends/hosted_neptune_backend.py | 5 +- .../metadata_containers/metadata_container.py | 7 +- src/neptune/metadata_containers/run.py | 7 -- 7 files changed, 6 insertions(+), 102 deletions(-) diff --git a/src/neptune/common/storage/storage_utils.py b/src/neptune/common/storage/storage_utils.py index b7fe5d966..bd9d11819 100644 --- a/src/neptune/common/storage/storage_utils.py +++ b/src/neptune/common/storage/storage_utils.py @@ -32,8 +32,6 @@ Union, ) -import six - from neptune.internal.utils.logger import get_logger _logger = get_logger() @@ -164,8 +162,7 @@ def __repr__(self): return self.to_str() -@six.add_metaclass(ABCMeta) -class ProgressIndicator(object): +class ProgressIndicator(metaclass=ABCMeta): @abstractmethod def progress(self, steps): pass diff --git a/src/neptune/common/websockets/websocket_client_adapter.py b/src/neptune/common/websockets/websocket_client_adapter.py index 61696d355..d6dd0a5d0 100644 --- a/src/neptune/common/websockets/websocket_client_adapter.py +++ b/src/neptune/common/websockets/websocket_client_adapter.py @@ -16,9 +16,8 @@ import os import ssl +import urllib.parse -from future.utils import PY3 -from six.moves import urllib from websocket import ( ABNF, create_connection, @@ -62,7 +61,7 @@ def recv(self): while opcode != ABNF.OPCODE_TEXT: opcode, data = self._ws_client.recv_data() - return data.decode("utf-8") if PY3 else data + return data.decode("utf-8") @property def connected(self): diff --git a/src/neptune/exceptions.py b/src/neptune/exceptions.py index bfa0379f9..ccb4fb1c2 100644 --- a/src/neptune/exceptions.py +++ b/src/neptune/exceptions.py @@ -66,7 +66,6 @@ "NeptuneProtectedPathException", "NeptuneCannotChangeStageManually", "OperationNotSupported", - "NeptuneLegacyProjectException", "NeptuneMissingRequirementException", "NeptuneLimitExceedException", "NeptuneFieldCountLimitExceedException", @@ -74,8 +73,6 @@ "FetchAttributeNotFoundException", "ArtifactNotFoundException", "PlotlyIncompatibilityException", - "NeptunePossibleLegacyUsageException", - "NeptuneLegacyIncompatibilityException", "NeptuneUnhandledArtifactSchemeException", "NeptuneUnhandledArtifactTypeException", "NeptuneLocalStorageAccessException", @@ -772,25 +769,6 @@ def __init__(self, message: str): super().__init__(f"Operation not supported: {message}") -class NeptuneLegacyProjectException(NeptuneException): - def __init__(self, project: QualifiedName): - message = """ -{h1} -----NeptuneLegacyProjectException--------------------------------------------------------- -{end} -Your project "{project}" has not been migrated to the new structure yet. -Unfortunately the neptune.new Python API is incompatible with projects using the old structure, -so please use legacy neptune Python API. -Don't worry - we are working hard on migrating all the projects and you will be able to use the neptune.new API soon. - -You can find documentation for the legacy neptune Python API here: - - https://docs-legacy.neptune.ai/index.html - -{correct}Need help?{end}-> https://docs.neptune.ai/getting_help -""" - super().__init__(message.format(project=project, **STYLES)) - - class NeptuneMissingRequirementException(NeptuneException): def __init__(self, package_name: str, framework_name: Optional[str]): message = """ @@ -955,53 +933,6 @@ def __init__(self, matplotlib_version, plotly_version, details): ) -class NeptunePossibleLegacyUsageException(NeptuneWrongInitParametersException): - def __init__(self): - message = """ -{h1} -----NeptunePossibleLegacyUsageException---------------------------------------------------------------- -{end} -It seems you are trying to use the legacy API, but you imported the new one. - -Simply update your import statement to: - {python}import neptune{end} - -You may want to check the legacy API docs: - - https://docs-legacy.neptune.ai - -If you want to update your code with the new API, we prepared a handy migration guide: - - https://docs.neptune.ai/about/legacy/#migrating-to-neptunenew - -You can read more about neptune.new in the release blog post: - - https://neptune.ai/blog/neptune-new - -You may also want to check the following docs page: - - https://docs-legacy.neptune.ai/getting-started/integrate-neptune-into-your-codebase.html - -{correct}Need help?{end}-> https://docs.neptune.ai/getting_help -""" - super().__init__(message.format(**STYLES)) - - -class NeptuneLegacyIncompatibilityException(NeptuneException): - def __init__(self): - message = """ -{h1} -----NeptuneLegacyIncompatibilityException---------------------------------------- -{end} -It seems you are passing the legacy Experiment object, when a Run object is expected. - -What can I do? - - Updating your code to the new Python API requires few changes, but to help you with this process we prepared a handy migration guide: - https://docs.neptune.ai/about/legacy/#migrating-to-neptunenew - - You can read more about neptune.new in the release blog post: - https://neptune.ai/blog/neptune-new - -{correct}Need help?{end}-> https://docs.neptune.ai/getting_help -""" # noqa: E501 - super().__init__(message.format(**STYLES)) - - class NeptuneUnhandledArtifactSchemeException(NeptuneException): def __init__(self, path: str): scheme = urlparse(path).scheme diff --git a/src/neptune/integrations/utils.py b/src/neptune/integrations/utils.py index ab734cc41..589d9a23a 100644 --- a/src/neptune/integrations/utils.py +++ b/src/neptune/integrations/utils.py @@ -13,21 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__all__ = ["expect_not_an_experiment", "join_paths", "verify_type", "RunType"] +__all__ = ["join_paths", "verify_type", "RunType"] from typing import Union from neptune import Run -from neptune.common.experiments import LegacyExperiment as Experiment -from neptune.exceptions import NeptuneLegacyIncompatibilityException from neptune.handler import Handler from neptune.internal.utils import verify_type from neptune.internal.utils.paths import join_paths - -def expect_not_an_experiment(run: Run): - if isinstance(run, Experiment): - raise NeptuneLegacyIncompatibilityException() - - RunType = Union[Run, Handler] diff --git a/src/neptune/internal/backends/hosted_neptune_backend.py b/src/neptune/internal/backends/hosted_neptune_backend.py index bf50d3f15..639aa58bc 100644 --- a/src/neptune/internal/backends/hosted_neptune_backend.py +++ b/src/neptune/internal/backends/hosted_neptune_backend.py @@ -61,7 +61,6 @@ MetadataContainerNotFound, MetadataInconsistency, NeptuneFeatureNotAvailableException, - NeptuneLegacyProjectException, NeptuneLimitExceedException, NeptuneObjectCreationConflict, ProjectNotFound, @@ -239,9 +238,7 @@ def get_project(self, project_id: QualifiedName) -> Project: **DEFAULT_REQUEST_KWARGS, ).response() project = response.result - project_version = project.version if hasattr(project, "version") else 1 - if project_version < 2: - raise NeptuneLegacyProjectException(project_id) + return Project( id=project.id, name=project.name, diff --git a/src/neptune/metadata_containers/metadata_container.py b/src/neptune/metadata_containers/metadata_container.py index 4dfd7e4e3..5dddf9036 100644 --- a/src/neptune/metadata_containers/metadata_container.py +++ b/src/neptune/metadata_containers/metadata_container.py @@ -50,10 +50,7 @@ NEPTUNE_ENABLE_DEFAULT_ASYNC_LAG_CALLBACK, NEPTUNE_ENABLE_DEFAULT_ASYNC_NO_PROGRESS_CALLBACK, ) -from neptune.exceptions import ( - MetadataInconsistency, - NeptunePossibleLegacyUsageException, -) +from neptune.exceptions import MetadataInconsistency from neptune.handler import Handler from neptune.internal.backends.api_model import ( ApiExperiment, @@ -321,8 +318,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.stop() def __getattr__(self, item): - if item in self.LEGACY_METHODS: - raise NeptunePossibleLegacyUsageException() raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{item}'") @abc.abstractmethod diff --git a/src/neptune/metadata_containers/run.py b/src/neptune/metadata_containers/run.py index 6ff38dd82..98151e017 100644 --- a/src/neptune/metadata_containers/run.py +++ b/src/neptune/metadata_containers/run.py @@ -49,7 +49,6 @@ from neptune.exceptions import ( InactiveRunException, NeedExistingRunForReadOnlyMode, - NeptunePossibleLegacyUsageException, NeptuneRunResumeAndCustomIdCollision, ) from neptune.internal.backends.api_model import ApiExperiment @@ -579,12 +578,6 @@ def generate_monitoring_namespace(*descriptors) -> str: def check_for_extra_kwargs(caller_name: str, kwargs: dict): - legacy_kwargs = ("project_qualified_name", "backend") - - for name in legacy_kwargs: - if name in kwargs: - raise NeptunePossibleLegacyUsageException() - if kwargs: first_key = next(iter(kwargs.keys())) raise TypeError(f"{caller_name}() got an unexpected keyword argument '{first_key}'") From 4b06c683d4c6f6c2987d867d6d47d966c0e3a34a Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 10:00:06 +0100 Subject: [PATCH 4/7] remove LEGACY_METHODS --- .../metadata_containers/metadata_container.py | 2 -- src/neptune/metadata_containers/run.py | 23 ------------------- 2 files changed, 25 deletions(-) diff --git a/src/neptune/metadata_containers/metadata_container.py b/src/neptune/metadata_containers/metadata_container.py index 5dddf9036..8175e93ee 100644 --- a/src/neptune/metadata_containers/metadata_container.py +++ b/src/neptune/metadata_containers/metadata_container.py @@ -120,8 +120,6 @@ def inner_fun(self: "MetadataContainer", *args, **kwargs): class MetadataContainer(AbstractContextManager, NeptuneObject): container_type: ContainerType - LEGACY_METHODS = set() - def __init__( self, *, diff --git a/src/neptune/metadata_containers/run.py b/src/neptune/metadata_containers/run.py index 98151e017..3a6b3766d 100644 --- a/src/neptune/metadata_containers/run.py +++ b/src/neptune/metadata_containers/run.py @@ -280,29 +280,6 @@ class Run(MetadataContainer): container_type = ContainerType.RUN - LEGACY_METHODS = ( - "create_experiment", - "send_metric", - "log_metric", - "send_text", - "log_text", - "send_image", - "log_image", - "send_artifact", - "log_artifact", - "delete_artifacts", - "download_artifact", - "download_sources", - "download_artifacts", - "reset_log", - "get_parameters", - "get_properties", - "set_property", - "remove_property", - "get_hardware_utilization", - "get_numeric_channels_values", - ) - def __init__( self, with_id: Optional[str] = None, From 0b02ed4f2cd73fc1dcf6a519f7177305d5d0420c Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 10:14:38 +0100 Subject: [PATCH 5/7] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad6fb9846..7ff1b7fe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Breaking changes - Deleted `neptune.new` package ([#1684](https://github.com/neptune-ai/neptune-client/pull/1684)) +- Deleted `neptune.legacy` package ([#1685](https://github.com/neptune-ai/neptune-client/pull/1685)) ### Features - ? From 2296dbf9ccbb806906a95449cef2c415e8e06b63 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 11:07:41 +0100 Subject: [PATCH 6/7] update codecov.yml --- codecov.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/codecov.yml b/codecov.yml index eb18e828b..2cc19725e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -81,16 +81,10 @@ component_management: - src/neptune/logging/** - src/neptune/metadata_containers/** - src/neptune/new/** - - src/netpune/types/** + - src/neptune/types/** - src/neptune/vendor/** - src/neptune/*.py - - component_id: legacy - name: legacy - paths: - - src/neptune/legacy/** - - src/neptune/common/** - - component_id: management name: management paths: From 1c0aeec538754dc99ca60034c4b82ba939eb38a0 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Mar 2024 11:20:35 +0100 Subject: [PATCH 7/7] remove legacy mention from pre-commit config --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a830f1c60..3eee51ec8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: rev: v1.5.4 hooks: - id: insert-license - files: ^src/neptune/(?!new/|legacy/|vendor/)[^/]+(?:/[^/]+)*\.py$ + files: ^src/neptune/(?!new/|vendor/)[^/]+(?:/[^/]+)*\.py$ args: [ "--license-filepath", ".github/license_header.txt", "--allow-past-years"] - repo: https://github.com/pycqa/flake8 rev: 5.0.4