diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b1bd7d8b..8fb73a521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Handle logging in the `AsyncOperationProcessor` with `OperationLogger` and signal queue ([#1610](https://github.com/neptune-ai/neptune-client/pull/1610)) - Stringify `Handler` paths ([#1623](https://github.com/neptune-ai/neptune-client/pull/1623)) - Added processor id to `ProcessorStopSignalData` ([#1625](https://github.com/neptune-ai/neptune-client/pull/1625)) +- Use the same logger instance for logging ([#1611](https://github.com/neptune-ai/neptune-client/pull/1611)) ## 1.8.6 diff --git a/src/neptune/attributes/atoms/integer.py b/src/neptune/attributes/atoms/integer.py index 98955bf08..000e2fc1a 100644 --- a/src/neptune/attributes/atoms/integer.py +++ b/src/neptune/attributes/atoms/integer.py @@ -24,12 +24,14 @@ ) from neptune.internal.container_type import ContainerType from neptune.internal.operation import AssignInt -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.types.atoms.integer import Integer as IntegerVal if typing.TYPE_CHECKING: from neptune.internal.backends.neptune_backend import NeptuneBackend +logger = get_logger() + class Integer(CopiableAtom): @staticmethod diff --git a/src/neptune/attributes/atoms/string.py b/src/neptune/attributes/atoms/string.py index 5749ec36b..4ddc040b0 100644 --- a/src/neptune/attributes/atoms/string.py +++ b/src/neptune/attributes/atoms/string.py @@ -20,7 +20,7 @@ from neptune.attributes.atoms.copiable_atom import CopiableAtom from neptune.internal.container_type import ContainerType from neptune.internal.operation import AssignString -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import path_to_str from neptune.types.atoms.string import String as StringVal @@ -28,6 +28,8 @@ from neptune.internal.backends.neptune_backend import NeptuneBackend from neptune.metadata_containers import MetadataContainer +logger = get_logger() + class String(CopiableAtom): diff --git a/src/neptune/attributes/namespace.py b/src/neptune/attributes/namespace.py index 0d1345651..ba80c9cc2 100644 --- a/src/neptune/attributes/namespace.py +++ b/src/neptune/attributes/namespace.py @@ -36,7 +36,7 @@ NoValue, atomic_attribute_types_map, ) -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import ( parse_path, path_to_str, @@ -46,6 +46,7 @@ if TYPE_CHECKING: from neptune.metadata_containers import MetadataContainer +logger = get_logger() RunStructure = ContainerStructure # backwards compatibility diff --git a/src/neptune/attributes/series/string_series.py b/src/neptune/attributes/series/string_series.py index e46146e18..a6edd6183 100644 --- a/src/neptune/attributes/series/string_series.py +++ b/src/neptune/attributes/series/string_series.py @@ -31,7 +31,7 @@ Operation, ) from neptune.internal.utils import is_collection -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import path_to_str from neptune.types.series.string_series import MAX_STRING_SERIES_VALUE_LENGTH from neptune.types.series.string_series import StringSeries as StringSeriesVal @@ -43,6 +43,8 @@ Data = str LogOperation = LogStrings +logger = get_logger() + class StringSeries( Series[Val, Data, LogOperation], FetchableSeries[StringSeriesValues], max_batch_size=10, operation_cls=LogOperation diff --git a/src/neptune/cli/clear.py b/src/neptune/cli/clear.py index bc9d96afd..c8b52ba35 100644 --- a/src/neptune/cli/clear.py +++ b/src/neptune/cli/clear.py @@ -31,12 +31,14 @@ from neptune.cli.status import StatusRunner from neptune.cli.utils import get_offline_dirs from neptune.constants import SYNC_DIRECTORY -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger if TYPE_CHECKING: from neptune.internal.backends.api_model import ApiExperiment from neptune.internal.id_formats import UniqueId +logger = get_logger(with_prefix=False) + class ClearRunner(AbstractBackendRunner): def clear(self, path: Path, force: bool = False, clear_eventual: bool = True) -> None: diff --git a/src/neptune/cli/status.py b/src/neptune/cli/status.py index d5057bdb0..8b5878597 100644 --- a/src/neptune/cli/status.py +++ b/src/neptune/cli/status.py @@ -33,7 +33,9 @@ from neptune.constants import OFFLINE_NAME_PREFIX from neptune.envs import PROJECT_ENV_NAME from neptune.internal.backends.api_model import ApiExperiment -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger + +logger = get_logger(with_prefix=False) offline_run_explainer = """ Runs that execute offline are not created on the server and are not assigned to projects; diff --git a/src/neptune/cli/sync.py b/src/neptune/cli/sync.py index 3e1641a71..8d18794e9 100644 --- a/src/neptune/cli/sync.py +++ b/src/neptune/cli/sync.py @@ -16,7 +16,6 @@ __all__ = ["SyncRunner"] -import logging import os import threading import time @@ -61,8 +60,9 @@ ) from neptune.internal.operation import Operation from neptune.internal.operation_processors.operation_storage import OperationStorage -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger +logger = get_logger(with_prefix=False) retries_timeout = int(os.getenv(NEPTUNE_SYNC_BATCH_TIMEOUT_ENV, "3600")) @@ -186,7 +186,7 @@ def _register_offline_container(self, project: Project, container_type: Containe logger.warning( "Exception occurred while trying to create a run" " on the Neptune server. Please try again later", ) - logging.exception(e) + logger.exception(e) return None @staticmethod diff --git a/src/neptune/cli/utils.py b/src/neptune/cli/utils.py index dfaf8363f..484822bea 100644 --- a/src/neptune/cli/utils.py +++ b/src/neptune/cli/utils.py @@ -24,7 +24,6 @@ "split_dir_name", ] -import logging import os import textwrap import threading @@ -59,7 +58,9 @@ UniqueId, ) from neptune.internal.operation import Operation -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger + +logger = get_logger(with_prefix=False) def get_metadata_container( @@ -74,7 +75,7 @@ def get_metadata_container( logger.warning("Can't fetch %s %s. Skipping.", public_container_type, container_id) except NeptuneException as e: logger.warning("Exception while fetching %s %s. Skipping.", public_container_type, container_id) - logging.exception(e) + logger.exception(e) return None diff --git a/src/neptune/common/backends/utils.py b/src/neptune/common/backends/utils.py index d1f24fbc6..159b1f7d3 100644 --- a/src/neptune/common/backends/utils.py +++ b/src/neptune/common/backends/utils.py @@ -16,7 +16,6 @@ __all__ = ["with_api_exceptions_handler", "get_retry_from_headers_or_default"] import itertools -import logging import os import time @@ -47,8 +46,9 @@ NeptuneInvalidApiTokenException, Unauthorized, ) +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +_logger = get_logger() MAX_RETRY_TIME = 30 MAX_RETRY_MULTIPLIER = 10 diff --git a/src/neptune/common/hardware/gpu/gpu_monitor.py b/src/neptune/common/hardware/gpu/gpu_monitor.py index 975d3b566..d06ebdabf 100644 --- a/src/neptune/common/hardware/gpu/gpu_monitor.py +++ b/src/neptune/common/hardware/gpu/gpu_monitor.py @@ -14,8 +14,10 @@ # limitations under the License. # -import logging - +from neptune.common.warnings import ( + NeptuneWarning, + warn_once, +) from neptune.vendor.pynvml import ( NVMLError, nvmlDeviceGetCount, @@ -25,8 +27,6 @@ nvmlInit, ) -_logger = logging.getLogger(__name__) - class GPUMonitor(object): @@ -62,14 +62,12 @@ def __nvml_get_or_else(self, getter, default=None): try: nvmlInit() return getter() - except NVMLError as e: + except NVMLError: if not GPUMonitor.nvml_error_printed: warning = ( "Info (NVML): %s. GPU usage metrics may not be reported. For more information, " - "see https://docs-legacy.neptune.ai/logging-and-managing-experiment-results" - "/logging-experiment" - "-data.html#hardware-consumption " + "see https://docs.neptune.ai/help/nvml_error/" ) - _logger.warning(warning, e) + warn_once(message=warning, exception=NeptuneWarning) GPUMonitor.nvml_error_printed = True return default diff --git a/src/neptune/common/storage/storage_utils.py b/src/neptune/common/storage/storage_utils.py index 8d67adec2..b7fe5d966 100644 --- a/src/neptune/common/storage/storage_utils.py +++ b/src/neptune/common/storage/storage_utils.py @@ -14,7 +14,6 @@ # limitations under the License. # import io -import logging import os import stat import time @@ -35,7 +34,9 @@ import six -_logger = logging.getLogger(__name__) +from neptune.internal.utils.logger import get_logger + +_logger = get_logger() @dataclass diff --git a/src/neptune/common/utils.py b/src/neptune/common/utils.py index 5883e233a..75b2996c1 100644 --- a/src/neptune/common/utils.py +++ b/src/neptune/common/utils.py @@ -16,7 +16,6 @@ import functools import glob as globlib -import logging import math import os import re @@ -37,8 +36,9 @@ ) from neptune.common.git_info import GitInfo from neptune.common.patterns import PROJECT_QUALIFIED_NAME_PATTERN +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +_logger = get_logger() IS_WINDOWS = sys.platform == "win32" IS_MACOS = sys.platform == "darwin" diff --git a/src/neptune/internal/backends/hosted_file_operations.py b/src/neptune/internal/backends/hosted_file_operations.py index 1c1034734..800cf5655 100644 --- a/src/neptune/internal/backends/hosted_file_operations.py +++ b/src/neptune/internal/backends/hosted_file_operations.py @@ -85,8 +85,9 @@ get_absolute_paths, get_common_root, ) -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger +logger = get_logger() DEFAULT_CHUNK_SIZE = 5 * BYTES_IN_ONE_MB DEFAULT_UPLOAD_CONFIG = AttributeUploadConfiguration(chunk_size=DEFAULT_CHUNK_SIZE) diff --git a/src/neptune/internal/backends/hosted_neptune_backend.py b/src/neptune/internal/backends/hosted_neptune_backend.py index e02684676..5ba9c381e 100644 --- a/src/neptune/internal/backends/hosted_neptune_backend.py +++ b/src/neptune/internal/backends/hosted_neptune_backend.py @@ -16,7 +16,6 @@ __all__ = ["HostedNeptuneBackend"] import itertools -import logging import os import re import typing @@ -136,6 +135,7 @@ from neptune.internal.utils import base64_decode from neptune.internal.utils.generic_attribute_mapper import map_attribute_result_to_value from neptune.internal.utils.git import GitInfo +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import path_to_str from neptune.internal.websockets.websockets_factory import WebsocketsFactory from neptune.management.exceptions import ObjectNotFound @@ -147,7 +147,7 @@ from neptune.internal.backends.api_model import ClientConfig -_logger = logging.getLogger(__name__) +_logger = get_logger() class HostedNeptuneBackend(NeptuneBackend): diff --git a/src/neptune/internal/backends/project_name_lookup.py b/src/neptune/internal/backends/project_name_lookup.py index 12c2af289..d9c78b202 100644 --- a/src/neptune/internal/backends/project_name_lookup.py +++ b/src/neptune/internal/backends/project_name_lookup.py @@ -15,7 +15,6 @@ # __all__ = ["project_name_lookup"] -import logging import os from typing import Optional @@ -25,8 +24,9 @@ from neptune.internal.backends.neptune_backend import NeptuneBackend from neptune.internal.id_formats import QualifiedName from neptune.internal.utils import verify_type +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +_logger = get_logger() def project_name_lookup(backend: NeptuneBackend, name: Optional[QualifiedName] = None) -> Project: diff --git a/src/neptune/internal/backends/utils.py b/src/neptune/internal/backends/utils.py index ea724ced9..ffc6ee93a 100644 --- a/src/neptune/internal/backends/utils.py +++ b/src/neptune/internal/backends/utils.py @@ -29,7 +29,6 @@ ] import dataclasses -import logging import os import socket from functools import ( @@ -78,9 +77,9 @@ Operation, ) from neptune.internal.utils import replace_patch_version -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +logger = get_logger() if TYPE_CHECKING: from neptune.internal.backends.neptune_backend import NeptuneBackend diff --git a/src/neptune/internal/disk_queue.py b/src/neptune/internal/disk_queue.py index 2bd2e5e35..1d0aee6d9 100644 --- a/src/neptune/internal/disk_queue.py +++ b/src/neptune/internal/disk_queue.py @@ -16,7 +16,6 @@ __all__ = ["QueueElement", "DiskQueue"] import json -import logging import os import shutil import threading @@ -39,12 +38,13 @@ should_clean_internal_data, ) from neptune.internal.utils.json_file_splitter import JsonFileSplitter +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.sync_offset_file import SyncOffsetFile T = TypeVar("T") Timestamp = float -_logger = logging.getLogger(__name__) +_logger = get_logger() @dataclass diff --git a/src/neptune/internal/hardware/gpu/gpu_monitor.py b/src/neptune/internal/hardware/gpu/gpu_monitor.py index c5272f7e5..ab683b19e 100644 --- a/src/neptune/internal/hardware/gpu/gpu_monitor.py +++ b/src/neptune/internal/hardware/gpu/gpu_monitor.py @@ -15,8 +15,7 @@ # __all__ = ["GPUMonitor"] -import logging - +from neptune.internal.utils.logger import get_logger from neptune.vendor.pynvml import ( NVMLError, nvmlDeviceGetCount, @@ -26,7 +25,7 @@ nvmlInit, ) -_logger = logging.getLogger(__name__) +_logger = get_logger() class GPUMonitor(object): diff --git a/src/neptune/internal/hardware/hardware_metric_reporting_job.py b/src/neptune/internal/hardware/hardware_metric_reporting_job.py index fdd503ef8..78c9bbb8c 100644 --- a/src/neptune/internal/hardware/hardware_metric_reporting_job.py +++ b/src/neptune/internal/hardware/hardware_metric_reporting_job.py @@ -15,7 +15,6 @@ # __all__ = ["HardwareMetricReportingJob"] -import logging import os import time from itertools import groupby @@ -36,12 +35,13 @@ from neptune.internal.background_job import BackgroundJob from neptune.internal.hardware.gpu.gpu_monitor import GPUMonitor from neptune.internal.threading.daemon import Daemon +from neptune.internal.utils.logger import get_logger from neptune.types.series import FloatSeries if TYPE_CHECKING: from neptune.metadata_containers import MetadataContainer -_logger = logging.getLogger(__name__) +_logger = get_logger() class HardwareMetricReportingJob(BackgroundJob): diff --git a/src/neptune/internal/notebooks/comm.py b/src/neptune/internal/notebooks/comm.py index 5143b62f9..8746157e9 100644 --- a/src/neptune/internal/notebooks/comm.py +++ b/src/neptune/internal/notebooks/comm.py @@ -15,9 +15,9 @@ # __all__ = ["send_checkpoint_created"] -import logging +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +_logger = get_logger() class MessageType(object): diff --git a/src/neptune/internal/notebooks/notebooks.py b/src/neptune/internal/notebooks/notebooks.py index 09fdf574c..07f55f462 100644 --- a/src/neptune/internal/notebooks/notebooks.py +++ b/src/neptune/internal/notebooks/notebooks.py @@ -15,14 +15,14 @@ # __all__ = ["create_checkpoint"] -import logging import threading from neptune.internal.backends.neptune_backend import NeptuneBackend from neptune.internal.notebooks.comm import send_checkpoint_created from neptune.internal.utils import is_ipython +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +_logger = get_logger() _checkpoints_lock = threading.Lock() _checkpoints = dict() diff --git a/src/neptune/internal/operation_processors/async_operation_processor.py b/src/neptune/internal/operation_processors/async_operation_processor.py index 403f1f8d6..e55f18b54 100644 --- a/src/neptune/internal/operation_processors/async_operation_processor.py +++ b/src/neptune/internal/operation_processors/async_operation_processor.py @@ -60,7 +60,7 @@ from neptune.internal.threading.daemon import Daemon from neptune.internal.utils.disk_utilization import ensure_disk_not_overutilize from neptune.internal.utils.files import should_clean_internal_data -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger if TYPE_CHECKING: from neptune.internal.backends.neptune_backend import NeptuneBackend @@ -69,6 +69,8 @@ from neptune.internal.operation_processors.operation_logger import ProcessorStopSignal from neptune.internal.signals_processing.signals import Signal +logger = get_logger() + class AsyncOperationProcessor(OperationProcessor): STOP_QUEUE_STATUS_UPDATE_FREQ_SECONDS = 30.0 diff --git a/src/neptune/internal/operation_processors/operation_logger.py b/src/neptune/internal/operation_processors/operation_logger.py index 46e0a6ce1..a8a223534 100644 --- a/src/neptune/internal/operation_processors/operation_logger.py +++ b/src/neptune/internal/operation_processors/operation_logger.py @@ -122,7 +122,7 @@ def log_remaining_operations(self, size_remaining: int) -> None: ) else: if size_remaining: - self._logger.warning( + self._logger.info( WAITING_FOR_OPERATIONS_MSG, size_remaining, ) diff --git a/src/neptune/internal/threading/daemon.py b/src/neptune/internal/threading/daemon.py index 57f542c6c..49f1f256e 100644 --- a/src/neptune/internal/threading/daemon.py +++ b/src/neptune/internal/threading/daemon.py @@ -21,7 +21,9 @@ from enum import Enum from neptune.common.exceptions import NeptuneConnectionLostException -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger + +logger = get_logger() class Daemon(threading.Thread): diff --git a/src/neptune/internal/types/stringify_value.py b/src/neptune/internal/types/stringify_value.py index 3591f0a2b..518fcc8e4 100644 --- a/src/neptune/internal/types/stringify_value.py +++ b/src/neptune/internal/types/stringify_value.py @@ -38,7 +38,9 @@ MAX_32_BIT_INT, MIN_32_BIT_INT, ) -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger + +logger = get_logger() def is_unsupported_float(value) -> bool: diff --git a/src/neptune/internal/utils/__init__.py b/src/neptune/internal/utils/__init__.py index cc8845f93..78fdefc91 100644 --- a/src/neptune/internal/utils/__init__.py +++ b/src/neptune/internal/utils/__init__.py @@ -38,7 +38,6 @@ ] import base64 -import logging import os from glob import glob from io import IOBase @@ -53,10 +52,11 @@ ) from neptune.internal.types.stringify_value import StringifyValue +from neptune.internal.utils.logger import get_logger T = TypeVar("T") -_logger = logging.getLogger(__name__) +_logger = get_logger() def replace_patch_version(version: str): diff --git a/src/neptune/internal/utils/dependency_tracking.py b/src/neptune/internal/utils/dependency_tracking.py index 2d25ef7bd..1aefaa351 100644 --- a/src/neptune/internal/utils/dependency_tracking.py +++ b/src/neptune/internal/utils/dependency_tracking.py @@ -31,6 +31,8 @@ Union, ) +from neptune.internal.utils.logger import get_logger + if sys.version_info >= (3, 8): from importlib.metadata import ( Distribution, @@ -39,12 +41,13 @@ else: from importlib_metadata import Distribution, distributions -from neptune.internal.utils.logger import logger from neptune.types import File if TYPE_CHECKING: from neptune import Run +logger = get_logger() + class DependencyTrackingStrategy(ABC): @abstractmethod diff --git a/src/neptune/internal/utils/files.py b/src/neptune/internal/utils/files.py index 6db544fda..6ec090b19 100644 --- a/src/neptune/internal/utils/files.py +++ b/src/neptune/internal/utils/files.py @@ -19,7 +19,9 @@ from pathlib import Path from neptune.envs import NEPTUNE_DISABLE_PARENT_DIR_DELETION -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger + +logger = get_logger() def remove_parent_folder_if_allowed(path: Path) -> None: diff --git a/src/neptune/internal/utils/git.py b/src/neptune/internal/utils/git.py index 15f238236..c8cb5307b 100644 --- a/src/neptune/internal/utils/git.py +++ b/src/neptune/internal/utils/git.py @@ -19,7 +19,6 @@ "track_uncommitted_changes", ] -import logging import warnings from dataclasses import dataclass from datetime import datetime @@ -34,6 +33,7 @@ DIFF_HEAD_INDEX_PATH, UPSTREAM_INDEX_DIFF, ) +from neptune.internal.utils.logger import get_logger from neptune.types import File from neptune.types.atoms.git_ref import ( GitRef, @@ -45,7 +45,7 @@ from neptune import Run -_logger = logging.getLogger(__name__) +_logger = get_logger() @dataclass diff --git a/src/neptune/internal/utils/images.py b/src/neptune/internal/utils/images.py index 731b58ce4..365ad43e1 100644 --- a/src/neptune/internal/utils/images.py +++ b/src/neptune/internal/utils/images.py @@ -29,7 +29,6 @@ import base64 import io -import logging import pickle import warnings from io import ( @@ -42,9 +41,9 @@ from pandas import DataFrame from neptune.exceptions import PlotlyIncompatibilityException -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger -_logger = logging.getLogger(__name__) +logger = get_logger() SEABORN_GRID_CLASSES = {"FacetGrid", "PairGrid", "JointGrid"} try: diff --git a/src/neptune/internal/utils/limits.py b/src/neptune/internal/utils/limits.py index 6913eb084..294973ade 100644 --- a/src/neptune/internal/utils/limits.py +++ b/src/neptune/internal/utils/limits.py @@ -15,10 +15,11 @@ # __all__ = ["custom_run_id_exceeds_length", "image_size_exceeds_limit_for_logging"] -import logging import warnings -_logger = logging.getLogger(__name__) +from neptune.internal.utils.logger import get_logger + +_logger = get_logger() _CUSTOM_RUN_ID_LENGTH = 36 diff --git a/src/neptune/internal/utils/logger.py b/src/neptune/internal/utils/logger.py index e3697e94d..4375a17fd 100644 --- a/src/neptune/internal/utils/logger.py +++ b/src/neptune/internal/utils/logger.py @@ -13,12 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__all__ = ["logger"] +__all__ = ["get_logger", "NEPTUNE_LOGGER_NAME"] import logging import sys -LOGGER_NAME = "neptune-client" +NEPTUNE_LOGGER_NAME = "neptune" +NEPTUNE_NO_PREFIX_LOGGER_NAME = "neptune_no_prefix" +LOG_FORMAT = "[%(name)s] [%(levelname)s] %(message)s" +NO_PREFIX_FORMAT = "%(message)s" + + +class CustomFormatter(logging.Formatter): + def format(self, record): + record.levelname = record.levelname.lower().ljust(len("warning")) + formatter = logging.Formatter(LOG_FORMAT) + return formatter.format(record) class GrabbableStdoutHandler(logging.StreamHandler): @@ -39,10 +49,33 @@ def stream(self): return sys.stdout -logger = logging.getLogger(LOGGER_NAME) +def get_logger(with_prefix: bool = True): + if with_prefix: + return logging.getLogger(NEPTUNE_LOGGER_NAME) + return logging.getLogger(NEPTUNE_NO_PREFIX_LOGGER_NAME) + + +def _set_up_logging(): + # setup neptune logger so that logging.getLogger(NEPTUNE_LOGGER_NAME) + # returns configured logger + neptune_logger = logging.getLogger(NEPTUNE_LOGGER_NAME) + neptune_logger.propagate = False + neptune_logger.setLevel(logging.DEBUG) + + stdout_handler = GrabbableStdoutHandler() + stdout_handler.setFormatter(CustomFormatter()) + neptune_logger.addHandler(stdout_handler) + + +def _set_up_no_prefix_logging(): + neptune_logger = logging.getLogger(NEPTUNE_NO_PREFIX_LOGGER_NAME) + neptune_logger.propagate = False + neptune_logger.setLevel(logging.DEBUG) + + stdout_handler = GrabbableStdoutHandler() + stdout_handler.setFormatter(logging.Formatter(NO_PREFIX_FORMAT)) + neptune_logger.addHandler(stdout_handler) + -logger.propagate = False -logger.setLevel(level=logging.DEBUG) -stdout_handler = GrabbableStdoutHandler() -stdout_handler.setFormatter(logging.Formatter("%(message)s")) -logger.addHandler(stdout_handler) +_set_up_logging() +_set_up_no_prefix_logging() diff --git a/src/neptune/internal/utils/ping_background_job.py b/src/neptune/internal/utils/ping_background_job.py index 6846b08c2..921d3590b 100644 --- a/src/neptune/internal/utils/ping_background_job.py +++ b/src/neptune/internal/utils/ping_background_job.py @@ -15,7 +15,6 @@ # __all__ = ["PingBackgroundJob"] -import logging from typing import ( TYPE_CHECKING, Optional, @@ -23,11 +22,12 @@ from neptune.internal.background_job import BackgroundJob from neptune.internal.threading.daemon import Daemon +from neptune.internal.utils.logger import get_logger if TYPE_CHECKING: from neptune.metadata_containers import MetadataContainer -_logger = logging.getLogger(__name__) +_logger = get_logger() class PingBackgroundJob(BackgroundJob): diff --git a/src/neptune/internal/utils/traceback_job.py b/src/neptune/internal/utils/traceback_job.py index 5e00f2e91..8908cdb5b 100644 --- a/src/neptune/internal/utils/traceback_job.py +++ b/src/neptune/internal/utils/traceback_job.py @@ -15,7 +15,6 @@ # __all__ = ["TracebackJob"] -import logging import uuid from typing import ( TYPE_CHECKING, @@ -25,12 +24,13 @@ from neptune.attributes.constants import SYSTEM_FAILED_ATTRIBUTE_PATH from neptune.internal.background_job import BackgroundJob +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.uncaught_exception_handler import instance as traceback_handler if TYPE_CHECKING: from neptune.metadata_containers import MetadataContainer -_logger = logging.getLogger(__name__) +_logger = get_logger() class TracebackJob(BackgroundJob): diff --git a/src/neptune/internal/utils/uncaught_exception_handler.py b/src/neptune/internal/utils/uncaught_exception_handler.py index b524f24d1..98a6ca3da 100644 --- a/src/neptune/internal/utils/uncaught_exception_handler.py +++ b/src/neptune/internal/utils/uncaught_exception_handler.py @@ -15,7 +15,6 @@ # __all__ = ["instance"] -import logging import sys import threading import traceback @@ -27,10 +26,12 @@ List, ) +from neptune.internal.utils.logger import get_logger + if TYPE_CHECKING: pass -_logger = logging.getLogger(__name__) +_logger = get_logger() class UncaughtExceptionHandler: diff --git a/src/neptune/internal/websockets/websocket_signals_background_job.py b/src/neptune/internal/websockets/websocket_signals_background_job.py index 8e493a531..04335c1e3 100644 --- a/src/neptune/internal/websockets/websocket_signals_background_job.py +++ b/src/neptune/internal/websockets/websocket_signals_background_job.py @@ -16,7 +16,6 @@ __all__ = ["WebsocketSignalsBackgroundJob"] import json -import logging import threading from json.decoder import JSONDecodeError from typing import ( @@ -35,13 +34,13 @@ from neptune.internal.background_job import BackgroundJob from neptune.internal.threading.daemon import Daemon from neptune.internal.utils import process_killer -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.internal.websockets.websockets_factory import WebsocketsFactory if TYPE_CHECKING: from neptune.metadata_containers import MetadataContainer -_logger = logging.getLogger(__name__) +logger = get_logger() class WebsocketSignalsBackgroundJob(BackgroundJob): diff --git a/src/neptune/management/internal/api.py b/src/neptune/management/internal/api.py index eaf4db29c..e400e7a9b 100644 --- a/src/neptune/management/internal/api.py +++ b/src/neptune/management/internal/api.py @@ -70,7 +70,7 @@ verify_type, ) from neptune.internal.utils.iteration import get_batches -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.management.exceptions import ( AccessRevokedOnDeletion, AccessRevokedOnMemberRemoval, @@ -100,6 +100,7 @@ normalize_project_name, ) +logger = get_logger() TRASH_BATCH_SIZE = 100 diff --git a/src/neptune/metadata_containers/metadata_container.py b/src/neptune/metadata_containers/metadata_container.py index b332b1ec3..56dddb0a8 100644 --- a/src/neptune/metadata_containers/metadata_container.py +++ b/src/neptune/metadata_containers/metadata_container.py @@ -84,7 +84,7 @@ verify_optional_callable, verify_type, ) -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import parse_path from neptune.internal.utils.uncaught_exception_handler import instance as uncaught_exception_handler from neptune.internal.value_to_attribute_visitor import ValueToAttributeVisitor @@ -100,6 +100,8 @@ if TYPE_CHECKING: from neptune.internal.signals_processing.signals import Signal +logger = get_logger() + def ensure_not_stopped(fun): @wraps(fun) @@ -450,8 +452,8 @@ def stop(self, *, seconds: Optional[Union[float, int]] = None) -> None: self._op_processor.stop(sec_left) if self._mode not in {Mode.OFFLINE, Mode.DEBUG}: - logger.info("Explore the metadata in the Neptune app:") - logger.info(self.get_url().rstrip("/") + "/metadata") + metadata_url = self.get_url().rstrip("/") + "/metadata" + logger.info(f"Explore the metadata in the Neptune app: {metadata_url}") self._backend.close() with self._forking_cond: @@ -643,7 +645,7 @@ def get_url(self) -> str: def _startup(self, debug_mode): if not debug_mode: - logger.info(self.get_url()) + logger.info(f"Neptune initialized. Open in the app: {self.get_url()}") self.start() diff --git a/src/neptune/metadata_containers/metadata_containers_table.py b/src/neptune/metadata_containers/metadata_containers_table.py index 5ebf49992..f2a912ecc 100644 --- a/src/neptune/metadata_containers/metadata_containers_table.py +++ b/src/neptune/metadata_containers/metadata_containers_table.py @@ -15,7 +15,6 @@ # __all__ = ["Table"] -import logging from datetime import datetime from typing import ( Any, @@ -33,13 +32,14 @@ ) from neptune.internal.backends.neptune_backend import NeptuneBackend from neptune.internal.container_type import ContainerType +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import ( join_paths, parse_path, ) from neptune.internal.utils.run_state import RunState -logger = logging.getLogger(__name__) +logger = get_logger() class TableEntry: diff --git a/src/neptune/types/namespace.py b/src/neptune/types/namespace.py index ede202355..e03ae8ae5 100644 --- a/src/neptune/types/namespace.py +++ b/src/neptune/types/namespace.py @@ -21,13 +21,14 @@ TypeVar, ) -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.internal.utils.paths import parse_path from neptune.types.value import Value if TYPE_CHECKING: from neptune.types.value_visitor import ValueVisitor +logger = get_logger() Ret = TypeVar("Ret") diff --git a/src/neptune/types/series/file_series.py b/src/neptune/types/series/file_series.py index c0f27a52e..41bb4fc48 100644 --- a/src/neptune/types/series/file_series.py +++ b/src/neptune/types/series/file_series.py @@ -27,13 +27,14 @@ from neptune.internal.types.stringify_value import extract_if_stringify_value from neptune.internal.utils import is_collection -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.types import File from neptune.types.series.series import Series if TYPE_CHECKING: from neptune.types.value_visitor import ValueVisitor +logger = get_logger() Ret = TypeVar("Ret") diff --git a/src/neptune/utils.py b/src/neptune/utils.py index 298a5b41e..7c64a7199 100644 --- a/src/neptune/utils.py +++ b/src/neptune/utils.py @@ -25,9 +25,11 @@ from neptune.internal.init.parameters import DEFAULT_STOP_TIMEOUT from neptune.internal.types.stringify_value import StringifyValue -from neptune.internal.utils.logger import logger +from neptune.internal.utils.logger import get_logger from neptune.typing import NeptuneObject +logger = get_logger() + def stringify_unsupported(value: Any) -> Union[StringifyValue, Mapping]: """Helper function that converts unsupported values in a collection or dictionary to strings. diff --git a/tests/unit/neptune/new/cli/test_status.py b/tests/unit/neptune/new/cli/test_status.py index 531d36706..618e23a6a 100644 --- a/tests/unit/neptune/new/cli/test_status.py +++ b/tests/unit/neptune/new/cli/test_status.py @@ -57,6 +57,7 @@ def test_list_containers(tmp_path, mocker, capsys, backend, status_runner, conta "Unsynchronized objects:", f"- {get_qualified_name(unsynced_container)}", "", + # this one without formatting as it got splitted by new line "Please run with the `neptune sync --help` to see example commands.", ] @@ -78,7 +79,15 @@ def test_list_offline_runs(tmp_path, mocker, capsys, status_runner): # then captured = capsys.readouterr() assert captured.err == "" - assert "Unsynchronized offline objects:\n- offline/run__{}".format(offline_run.id) in captured.out + assert set(captured.out.splitlines()).issuperset( + set( + [ + "Unsynchronized offline objects:", + f"- offline/run__{offline_run.id}", + "", + ] + ) + ) def test_list_trashed_containers(tmp_path, mocker, capsys, backend, status_runner): @@ -104,6 +113,7 @@ def test_list_trashed_containers(tmp_path, mocker, capsys, backend, status_runne "Unsynchronized objects:", f"- {get_qualified_name(unsynced_container)} (Trashed)", "", + # this one without formatting as it got splitted by new line "Please run with the `neptune sync --help` to see example commands.", ] diff --git a/tests/unit/neptune/new/internal/operation_processors/test_operation_logger.py b/tests/unit/neptune/new/internal/operation_processors/test_operation_logger.py index 447ded59a..4ea6b1e68 100644 --- a/tests/unit/neptune/new/internal/operation_processors/test_operation_logger.py +++ b/tests/unit/neptune/new/internal/operation_processors/test_operation_logger.py @@ -42,7 +42,7 @@ def test_log_connection_interruption(self): def test_log_remaining_operations(self): self.logger.log_remaining_operations(10) - self.logger._logger.warning.assert_called_once_with(WAITING_FOR_OPERATIONS_MSG, 10) + self.logger._logger.info.assert_called_once_with(WAITING_FOR_OPERATIONS_MSG, 10) self.logger._logger.warning.reset_mock() diff --git a/tests/unit/neptune/new/internal/utils/test_images.py b/tests/unit/neptune/new/internal/utils/test_images.py index beb032c96..c75656630 100644 --- a/tests/unit/neptune/new/internal/utils/test_images.py +++ b/tests/unit/neptune/new/internal/utils/test_images.py @@ -18,6 +18,7 @@ import os import sys import unittest +from functools import partial from typing import Optional from uuid import uuid4 @@ -41,6 +42,7 @@ get_html_content, get_image_content, ) +from tests.unit.neptune.new.utils.logging import format_log matplotlib.use("agg") @@ -76,14 +78,19 @@ def test_get_image_content_from_3d_grayscale_array(self): expected_array = numpy.array([[1, 0], [-3, 4], [5, 6]]) * 255 expected_image = Image.fromarray(expected_array.astype(numpy.uint8)) + # when + _log = partial(format_log, "WARNING") + # expect stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self.assertEqual(get_image_content(image_array), self._encode_pil_image(expected_image)) self.assertEqual( stdout.getvalue(), - "The smallest value in the array is -3 and the largest value in the array is 6." - " To be interpreted as colors correctly values in the array need to be in the [0, 1] range.\n", + _log( + "The smallest value in the array is -3 and the largest value in the array is 6." + " To be interpreted as colors correctly values in the array need to be in the [0, 1] range.\n", + ), ) def test_get_image_content_from_rgb_array(self): diff --git a/tests/unit/neptune/new/test_logging.py b/tests/unit/neptune/new/test_logging.py new file mode 100644 index 000000000..d34a46f8d --- /dev/null +++ b/tests/unit/neptune/new/test_logging.py @@ -0,0 +1,40 @@ +from contextlib import contextmanager +from functools import partial + +import pytest + +from neptune.internal.utils.logger import get_logger +from tests.unit.neptune.new.utils.logging import format_log + + +@contextmanager +def assert_out(capsys: pytest.CaptureFixture, out_msg: str = "", err_msg: str = ""): + _ = capsys.readouterr() + yield + captured = capsys.readouterr() + assert out_msg == captured.out + assert err_msg == captured.err + + +def test_internal_logger_loglevels(capsys: pytest.CaptureFixture): + # given + logger = get_logger() + + # when + _log = partial(format_log, msg="message\n") + + # then + with assert_out(capsys, _log("DEBUG")): + logger.debug("message") + + with assert_out(capsys, _log("INFO")): + logger.info("message") + + with assert_out(capsys, _log("WARNING")): + logger.warning("message") + + with assert_out(capsys, _log("ERROR")): + logger.error("message") + + with assert_out(capsys, _log("CRITICAL")): + logger.critical("message") diff --git a/tests/unit/neptune/new/utils/logging.py b/tests/unit/neptune/new/utils/logging.py new file mode 100644 index 000000000..dad034d5e --- /dev/null +++ b/tests/unit/neptune/new/utils/logging.py @@ -0,0 +1,6 @@ +from neptune.internal.utils.logger import NEPTUNE_LOGGER_NAME + + +def format_log(loglevel: str, msg: str) -> str: + loglevel = loglevel.lower().ljust(len("warning")) + return f"[{NEPTUNE_LOGGER_NAME}] [{loglevel}] {msg}"