diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd6085e..72f90a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Create a new subpackage for Scrapy pipelines - Remove some noqas thanks to the new Ruff release +- Replace relative imports with absolute imports - Replace asserts with custom checks in Scrapy subpackage ### Fixed diff --git a/pyproject.toml b/pyproject.toml index fe7126bd..a41631a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,8 @@ requires-python = ">=3.8" # compatibility with a wide range of external packages. This decision was discussed in detail in the following PR: # https://github.com/apify/apify-sdk-python/pull/154 dependencies = [ - "apify-client ~= 1.6.0", - "apify-shared ~= 1.1.0", + "apify-client ~= 1.6.2", + "apify-shared ~= 1.1.1", "aiofiles >= 22.1.0", "aioshutil >= 1.0", "colorama >= 0.4.6", @@ -111,7 +111,6 @@ ignore = [ "S303", # Use of insecure MD2, MD4, MD5, or SHA1 hash function "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes "TD002", # Missing author in TODO; try: `# TODO(): ...` or `# TODO @: ... - "TID252", # Relative imports from parent modules are bannedRuff "TRY003", # Avoid specifying long messages outside the exception class # TODO: Remove this once the following issue is fixed @@ -139,6 +138,7 @@ indent-style = "space" "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable "S101", # Use of assert detected "T20", # flake8-print + "TID252", # Relative imports from parent modules are banned "TRY301", # Abstract `raise` to an inner function ] @@ -147,7 +147,7 @@ docstring-quotes = "double" inline-quotes = "single" [tool.ruff.lint.isort] -known-first-party = ["apify", "apify_client", "apify_shared"] +known-local-folder = ["apify"] [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/src/apify/_crypto.py b/src/apify/_crypto.py index 5108a912..237bc8cd 100644 --- a/src/apify/_crypto.py +++ b/src/apify/_crypto.py @@ -4,14 +4,13 @@ import secrets from typing import Any +from apify_shared.utils import ignore_docs from cryptography.exceptions import InvalidTag as InvalidTagException from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import padding, rsa from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from apify_shared.utils import ignore_docs - -from .consts import ENCRYPTED_INPUT_VALUE_REGEXP +from apify.consts import ENCRYPTED_INPUT_VALUE_REGEXP ENCRYPTION_KEY_LENGTH = 32 ENCRYPTION_IV_LENGTH = 16 diff --git a/src/apify/_memory_storage/file_storage_utils.py b/src/apify/_memory_storage/file_storage_utils.py index c4591ba8..64645001 100644 --- a/src/apify/_memory_storage/file_storage_utils.py +++ b/src/apify/_memory_storage/file_storage_utils.py @@ -4,10 +4,9 @@ import aiofiles from aiofiles.os import makedirs - from apify_shared.utils import json_dumps -from .._utils import force_remove +from apify._utils import force_remove async def update_metadata(*, data: dict, entity_directory: str, write_metadata: bool) -> None: diff --git a/src/apify/_memory_storage/memory_storage_client.py b/src/apify/_memory_storage/memory_storage_client.py index 3c42cdad..ed55cc46 100644 --- a/src/apify/_memory_storage/memory_storage_client.py +++ b/src/apify/_memory_storage/memory_storage_client.py @@ -8,17 +8,16 @@ import aioshutil from aiofiles import ospath from aiofiles.os import rename, scandir - from apify_shared.consts import ApifyEnvVars from apify_shared.utils import ignore_docs -from .._utils import maybe_parse_bool -from .resource_clients.dataset import DatasetClient -from .resource_clients.dataset_collection import DatasetCollectionClient -from .resource_clients.key_value_store import KeyValueStoreClient -from .resource_clients.key_value_store_collection import KeyValueStoreCollectionClient -from .resource_clients.request_queue import RequestQueueClient -from .resource_clients.request_queue_collection import RequestQueueCollectionClient +from apify._memory_storage.resource_clients.dataset import DatasetClient +from apify._memory_storage.resource_clients.dataset_collection import DatasetCollectionClient +from apify._memory_storage.resource_clients.key_value_store import KeyValueStoreClient +from apify._memory_storage.resource_clients.key_value_store_collection import KeyValueStoreCollectionClient +from apify._memory_storage.resource_clients.request_queue import RequestQueueClient +from apify._memory_storage.resource_clients.request_queue_collection import RequestQueueCollectionClient +from apify._utils import maybe_parse_bool """ Memory storage emulates data storages that are available on the Apify platform. diff --git a/src/apify/_memory_storage/resource_clients/base_resource_client.py b/src/apify/_memory_storage/resource_clients/base_resource_client.py index 0c02c2dd..73dcf052 100644 --- a/src/apify/_memory_storage/resource_clients/base_resource_client.py +++ b/src/apify/_memory_storage/resource_clients/base_resource_client.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from typing_extensions import Self - from ..memory_storage_client import MemoryStorageClient + from apify._memory_storage.memory_storage_client import MemoryStorageClient @ignore_docs diff --git a/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py b/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py index 4d1f8998..2f41876a 100644 --- a/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py +++ b/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py @@ -7,11 +7,11 @@ from apify_shared.models import ListPage from apify_shared.utils import ignore_docs -from ..file_storage_utils import update_metadata -from .base_resource_client import BaseResourceClient +from apify._memory_storage.file_storage_utils import update_metadata +from apify._memory_storage.resource_clients.base_resource_client import BaseResourceClient if TYPE_CHECKING: - from ..memory_storage_client import MemoryStorageClient + from apify._memory_storage.memory_storage_client import MemoryStorageClient ResourceClientType = TypeVar('ResourceClientType', bound=BaseResourceClient, contravariant=True) # noqa: PLC0105 diff --git a/src/apify/_memory_storage/resource_clients/dataset.py b/src/apify/_memory_storage/resource_clients/dataset.py index 12d3f2a7..f8c82655 100644 --- a/src/apify/_memory_storage/resource_clients/dataset.py +++ b/src/apify/_memory_storage/resource_clients/dataset.py @@ -7,20 +7,19 @@ from typing import TYPE_CHECKING, Any, AsyncIterator import aioshutil - from apify_shared.models import ListPage from apify_shared.utils import ignore_docs -from ..._crypto import crypto_random_object_id -from ..._utils import force_rename, raise_on_duplicate_storage, raise_on_non_existing_storage -from ...consts import StorageTypes -from ..file_storage_utils import _update_dataset_items, update_metadata -from .base_resource_client import BaseResourceClient +from apify._crypto import crypto_random_object_id +from apify._memory_storage.file_storage_utils import _update_dataset_items, update_metadata +from apify._memory_storage.resource_clients.base_resource_client import BaseResourceClient +from apify._utils import force_rename, raise_on_duplicate_storage, raise_on_non_existing_storage +from apify.consts import StorageTypes if TYPE_CHECKING: from apify_shared.types import JSONSerializable - from ..memory_storage_client import MemoryStorageClient + from apify._memory_storage.memory_storage_client import MemoryStorageClient # This is what API returns in the x-apify-pagination-limit # header when no limit query parameter is used. diff --git a/src/apify/_memory_storage/resource_clients/dataset_collection.py b/src/apify/_memory_storage/resource_clients/dataset_collection.py index 195ad793..0ef7b3f0 100644 --- a/src/apify/_memory_storage/resource_clients/dataset_collection.py +++ b/src/apify/_memory_storage/resource_clients/dataset_collection.py @@ -4,8 +4,8 @@ from apify_shared.utils import ignore_docs -from .base_resource_collection_client import BaseResourceCollectionClient -from .dataset import DatasetClient +from apify._memory_storage.resource_clients.base_resource_collection_client import BaseResourceCollectionClient +from apify._memory_storage.resource_clients.dataset import DatasetClient if TYPE_CHECKING: from apify_shared.models import ListPage diff --git a/src/apify/_memory_storage/resource_clients/key_value_store.py b/src/apify/_memory_storage/resource_clients/key_value_store.py index 9f5e34bd..2920089d 100644 --- a/src/apify/_memory_storage/resource_clients/key_value_store.py +++ b/src/apify/_memory_storage/resource_clients/key_value_store.py @@ -13,11 +13,12 @@ import aiofiles import aioshutil from aiofiles.os import makedirs - from apify_shared.utils import ignore_docs, is_file_or_bytes, json_dumps -from ..._crypto import crypto_random_object_id -from ..._utils import ( +from apify._crypto import crypto_random_object_id +from apify._memory_storage.file_storage_utils import update_metadata +from apify._memory_storage.resource_clients.base_resource_client import BaseResourceClient +from apify._utils import ( force_remove, force_rename, guess_file_extension, @@ -25,15 +26,13 @@ raise_on_duplicate_storage, raise_on_non_existing_storage, ) -from ...consts import DEFAULT_API_PARAM_LIMIT, StorageTypes -from ...log import logger -from ..file_storage_utils import update_metadata -from .base_resource_client import BaseResourceClient +from apify.consts import DEFAULT_API_PARAM_LIMIT, StorageTypes +from apify.log import logger if TYPE_CHECKING: from typing_extensions import NotRequired - from ..memory_storage_client import MemoryStorageClient + from apify._memory_storage.memory_storage_client import MemoryStorageClient class KeyValueStoreRecord(TypedDict): diff --git a/src/apify/_memory_storage/resource_clients/key_value_store_collection.py b/src/apify/_memory_storage/resource_clients/key_value_store_collection.py index 4e063f19..9acb156e 100644 --- a/src/apify/_memory_storage/resource_clients/key_value_store_collection.py +++ b/src/apify/_memory_storage/resource_clients/key_value_store_collection.py @@ -4,8 +4,8 @@ from apify_shared.utils import ignore_docs -from .base_resource_collection_client import BaseResourceCollectionClient -from .key_value_store import KeyValueStoreClient +from apify._memory_storage.resource_clients.base_resource_collection_client import BaseResourceCollectionClient +from apify._memory_storage.resource_clients.key_value_store import KeyValueStoreClient if TYPE_CHECKING: from apify_shared.models import ListPage diff --git a/src/apify/_memory_storage/resource_clients/request_queue.py b/src/apify/_memory_storage/resource_clients/request_queue.py index d9217e89..1798c586 100644 --- a/src/apify/_memory_storage/resource_clients/request_queue.py +++ b/src/apify/_memory_storage/resource_clients/request_queue.py @@ -8,18 +8,17 @@ from typing import TYPE_CHECKING import aioshutil -from sortedcollections import ValueSortedDict - from apify_shared.utils import filter_out_none_values_recursively, ignore_docs, json_dumps +from sortedcollections import ValueSortedDict -from ..._crypto import crypto_random_object_id -from ..._utils import force_rename, raise_on_duplicate_storage, raise_on_non_existing_storage, unique_key_to_request_id -from ...consts import StorageTypes -from ..file_storage_utils import delete_request, update_metadata, update_request_queue_item -from .base_resource_client import BaseResourceClient +from apify._crypto import crypto_random_object_id +from apify._memory_storage.file_storage_utils import delete_request, update_metadata, update_request_queue_item +from apify._memory_storage.resource_clients.base_resource_client import BaseResourceClient +from apify._utils import force_rename, raise_on_duplicate_storage, raise_on_non_existing_storage, unique_key_to_request_id +from apify.consts import StorageTypes if TYPE_CHECKING: - from ..memory_storage_client import MemoryStorageClient + from apify._memory_storage.memory_storage_client import MemoryStorageClient @ignore_docs diff --git a/src/apify/_memory_storage/resource_clients/request_queue_collection.py b/src/apify/_memory_storage/resource_clients/request_queue_collection.py index 064d1f8b..dd69c918 100644 --- a/src/apify/_memory_storage/resource_clients/request_queue_collection.py +++ b/src/apify/_memory_storage/resource_clients/request_queue_collection.py @@ -4,8 +4,8 @@ from apify_shared.utils import ignore_docs -from .base_resource_collection_client import BaseResourceCollectionClient -from .request_queue import RequestQueueClient +from apify._memory_storage.resource_clients.base_resource_collection_client import BaseResourceCollectionClient +from apify._memory_storage.resource_clients.request_queue import RequestQueueClient if TYPE_CHECKING: from apify_shared.models import ListPage diff --git a/src/apify/_utils.py b/src/apify/_utils.py index f1d3e595..d31c62ba 100644 --- a/src/apify/_utils.py +++ b/src/apify/_utils.py @@ -35,7 +35,6 @@ import psutil from aiofiles import ospath from aiofiles.os import remove, rename - from apify_shared.consts import ( BOOL_ENV_VARS, BOOL_ENV_VARS_TYPE, @@ -57,7 +56,7 @@ maybe_extract_enum_member_value, ) -from .consts import REQUEST_ID_LENGTH, StorageTypes +from apify.consts import REQUEST_ID_LENGTH, StorageTypes T = TypeVar('T') diff --git a/src/apify/actor.py b/src/apify/actor.py index a952c688..a7c98eea 100644 --- a/src/apify/actor.py +++ b/src/apify/actor.py @@ -12,8 +12,8 @@ from apify_shared.consts import ActorEnvVars, ActorEventTypes, ActorExitCodes, ApifyEnvVars, WebhookEventType from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value -from ._crypto import decrypt_input_secrets, load_private_key -from ._utils import ( +from apify._crypto import decrypt_input_secrets, load_private_key +from apify._utils import ( dualproperty, fetch_and_parse_env_var, get_cpu_usage_percent, @@ -23,18 +23,18 @@ run_func_at_interval_async, wrap_internal, ) -from .config import Configuration -from .consts import EVENT_LISTENERS_TIMEOUT_SECS -from .event_manager import EventManager -from .log import logger -from .proxy_configuration import ProxyConfiguration -from .storages import Dataset, KeyValueStore, RequestQueue, StorageClientManager +from apify.config import Configuration +from apify.consts import EVENT_LISTENERS_TIMEOUT_SECS +from apify.event_manager import EventManager +from apify.log import logger +from apify.proxy_configuration import ProxyConfiguration +from apify.storages import Dataset, KeyValueStore, RequestQueue, StorageClientManager if TYPE_CHECKING: import logging from types import TracebackType - from ._memory_storage import MemoryStorageClient + from apify._memory_storage import MemoryStorageClient T = TypeVar('T') MainReturnType = TypeVar('MainReturnType') diff --git a/src/apify/config.py b/src/apify/config.py index 1eeb738a..86e82e74 100644 --- a/src/apify/config.py +++ b/src/apify/config.py @@ -2,7 +2,7 @@ from apify_shared.consts import ActorEnvVars, ApifyEnvVars -from ._utils import fetch_and_parse_env_var +from apify._utils import fetch_and_parse_env_var class Configuration: diff --git a/src/apify/event_manager.py b/src/apify/event_manager.py index e7c8ad7c..edb2595f 100644 --- a/src/apify/event_manager.py +++ b/src/apify/event_manager.py @@ -8,16 +8,15 @@ from typing import TYPE_CHECKING, Any, Callable, Coroutine, Union import websockets.client -from pyee.asyncio import AsyncIOEventEmitter - from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value, parse_date_fields +from pyee.asyncio import AsyncIOEventEmitter -from .log import logger +from apify.log import logger if TYPE_CHECKING: from apify_shared.consts import ActorEventTypes - from .config import Configuration + from apify.config import Configuration ListenerType = Union[Callable[[], None], Callable[[Any], None], Callable[[], Coroutine[Any, Any, None]], Callable[[Any], Coroutine[Any, Any, None]]] diff --git a/src/apify/log.py b/src/apify/log.py index 68d41037..8e9fcecf 100644 --- a/src/apify/log.py +++ b/src/apify/log.py @@ -6,9 +6,8 @@ import traceback from typing import Any -from colorama import Fore, Style, just_fix_windows_console - from apify_shared.utils import ignore_docs +from colorama import Fore, Style, just_fix_windows_console just_fix_windows_console() diff --git a/src/apify/proxy_configuration.py b/src/apify/proxy_configuration.py index 22e46447..0370959b 100644 --- a/src/apify/proxy_configuration.py +++ b/src/apify/proxy_configuration.py @@ -7,17 +7,15 @@ from urllib.parse import urljoin, urlparse import httpx - from apify_shared.consts import ApifyEnvVars from apify_shared.utils import ignore_docs -from .config import Configuration -from .log import logger +from apify.config import Configuration +from apify.log import logger if TYPE_CHECKING: - from typing_extensions import NotRequired - from apify_client import ApifyClientAsync + from typing_extensions import NotRequired APIFY_PROXY_VALUE_REGEX = re.compile(r'^[\w._~]+$') COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$') diff --git a/src/apify/scrapy/middlewares/apify_proxy.py b/src/apify/scrapy/middlewares/apify_proxy.py index b21bb206..c2aeca65 100644 --- a/src/apify/scrapy/middlewares/apify_proxy.py +++ b/src/apify/scrapy/middlewares/apify_proxy.py @@ -12,9 +12,9 @@ 'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".', ) from exc -from ...actor import Actor -from ...proxy_configuration import ProxyConfiguration -from ..utils import get_basic_auth_header +from apify.actor import Actor +from apify.proxy_configuration import ProxyConfiguration +from apify.scrapy.utils import get_basic_auth_header class ApifyHttpProxyMiddleware: diff --git a/src/apify/scrapy/middlewares/apify_retry.py b/src/apify/scrapy/middlewares/apify_retry.py index 1ec4da1e..ebedca87 100644 --- a/src/apify/scrapy/middlewares/apify_retry.py +++ b/src/apify/scrapy/middlewares/apify_retry.py @@ -13,11 +13,11 @@ 'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".', ) from exc -from ...actor import Actor -from ..utils import nested_event_loop, open_queue_with_custom_client, to_apify_request +from apify.actor import Actor +from apify.scrapy.utils import nested_event_loop, open_queue_with_custom_client, to_apify_request if TYPE_CHECKING: - from ...storages import RequestQueue + from apify.storages import RequestQueue class ApifyRetryMiddleware(RetryMiddleware): diff --git a/src/apify/scrapy/pipelines/actor_dataset_push.py b/src/apify/scrapy/pipelines/actor_dataset_push.py index 458b13b0..e75262da 100644 --- a/src/apify/scrapy/pipelines/actor_dataset_push.py +++ b/src/apify/scrapy/pipelines/actor_dataset_push.py @@ -9,7 +9,7 @@ 'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".', ) from exc -from ...actor import Actor +from apify.actor import Actor class ActorDatasetPushPipeline: diff --git a/src/apify/scrapy/scheduler.py b/src/apify/scrapy/scheduler.py index a91e9a63..1fe4d761 100644 --- a/src/apify/scrapy/scheduler.py +++ b/src/apify/scrapy/scheduler.py @@ -12,10 +12,10 @@ 'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".', ) from exc -from .._crypto import crypto_random_object_id -from ..actor import Actor -from ..storages import RequestQueue -from .utils import nested_event_loop, open_queue_with_custom_client, to_apify_request, to_scrapy_request +from apify._crypto import crypto_random_object_id +from apify.actor import Actor +from apify.scrapy.utils import nested_event_loop, open_queue_with_custom_client, to_apify_request, to_scrapy_request +from apify.storages import RequestQueue class ApifyScheduler(BaseScheduler): diff --git a/src/apify/scrapy/utils.py b/src/apify/scrapy/utils.py index 9a5be6d3..272288ef 100644 --- a/src/apify/scrapy/utils.py +++ b/src/apify/scrapy/utils.py @@ -15,9 +15,9 @@ 'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".', ) from exc -from .._crypto import crypto_random_object_id -from ..actor import Actor -from ..storages import RequestQueue, StorageClientManager +from apify._crypto import crypto_random_object_id +from apify.actor import Actor +from apify.storages import RequestQueue, StorageClientManager nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() diff --git a/src/apify/storages/base_storage.py b/src/apify/storages/base_storage.py index a7304d03..54697511 100644 --- a/src/apify/storages/base_storage.py +++ b/src/apify/storages/base_storage.py @@ -6,10 +6,10 @@ from apify_shared.utils import ignore_docs -from .._memory_storage import MemoryStorageClient -from .._memory_storage.resource_clients import BaseResourceClient, BaseResourceCollectionClient -from ..config import Configuration -from .storage_client_manager import StorageClientManager +from apify._memory_storage import MemoryStorageClient +from apify._memory_storage.resource_clients import BaseResourceClient, BaseResourceCollectionClient +from apify.config import Configuration +from apify.storages.storage_client_manager import StorageClientManager if TYPE_CHECKING: from apify_client import ApifyClientAsync diff --git a/src/apify/storages/dataset.py b/src/apify/storages/dataset.py index 79d14a8a..ce4429c7 100644 --- a/src/apify/storages/dataset.py +++ b/src/apify/storages/dataset.py @@ -7,10 +7,10 @@ from apify_shared.utils import ignore_docs, json_dumps -from .._utils import wrap_internal -from ..consts import MAX_PAYLOAD_SIZE_BYTES -from .base_storage import BaseStorage -from .key_value_store import KeyValueStore +from apify._utils import wrap_internal +from apify.consts import MAX_PAYLOAD_SIZE_BYTES +from apify.storages.base_storage import BaseStorage +from apify.storages.key_value_store import KeyValueStore if TYPE_CHECKING: from apify_client import ApifyClientAsync @@ -18,9 +18,9 @@ from apify_shared.models import ListPage from apify_shared.types import JSONSerializable - from .._memory_storage import MemoryStorageClient - from .._memory_storage.resource_clients import DatasetClient, DatasetCollectionClient - from ..config import Configuration + from apify._memory_storage import MemoryStorageClient + from apify._memory_storage.resource_clients import DatasetClient, DatasetCollectionClient + from apify.config import Configuration # 0.01% SAFETY_BUFFER_PERCENT = 0.01 / 100 diff --git a/src/apify/storages/key_value_store.py b/src/apify/storages/key_value_store.py index 2854ef5f..71d843ae 100644 --- a/src/apify/storages/key_value_store.py +++ b/src/apify/storages/key_value_store.py @@ -5,15 +5,15 @@ from apify_client.clients import KeyValueStoreClientAsync, KeyValueStoreCollectionClientAsync from apify_shared.utils import ignore_docs -from .._utils import wrap_internal -from .base_storage import BaseStorage +from apify._utils import wrap_internal +from apify.storages.base_storage import BaseStorage if TYPE_CHECKING: from apify_client import ApifyClientAsync - from .._memory_storage import MemoryStorageClient - from .._memory_storage.resource_clients import KeyValueStoreClient, KeyValueStoreCollectionClient - from ..config import Configuration + from apify._memory_storage import MemoryStorageClient + from apify._memory_storage.resource_clients import KeyValueStoreClient, KeyValueStoreCollectionClient + from apify.config import Configuration T = TypeVar('T') diff --git a/src/apify/storages/request_queue.py b/src/apify/storages/request_queue.py index 4c1f5822..cb0ccfb8 100644 --- a/src/apify/storages/request_queue.py +++ b/src/apify/storages/request_queue.py @@ -8,19 +8,19 @@ from apify_shared.utils import ignore_docs -from .._crypto import crypto_random_object_id -from .._utils import LRUCache, budget_ow, unique_key_to_request_id -from ..consts import REQUEST_QUEUE_HEAD_MAX_LIMIT -from ..log import logger -from .base_storage import BaseStorage +from apify._crypto import crypto_random_object_id +from apify._utils import LRUCache, budget_ow, unique_key_to_request_id +from apify.consts import REQUEST_QUEUE_HEAD_MAX_LIMIT +from apify.log import logger +from apify.storages.base_storage import BaseStorage if TYPE_CHECKING: from apify_client import ApifyClientAsync from apify_client.clients import RequestQueueClientAsync, RequestQueueCollectionClientAsync - from .._memory_storage import MemoryStorageClient - from .._memory_storage.resource_clients import RequestQueueClient, RequestQueueCollectionClient - from ..config import Configuration + from apify._memory_storage import MemoryStorageClient + from apify._memory_storage.resource_clients import RequestQueueClient, RequestQueueCollectionClient + from apify.config import Configuration MAX_CACHED_REQUESTS = 1_000_000 diff --git a/src/apify/storages/storage_client_manager.py b/src/apify/storages/storage_client_manager.py index bee2c781..52207248 100644 --- a/src/apify/storages/storage_client_manager.py +++ b/src/apify/storages/storage_client_manager.py @@ -4,8 +4,8 @@ from apify_shared.utils import ignore_docs -from .._memory_storage import MemoryStorageClient -from ..config import Configuration +from apify._memory_storage import MemoryStorageClient +from apify.config import Configuration if TYPE_CHECKING: from apify_client import ApifyClientAsync diff --git a/tests/integration/actor_source_base/src/__main__.py b/tests/integration/actor_source_base/src/__main__.py index 643eb63c..0d1d65af 100644 --- a/tests/integration/actor_source_base/src/__main__.py +++ b/tests/integration/actor_source_base/src/__main__.py @@ -3,9 +3,8 @@ import asyncio import logging -from apify.log import ActorLogFormatter - from .main import main +from apify.log import ActorLogFormatter handler = logging.StreamHandler() handler.setFormatter(ActorLogFormatter()) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 93c46710..c68d441a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -10,15 +10,14 @@ from typing import TYPE_CHECKING, AsyncIterator, Awaitable, Callable, Mapping, Protocol import pytest +from apify_client import ApifyClientAsync +from apify_shared.consts import ActorJobStatus, ActorSourceType from filelock import FileLock +from ._utils import generate_unique_resource_name from apify import Actor from apify.config import Configuration from apify.storages import Dataset, KeyValueStore, RequestQueue, StorageClientManager -from apify_client import ApifyClientAsync -from apify_shared.consts import ActorJobStatus, ActorSourceType - -from ._utils import generate_unique_resource_name if TYPE_CHECKING: from apify_client.clients.resource_clients import ActorClientAsync diff --git a/tests/integration/test_actor_api_helpers.py b/tests/integration/test_actor_api_helpers.py index 025d9bee..2d9e96e0 100644 --- a/tests/integration/test_actor_api_helpers.py +++ b/tests/integration/test_actor_api_helpers.py @@ -4,11 +4,10 @@ import json from typing import TYPE_CHECKING +from ._utils import generate_unique_resource_name from apify import Actor from apify._crypto import crypto_random_object_id -from ._utils import generate_unique_resource_name - if TYPE_CHECKING: from apify_client import ApifyClientAsync diff --git a/tests/integration/test_actor_dataset.py b/tests/integration/test_actor_dataset.py index 1486dbca..81a4f938 100644 --- a/tests/integration/test_actor_dataset.py +++ b/tests/integration/test_actor_dataset.py @@ -2,14 +2,13 @@ from typing import TYPE_CHECKING -from apify import Actor from apify_shared.consts import ApifyEnvVars from ._utils import generate_unique_resource_name +from apify import Actor if TYPE_CHECKING: import pytest - from apify_client import ApifyClientAsync from .conftest import ActorFactory diff --git a/tests/integration/test_actor_events.py b/tests/integration/test_actor_events.py index 9e63c908..b4436b7e 100644 --- a/tests/integration/test_actor_events.py +++ b/tests/integration/test_actor_events.py @@ -3,9 +3,10 @@ import asyncio from typing import TYPE_CHECKING -from apify import Actor from apify_shared.consts import ActorEventTypes +from apify import Actor + if TYPE_CHECKING: from .conftest import ActorFactory diff --git a/tests/integration/test_actor_key_value_store.py b/tests/integration/test_actor_key_value_store.py index e1edb958..3bd6df62 100644 --- a/tests/integration/test_actor_key_value_store.py +++ b/tests/integration/test_actor_key_value_store.py @@ -2,14 +2,13 @@ from typing import TYPE_CHECKING -from apify import Actor from apify_shared.consts import ApifyEnvVars from ._utils import generate_unique_resource_name +from apify import Actor if TYPE_CHECKING: import pytest - from apify_client import ApifyClientAsync from .conftest import ActorFactory diff --git a/tests/integration/test_actor_request_queue.py b/tests/integration/test_actor_request_queue.py index d559fc73..076115e3 100644 --- a/tests/integration/test_actor_request_queue.py +++ b/tests/integration/test_actor_request_queue.py @@ -2,14 +2,13 @@ from typing import TYPE_CHECKING -from apify import Actor from apify_shared.consts import ApifyEnvVars from ._utils import generate_unique_resource_name +from apify import Actor if TYPE_CHECKING: import pytest - from apify_client import ApifyClientAsync from .conftest import ActorFactory diff --git a/tests/unit/actor/test_actor_create_proxy_configuration.py b/tests/unit/actor/test_actor_create_proxy_configuration.py index 29c6e928..593ee080 100644 --- a/tests/unit/actor/test_actor_create_proxy_configuration.py +++ b/tests/unit/actor/test_actor_create_proxy_configuration.py @@ -4,11 +4,11 @@ import httpx import pytest - -from apify import Actor from apify_client import ApifyClientAsync from apify_shared.consts import ApifyEnvVars +from apify import Actor + if TYPE_CHECKING: from respx import MockRouter diff --git a/tests/unit/actor/test_actor_dataset.py b/tests/unit/actor/test_actor_dataset.py index beb294b5..d9ba9c66 100644 --- a/tests/unit/actor/test_actor_dataset.py +++ b/tests/unit/actor/test_actor_dataset.py @@ -3,9 +3,9 @@ from typing import TYPE_CHECKING import pytest +from apify_shared.consts import ActorEnvVars from apify import Actor -from apify_shared.consts import ActorEnvVars if TYPE_CHECKING: from apify._memory_storage import MemoryStorageClient diff --git a/tests/unit/actor/test_actor_env_helpers.py b/tests/unit/actor/test_actor_env_helpers.py index f95f7c94..a6e3d6fd 100644 --- a/tests/unit/actor/test_actor_env_helpers.py +++ b/tests/unit/actor/test_actor_env_helpers.py @@ -5,9 +5,10 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any -from apify import Actor from apify_shared.consts import BOOL_ENV_VARS, DATETIME_ENV_VARS, FLOAT_ENV_VARS, INTEGER_ENV_VARS, STRING_ENV_VARS, ActorEnvVars, ApifyEnvVars +from apify import Actor + if TYPE_CHECKING: import pytest diff --git a/tests/unit/actor/test_actor_helpers.py b/tests/unit/actor/test_actor_helpers.py index daaed48b..92d0716d 100644 --- a/tests/unit/actor/test_actor_helpers.py +++ b/tests/unit/actor/test_actor_helpers.py @@ -2,10 +2,11 @@ from typing import TYPE_CHECKING -from apify import Actor from apify_client import ApifyClientAsync from apify_shared.consts import ApifyEnvVars, WebhookEventType +from apify import Actor + if TYPE_CHECKING: import pytest diff --git a/tests/unit/actor/test_actor_key_value_store.py b/tests/unit/actor/test_actor_key_value_store.py index bb9453ae..3de07378 100644 --- a/tests/unit/actor/test_actor_key_value_store.py +++ b/tests/unit/actor/test_actor_key_value_store.py @@ -3,14 +3,13 @@ from typing import TYPE_CHECKING import pytest - -from apify import Actor -from apify._crypto import public_encrypt -from apify.consts import ENCRYPTED_INPUT_VALUE_PREFIX from apify_shared.consts import ApifyEnvVars from apify_shared.utils import json_dumps from ..test_crypto import PRIVATE_KEY_PASSWORD, PRIVATE_KEY_PEM_BASE64, PUBLIC_KEY +from apify import Actor +from apify._crypto import public_encrypt +from apify.consts import ENCRYPTED_INPUT_VALUE_PREFIX if TYPE_CHECKING: from apify._memory_storage import MemoryStorageClient diff --git a/tests/unit/actor/test_actor_lifecycle.py b/tests/unit/actor/test_actor_lifecycle.py index 0dfbf018..ef187f47 100644 --- a/tests/unit/actor/test_actor_lifecycle.py +++ b/tests/unit/actor/test_actor_lifecycle.py @@ -7,9 +7,9 @@ from unittest.mock import AsyncMock import pytest +from apify_shared.consts import ActorEventTypes, ApifyEnvVars from apify import Actor -from apify_shared.consts import ActorEventTypes, ApifyEnvVars class TestActorInit: diff --git a/tests/unit/actor/test_actor_log.py b/tests/unit/actor/test_actor_log.py index 38a7281c..3bfe153c 100644 --- a/tests/unit/actor/test_actor_log.py +++ b/tests/unit/actor/test_actor_log.py @@ -5,9 +5,10 @@ import sys from typing import TYPE_CHECKING +from apify_client import __version__ as apify_client_version + from apify import Actor, __version__ from apify.log import logger -from apify_client import __version__ as apify_client_version if TYPE_CHECKING: import pytest diff --git a/tests/unit/actor/test_actor_memory_storage_e2e.py b/tests/unit/actor/test_actor_memory_storage_e2e.py index 0721685d..dd1d541e 100644 --- a/tests/unit/actor/test_actor_memory_storage_e2e.py +++ b/tests/unit/actor/test_actor_memory_storage_e2e.py @@ -4,10 +4,10 @@ from typing import Callable import pytest +from apify_shared.consts import ApifyEnvVars from apify import Actor from apify.storages import StorageClientManager -from apify_shared.consts import ApifyEnvVars @pytest.mark.parametrize('purge_on_start', [True, False]) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index eac8ae8f..39e2cb17 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -7,13 +7,13 @@ from typing import TYPE_CHECKING, Any, Callable, get_type_hints import pytest +from apify_client.client import ApifyClientAsync +from apify_shared.consts import ApifyEnvVars from apify import Actor from apify._memory_storage import MemoryStorageClient from apify.config import Configuration from apify.storages import Dataset, KeyValueStore, RequestQueue, StorageClientManager -from apify_client.client import ApifyClientAsync -from apify_shared.consts import ApifyEnvVars if TYPE_CHECKING: from pathlib import Path diff --git a/tests/unit/memory_storage/resource_clients/test_key_value_store.py b/tests/unit/memory_storage/resource_clients/test_key_value_store.py index 209599e8..86c8e224 100644 --- a/tests/unit/memory_storage/resource_clients/test_key_value_store.py +++ b/tests/unit/memory_storage/resource_clients/test_key_value_store.py @@ -8,10 +8,10 @@ from typing import TYPE_CHECKING import pytest +from apify_shared.utils import json_dumps from apify._crypto import crypto_random_object_id from apify._utils import maybe_parse_body -from apify_shared.utils import json_dumps if TYPE_CHECKING: from pathlib import Path diff --git a/tests/unit/memory_storage/test_memory_storage.py b/tests/unit/memory_storage/test_memory_storage.py index 7a5f1c1c..3d32398e 100644 --- a/tests/unit/memory_storage/test_memory_storage.py +++ b/tests/unit/memory_storage/test_memory_storage.py @@ -4,9 +4,9 @@ from typing import TYPE_CHECKING import pytest +from apify_shared.consts import ApifyEnvVars from apify._memory_storage import MemoryStorageClient -from apify_shared.consts import ApifyEnvVars if TYPE_CHECKING: from pathlib import Path diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index c1f77d86..b7770e38 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -3,9 +3,10 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING -from apify.config import Configuration from apify_shared.consts import ActorEnvVars, ApifyEnvVars +from apify.config import Configuration + if TYPE_CHECKING: import pytest diff --git a/tests/unit/test_event_manager.py b/tests/unit/test_event_manager.py index 145c954c..539bd472 100644 --- a/tests/unit/test_event_manager.py +++ b/tests/unit/test_event_manager.py @@ -11,10 +11,10 @@ import pytest import websockets import websockets.server +from apify_shared.consts import ActorEnvVars, ActorEventTypes from apify.config import Configuration from apify.event_manager import EventManager -from apify_shared.consts import ActorEnvVars, ActorEventTypes class TestEventManagerLocal: diff --git a/tests/unit/test_proxy_configuration.py b/tests/unit/test_proxy_configuration.py index 1db2349a..03dc8b41 100644 --- a/tests/unit/test_proxy_configuration.py +++ b/tests/unit/test_proxy_configuration.py @@ -6,11 +6,11 @@ import httpx import pytest - -from apify.proxy_configuration import ProxyConfiguration, is_url from apify_client import ApifyClientAsync from apify_shared.consts import ApifyEnvVars +from apify.proxy_configuration import ProxyConfiguration, is_url + if TYPE_CHECKING: from respx import MockRouter diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ae09df13..ab46cac9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10,6 +10,7 @@ import pytest from aiofiles.os import mkdir +from apify_shared.consts import ActorEnvVars, ApifyEnvVars from apify._utils import ( budget_ow, @@ -28,7 +29,6 @@ unique_key_to_request_id, ) from apify.consts import StorageTypes -from apify_shared.consts import ActorEnvVars, ApifyEnvVars if TYPE_CHECKING: from pathlib import Path