From 184fc0546e3fa3d7db8072e9661ef6931500cf7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 23 Sep 2024 11:18:19 +0200 Subject: [PATCH 1/2] docs: hotfix API reference docs --- src/apify/_crypto.py | 2 ++ src/apify/_platform_event_manager.py | 1 - src/apify/_proxy_configuration.py | 1 + src/apify/log.py | 1 + src/apify/scrapy/requests.py | 2 ++ src/apify/scrapy/utils.py | 2 ++ website/pydoc-markdown.yml | 1 + website/transformDocs.js | 37 +++++++++++++++++----------- 8 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/apify/_crypto.py b/src/apify/_crypto.py index c97549fc..d3257582 100644 --- a/src/apify/_crypto.py +++ b/src/apify/_crypto.py @@ -114,6 +114,7 @@ def private_decrypt( return decipher_bytes.decode('utf-8') +@ignore_docs def load_private_key(private_key_file_base64: str, private_key_password: str) -> rsa.RSAPrivateKey: private_key = serialization.load_pem_private_key( base64.b64decode(private_key_file_base64.encode('utf-8')), @@ -133,6 +134,7 @@ def _load_public_key(public_key_file_base64: str) -> rsa.RSAPublicKey: return public_key +@ignore_docs def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> Any: """Decrypt input secrets.""" if not isinstance(input_data, dict): diff --git a/src/apify/_platform_event_manager.py b/src/apify/_platform_event_manager.py index 88bab198..e1efb535 100644 --- a/src/apify/_platform_event_manager.py +++ b/src/apify/_platform_event_manager.py @@ -126,7 +126,6 @@ class UnknownEvent(BaseModel): ) -@ignore_docs class PlatformEventManager(EventManager): """A class for managing Actor events. diff --git a/src/apify/_proxy_configuration.py b/src/apify/_proxy_configuration.py index 3b3c49fb..214fb4c9 100644 --- a/src/apify/_proxy_configuration.py +++ b/src/apify/_proxy_configuration.py @@ -27,6 +27,7 @@ SESSION_ID_MAX_LENGTH = 50 +@ignore_docs def is_url(url: str) -> bool: """Check if the given string is a valid URL.""" try: diff --git a/src/apify/log.py b/src/apify/log.py index 778cedae..a976a8e7 100644 --- a/src/apify/log.py +++ b/src/apify/log.py @@ -15,6 +15,7 @@ logger = logging.getLogger(logger_name) +@ignore_docs class ActorLogFormatter(CrawleeLogFormatter): # noqa: D101 Inherited from parent class pass diff --git a/src/apify/scrapy/requests.py b/src/apify/scrapy/requests.py index 7287a69a..d6d37b24 100644 --- a/src/apify/scrapy/requests.py +++ b/src/apify/scrapy/requests.py @@ -28,6 +28,7 @@ def _is_request_produced_by_middleware(scrapy_request: Request) -> bool: return bool(scrapy_request.meta.get('redirect_times')) or bool(scrapy_request.meta.get('retry_times')) +@ignore_docs def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest | None: """Convert a Scrapy request to an Apify request. @@ -98,6 +99,7 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest return apify_request +@ignore_docs def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request: """Convert an Apify request to a Scrapy request. diff --git a/src/apify/scrapy/utils.py b/src/apify/scrapy/utils.py index 5afadb82..9f52e696 100644 --- a/src/apify/scrapy/utils.py +++ b/src/apify/scrapy/utils.py @@ -18,6 +18,7 @@ nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() +@ignore_docs def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes: """Generate a basic authentication header for the given username and password.""" string = f'{unquote(username)}:{unquote(password)}' @@ -25,6 +26,7 @@ def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'la return b'Basic ' + b64encode(user_pass) +@ignore_docs def get_running_event_loop_id() -> int: """Get the ID of the currently running event loop. diff --git a/website/pydoc-markdown.yml b/website/pydoc-markdown.yml index 9623e24e..200eebb4 100644 --- a/website/pydoc-markdown.yml +++ b/website/pydoc-markdown.yml @@ -4,6 +4,7 @@ loaders: processors: - type: filter skip_empty_modules: true + documented_only: false - type: crossref renderer: type: docusaurus diff --git a/website/transformDocs.js b/website/transformDocs.js index 267ce3b3..dc98e635 100644 --- a/website/transformDocs.js +++ b/website/transformDocs.js @@ -57,15 +57,9 @@ const TYPEDOC_KINDS = { } const GROUP_ORDER = [ - 'Main Classes', - 'Main Clients', - 'Resource Clients', - 'Async Resource Clients', - 'Helper Classes', - 'Errors', - 'Constructors', - 'Methods', - 'Properties', + 'Classes', + 'Data structures', + 'Scrapy Integration', 'Constants', 'Enumeration Members' ]; @@ -79,22 +73,23 @@ const groupSort = (g1, g2) => { function getGroupName(object) { const groupPredicates = { + 'Scrapy integration': (x) => ['ApifyScheduler', 'ActorDatasetPushPipeline', 'ApifyHttpProxyMiddleware', 'apply_apify_settings'].includes(x.name), + 'Data structures': (x) => ['BaseModel', 'TypedDict'].some(base => x?.bases?.includes(base)) || x?.decorations?.some(d => d.name === 'dataclass'), 'Errors': (x) => x.name.toLowerCase().includes('error'), - 'Main Classes': (x) => ['Actor', 'Dataset', 'KeyValueStore', 'RequestQueue'].includes(x.name), + 'Classes': (x) => x.kindString === 'Class', 'Main Clients': (x) => ['ApifyClient', 'ApifyClientAsync'].includes(x.name), 'Async Resource Clients': (x) => x.name.toLowerCase().includes('async'), 'Resource Clients': (x) => x.kindString === 'Class' && x.name.toLowerCase().includes('client'), - 'Helper Classes': (x) => x.kindString === 'Class', 'Methods': (x) => x.kindString === 'Method', 'Constructors': (x) => x.kindString === 'Constructor', 'Properties': (x) => x.kindString === 'Property', 'Constants': (x) => x.kindString === 'Enumeration', - 'Enumeration Members': (x) => x.kindString === 'Enumeration Member', + 'Enumeration members': (x) => x.kindString === 'Enumeration Member', }; const [group] = Object.entries(groupPredicates).find( ([_, predicate]) => predicate(object) - ); + ) ?? ['Other']; return group; } @@ -162,7 +157,8 @@ function extractArgsAndReturns(docstring) { // Objects with decorators named 'ignore_docs' or with empty docstrings will be ignored function isHidden(member) { - return member.decorations?.some(d => d.name === 'ignore_docs') || member.name === 'ignore_docs' || !member.docstring?.content; + return member.decorations?.some(d => d.name === 'ignore_docs') + || member.name === 'ignore_docs'; } // Each object in the Typedoc structure has an unique ID, @@ -207,6 +203,14 @@ function convertObject(obj, parent, module) { moduleName = moduleShortcuts[fullName].replace(`.${member.name}`, ''); } + if(member.name === 'Actor' || (member.name.endsWith('Client') && !member.name.endsWith('StorageClient')) || member.name === 'ListPage') { + continue; + } + + if (member.name === '_ActorType') { + member.name = 'Actor'; + } + // Create the Typedoc member object let typedocMember = { id: oid++, @@ -214,6 +218,7 @@ function convertObject(obj, parent, module) { module: moduleName, // This is an extension to the original Typedoc structure, to support showing where the member is exported from ...typedocKind, flags: {}, + bases: member.bases, comment: member.docstring ? { summary: [{ kind: 'text', @@ -231,6 +236,10 @@ function convertObject(obj, parent, module) { }], }; + if(!GROUP_ORDER.includes(getGroupName(typedocMember)) && parent.kindString === 'Project'){ + continue; + } + if(typedocMember.kindString === 'Method') { const { parameters, returns } = extractArgsAndReturns(member.docstring?.content ?? ''); From 8d58d7e4d38db46690ebee52b9ee3bc3c83e6b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 23 Sep 2024 11:26:18 +0200 Subject: [PATCH 2/2] chore: add missing imports --- src/apify/_platform_event_manager.py | 1 - src/apify/log.py | 1 + src/apify/scrapy/requests.py | 2 ++ src/apify/scrapy/utils.py | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/apify/_platform_event_manager.py b/src/apify/_platform_event_manager.py index e1efb535..4cb20b75 100644 --- a/src/apify/_platform_event_manager.py +++ b/src/apify/_platform_event_manager.py @@ -8,7 +8,6 @@ from pydantic import BaseModel, Discriminator, Field, TypeAdapter from typing_extensions import Self, Unpack, override -from apify_shared.utils import ignore_docs from crawlee.events._event_manager import EventManager, EventManagerOptions from crawlee.events._local_event_manager import LocalEventManager from crawlee.events._types import ( diff --git a/src/apify/log.py b/src/apify/log.py index a976a8e7..e1f29400 100644 --- a/src/apify/log.py +++ b/src/apify/log.py @@ -3,6 +3,7 @@ import logging from typing import TYPE_CHECKING +from apify_shared.utils import ignore_docs from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level if TYPE_CHECKING: diff --git a/src/apify/scrapy/requests.py b/src/apify/scrapy/requests.py index d6d37b24..dec279cf 100644 --- a/src/apify/scrapy/requests.py +++ b/src/apify/scrapy/requests.py @@ -4,6 +4,8 @@ import pickle from typing import Any, cast +from apify_shared.utils import ignore_docs + try: from scrapy import Request, Spider from scrapy.http.headers import Headers diff --git a/src/apify/scrapy/utils.py b/src/apify/scrapy/utils.py index 9f52e696..f22a60de 100644 --- a/src/apify/scrapy/utils.py +++ b/src/apify/scrapy/utils.py @@ -4,6 +4,8 @@ from base64 import b64encode from urllib.parse import unquote +from apify_shared.utils import ignore_docs + try: from scrapy.settings import Settings # noqa: TCH002 from scrapy.utils.project import get_project_settings