Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 33 additions & 20 deletions src/apify/_charging.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
from dataclasses import dataclass
from datetime import UTC, datetime
from decimal import Decimal
from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypedDict
from typing import TYPE_CHECKING, Literal, Protocol, TypedDict

from pydantic import Field
from pydantic import ConfigDict
from pydantic.alias_generators import to_camel

import apify_client._models as _client_models
from apify_client._models import ActorChargeEvent as ClientActorChargeEvent
Expand Down Expand Up @@ -56,7 +57,9 @@
class ActorChargeEvent(ClientActorChargeEvent):
"""Definition of a single chargeable event in the pay-per-event pricing model."""

event_description: Annotated[str | None, Field(alias='eventDescription')] = None
model_config = ConfigDict(alias_generator=to_camel)

event_description: str | None = None
"""Human-readable description of the event.

Required in apify-client but omitted from the env var, so it is relaxed to optional.
Expand All @@ -67,75 +70,85 @@ class ActorChargeEvent(ClientActorChargeEvent):
class PricingPerEvent(ClientPricingPerEvent):
"""Pay-per-event pricing details - the chargeable events and their prices."""

actor_charge_events: Annotated[dict[str, ActorChargeEvent] | None, Field(alias='actorChargeEvents')] = None
model_config = ConfigDict(alias_generator=to_camel)

actor_charge_events: dict[str, ActorChargeEvent] | None = None
"""Mapping of event name to its charge definition."""


@docs_group('Charging')
class FreeActorPricingInfo(ClientFree):
"""Pricing info for an Actor offered free of charge."""

apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
model_config = ConfigDict(alias_generator=to_camel)

apify_margin_percentage: float | None = None
"""Apify's margin on the price, as a percentage."""

created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
created_at: datetime | None = None
"""Timestamp when this pricing info was created."""

started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
started_at: datetime | None = None
"""Timestamp when this pricing became effective."""


@docs_group('Charging')
class FlatPricePerMonthActorPricingInfo(ClientFlatPricePerMonth):
"""Pricing info for an Actor billed at a flat monthly price."""

apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
model_config = ConfigDict(alias_generator=to_camel)

apify_margin_percentage: float | None = None
"""Apify's margin on the price, as a percentage."""

created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
created_at: datetime | None = None
"""Timestamp when this pricing info was created."""

started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
started_at: datetime | None = None
"""Timestamp when this pricing became effective."""

trial_minutes: Annotated[int | None, Field(alias='trialMinutes')] = None
trial_minutes: int | None = None
"""Length of the free trial period, in minutes."""

price_per_unit_usd: Annotated[float | None, Field(alias='pricePerUnitUsd')] = None
price_per_unit_usd: float | None = None
"""Price per unit, in USD."""


@docs_group('Charging')
class PricePerDatasetItemActorPricingInfo(ClientPricePerDatasetItem):
"""Pricing info for an Actor billed per dataset item produced."""

apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
model_config = ConfigDict(alias_generator=to_camel)

apify_margin_percentage: float | None = None
"""Apify's margin on the price, as a percentage."""

created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
created_at: datetime | None = None
"""Timestamp when this pricing info was created."""

started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
started_at: datetime | None = None
"""Timestamp when this pricing became effective."""

unit_name: Annotated[str | None, Field(alias='unitName')] = None
unit_name: str | None = None
"""Name of the billed unit."""


@docs_group('Charging')
class PayPerEventActorPricingInfo(ClientPayPerEvent):
"""Pricing info for an Actor billed per charged event."""

apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
model_config = ConfigDict(alias_generator=to_camel)

apify_margin_percentage: float | None = None
"""Apify's margin on the price, as a percentage."""

created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
created_at: datetime | None = None
"""Timestamp when this pricing info was created."""

started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
started_at: datetime | None = None
"""Timestamp when this pricing became effective."""

pricing_per_event: Annotated[PricingPerEvent, Field(alias='pricingPerEvent')]
pricing_per_event: PricingPerEvent
"""The pay-per-event pricing details."""


Expand Down
37 changes: 28 additions & 9 deletions src/apify/events/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
from datetime import datetime
from typing import Annotated, Any, Literal

from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel

from crawlee.events._types import (
Event,
Expand All @@ -29,28 +30,30 @@
class SystemInfoEventData(BaseModel):
"""Resource usage metrics carried by a `systemInfo` event."""

mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

mem_avg_bytes: float
"""Average memory usage over the measured interval, in bytes."""

mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
mem_current_bytes: float
"""Current memory usage, in bytes."""

mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
mem_max_bytes: float
"""Peak memory usage observed so far, in bytes."""

cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
cpu_avg_usage: float
"""Average CPU usage over the measured interval, in percent."""

cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
cpu_max_usage: float
"""Peak CPU usage observed so far, in percent."""

cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
cpu_current_usage: float
"""Current CPU usage, in percent."""

is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
is_cpu_overloaded: bool
"""Whether the CPU is currently overloaded."""

created_at: Annotated[datetime, Field(alias='createdAt')]
created_at: datetime
"""Timestamp when the metrics were collected."""

def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
Expand All @@ -73,6 +76,8 @@ def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
class PersistStateEvent(BaseModel):
"""A `persistState` event instructing the Actor to persist its state."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal[Event.PERSIST_STATE]
"""The event name."""

Expand All @@ -84,6 +89,8 @@ class PersistStateEvent(BaseModel):
class SystemInfoEvent(BaseModel):
"""A `systemInfo` event carrying the Actor's resource usage metrics."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal[Event.SYSTEM_INFO]
"""The event name."""

Expand All @@ -95,6 +102,8 @@ class SystemInfoEvent(BaseModel):
class MigratingEvent(BaseModel):
"""A `migrating` event signalling the Actor is about to be migrated to another host."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal[Event.MIGRATING]
"""The event name."""

Expand All @@ -106,6 +115,8 @@ class MigratingEvent(BaseModel):
class AbortingEvent(BaseModel):
"""An `aborting` event signalling the Actor run is being aborted."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal[Event.ABORTING]
"""The event name."""

Expand All @@ -117,6 +128,8 @@ class AbortingEvent(BaseModel):
class ExitEvent(BaseModel):
"""An `exit` event signalling the Actor process is about to exit."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal[Event.EXIT]
"""The event name."""

Expand All @@ -128,6 +141,8 @@ class ExitEvent(BaseModel):
class EventWithoutData(BaseModel):
"""A framework-level event that carries no payload (e.g. browser and page lifecycle events)."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal[
Event.SESSION_RETIRED,
Event.BROWSER_LAUNCHED,
Expand All @@ -146,6 +161,8 @@ class EventWithoutData(BaseModel):
class DeprecatedEvent(BaseModel):
"""A deprecated event kept for backward compatibility (e.g. `cpuInfo`)."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: Literal['cpuInfo']
"""The event name."""

Expand All @@ -157,6 +174,8 @@ class DeprecatedEvent(BaseModel):
class UnknownEvent(BaseModel):
"""A fallback for any event whose name is not recognized by the SDK."""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

name: str
"""The event name."""

Expand Down
9 changes: 6 additions & 3 deletions src/apify/request_loaders/_apify_request_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from itertools import chain
from typing import Annotated, Any

from pydantic import BaseModel, Field, TypeAdapter
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
from pydantic.alias_generators import to_camel

from crawlee._types import HttpMethod
from crawlee.http_clients import HttpClient, ImpitHttpClient
Expand All @@ -20,14 +21,16 @@


class _RequestDetails(BaseModel):
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

method: HttpMethod = 'GET'
payload: str = ''
headers: Annotated[dict[str, str], Field(default_factory=dict)]
user_data: Annotated[dict[str, str], Field(default_factory=dict, alias='userData')]
user_data: Annotated[dict[str, str], Field(default_factory=dict)]


class _RequestsFromUrlInput(_RequestDetails):
requests_from_url: str = Field(alias='requestsFromUrl')
requests_from_url: str


class _SimpleUrlInput(_RequestDetails):
Expand Down
36 changes: 25 additions & 11 deletions src/apify/storage_clients/_apify/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Annotated

from pydantic import BaseModel, ConfigDict, Field
from pydantic import AliasChoices, BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_camel

from apify_client._models import RequestQueueStats
from crawlee.storage_clients.models import KeyValueStoreMetadata, RequestQueueMetadata
Expand All @@ -22,7 +23,9 @@ class ApifyKeyValueStoreMetadata(KeyValueStoreMetadata):
Includes additional Apify-specific fields.
"""

url_signing_secret_key: Annotated[str | None, Field(alias='urlSigningSecretKey', default=None)]
model_config = ConfigDict(alias_generator=to_camel)

url_signing_secret_key: str | None = None
"""The secret key used for signing URLs for secure access to key-value store records."""


Expand All @@ -34,24 +37,31 @@ class RequestQueueHead(BaseModel):
including metadata about the queue's state and lock information for the requests.
"""

model_config = ConfigDict(populate_by_name=True, extra='allow')
model_config = ConfigDict(populate_by_name=True, extra='allow', alias_generator=to_camel)

limit: Annotated[int | None, Field(alias='limit', default=None)]
limit: int | None = None
"""The maximum number of requests that were requested from the queue."""

had_multiple_clients: Annotated[bool, Field(alias='hadMultipleClients', default=False)]
had_multiple_clients: bool = False
"""Indicates whether the queue has been accessed by multiple clients (consumers)."""

queue_modified_at: Annotated[datetime, Field(alias='queueModifiedAt')]
queue_modified_at: datetime
"""The timestamp when the queue was last modified."""

lock_time: Annotated[timedelta | None, Field(alias='lockSecs', default=None)]
"""The duration for which the returned requests are locked and cannot be processed by other clients."""
lock_time: Annotated[
timedelta | None,
Field(validation_alias=AliasChoices('lockSecs', 'lockTime'), serialization_alias='lockSecs'),
] = None
"""The duration for which the returned requests are locked and cannot be processed by other clients.

The platform's API names this field `lockSecs`, so it is serialized under that alias instead of the
`lockTime` that `to_camel` would derive from the field name.
"""

queue_has_locked_requests: Annotated[bool | None, Field(alias='queueHasLockedRequests', default=False)]
queue_has_locked_requests: bool | None = False
"""Indicates whether the queue contains any locked requests."""

items: Annotated[list[Request], Field(alias='items', default_factory=list[Request])]
items: Annotated[list[Request], Field(default_factory=list[Request])]
"""The list of request objects retrieved from the beginning of the queue."""

@classmethod
Expand All @@ -77,6 +87,8 @@ class CachedRequest(BaseModel):
Only internal structure.
"""

model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

id: str
"""Id of the request."""

Expand All @@ -91,5 +103,7 @@ class CachedRequest(BaseModel):


class ApifyRequestQueueMetadata(RequestQueueMetadata):
stats: Annotated[RequestQueueStats, Field(alias='stats', default_factory=RequestQueueStats)]
model_config = ConfigDict(alias_generator=to_camel)

stats: Annotated[RequestQueueStats, Field(default_factory=RequestQueueStats)]
"""Additional statistics about the request queue."""
Loading