refactor: decoupling indexer and kmd algosdk dependencies; refining byte field handling in generated clients

.pre-commit-config.yaml

@@ -1,8 +1,8 @@
 repos:
   - repo: local
     hooks:
-      - id: ruff-format
-        name: ruff-format
+      - id: format
+        name: format
         description: "Run 'ruff format' for extremely fast Python formatting"
         entry: uv run ruff format
         language: system
@@ -11,30 +11,20 @@ repos:
         require_serial: true
         additional_dependencies: []
         minimum_pre_commit_version: "2.9.2"
+        pass_filenames: false
         files: "^(src|tests)/"
-      - id: ruff
-        name: ruff
-        description: "Run 'ruff' for extremely fast Python linting"
-        entry: uv run ruff check
-        language: system
-        "types": [python]
-        args: [--fix]
-        require_serial: false
-        additional_dependencies: []
-        minimum_pre_commit_version: "0"
-        files: "^(src|tests)/"
-        exclude: "^tests/artifacts/"
-      - id: mypy
-        name: mypy
-        description: "`mypy` will check Python types for correctness"
-        entry: uv run mypy
+      - id: lint
+        name: lint
+        description: "`mypy` and 'ruff' will check Python types for correctness"
+        entry: uv run poe lint
         language: system
         types_or: [python, pyi]
+        args: []
         require_serial: true
         additional_dependencies: []
         minimum_pre_commit_version: "2.9.2"
+        pass_filenames: false
         files: "^(src|tests)/"
-        exclude: "^tests/artifacts/"
       - id: docstrings-check
         name: docstrings-check
         description: "Check docstrings for correctness"

MIGRATION-NOTES.md

@@ -0,0 +1,13 @@
+# Migration Notes
+
+A collection of notes to consolidate todos during decoupling efforts (similar doc exists on ts version as well).
+
+## API 
+
+### Generator  
+
+- Currently generated models for KMD have explicit request models resulting in slightly different signatures in contrast with indexer and algod and requiring imports of explicit typed request models specifically on kmd client calls. Do we want to further refine the generation to auto flatten the keys to ensure it does not define an explicit request models or change those models to TypedDicts to reduce the import overhead?
+
+### KMD
+
+- algokit-core repo on a branch called feat/account-manager, we had a minor refinement in OAS spec for kmd adding a default value for wallet driver field as well as the generator adjustments to ensure generated model for related endpoint falls back to default 'sqlite' value. Do we want to restore this approach?:

api/oas-generator/src/oas_generator/builder.py

@@ -27,6 +27,7 @@ class TypeInfo:
     list_inner_model: str | None = None
     list_inner_enum: str | None = None
     is_bytes: bool = False
+    list_inner_is_bytes: bool = False
     is_signed_transaction: bool = False
     needs_datetime: bool = False
     imports: set[str] = field(default_factory=set)
@@ -174,6 +175,7 @@ def _resolve_array(self, schema: ctx.RawSchema, *, hint: str) -> TypeInfo:
             is_list=True,
             list_inner_model=inner.model,
             list_inner_enum=inner.enum,
+            list_inner_is_bytes=inner.is_bytes,
             is_signed_transaction=inner.is_signed_transaction,
             needs_datetime=inner.needs_datetime,
             imports=set(inner.imports),
@@ -283,6 +285,10 @@ def _build_model(self, entry: SchemaEntry) -> ctx.ModelDescriptor:  # noqa: C901
                 imports.add("from ._serde_helpers import encode_model_mapping, mapping_encoder")
             if "decode_model_mapping" in field.metadata or "mapping_decoder" in field.metadata:
                 imports.add("from ._serde_helpers import decode_model_mapping, mapping_decoder")
+            if "encode_bytes_base64" in field.metadata or "decode_bytes_base64" in field.metadata:
+                imports.add("from ._serde_helpers import decode_bytes_base64, encode_bytes_base64")
+            if "encode_bytes_sequence" in field.metadata or "decode_bytes_sequence" in field.metadata:
+                imports.add("from ._serde_helpers import decode_bytes_sequence, encode_bytes_sequence")
             if "nested(" in field.metadata:
                 uses_nested = True
             if "flatten(" in field.metadata:
@@ -391,6 +397,22 @@ def _build_metadata(self, wire_name: str, type_info: TypeInfo) -> str:
                 f"            decode=lambda raw: decode_enum_sequence(lambda: {type_info.list_inner_enum}, raw),\n"
                 "        )"
             )
+        if type_info.is_list and type_info.list_inner_is_bytes:
+            return (
+                "wire(\n"
+                f'            "{alias}",\n'
+                "            encode=encode_bytes_sequence,\n"
+                "            decode=decode_bytes_sequence,\n"
+                "        )"
+            )
+        if type_info.is_bytes:
+            return (
+                "wire(\n"
+                f'            "{alias}",\n'
+                "            encode=encode_bytes_base64,\n"
+                "            decode=decode_bytes_base64,\n"
+                "        )"
+            )
         if type_info.is_signed_transaction:
             return f'nested("{alias}", lambda: SignedTransaction)'
         return f'wire("{alias}")'

api/oas-generator/src/oas_generator/naming.py

@@ -7,6 +7,7 @@
 _LOWER_TO_UPPER = re.compile(r"([a-z0-9])([A-Z])")
 _PY_RESERVED = {*keyword.kwlist, *keyword.softkwlist, *dir(builtins), "self", "cls"}
 
+
 @dataclass(slots=True)
 class IdentifierSanitizer:
     """Deterministic naming helper shared across generator stages."""

api/oas-generator/src/oas_generator/renderer/templates/models/_serde_helpers.py.j2

@@ -1,14 +1,65 @@
-
+import base64
+from binascii import Error as BinasciiError
 
 from collections.abc import Iterable, Mapping
-from enum import Enum
 from dataclasses import is_dataclass
-from typing import Callable, TypeVar
+from enum import Enum
+from typing import Callable, TypeAlias, TypeVar
 
 from algokit_common.serde import from_wire, to_wire
 
 T = TypeVar("T")
 E = TypeVar("E", bound=Enum)
+BytesLike: TypeAlias = bytes | bytearray | memoryview
+
+
+def _coerce_bytes(value: bytes | bytearray | memoryview) -> bytes:
+    if isinstance(value, memoryview):
+        return value.tobytes()
+    if isinstance(value, bytearray):
+        return bytes(value)
+    return value
+
+
+def encode_bytes_base64(value: BytesLike) -> str:
+    return base64.b64encode(_coerce_bytes(value)).decode("ascii")
+
+
+def decode_bytes_base64(raw: object) -> bytes:
+    if isinstance(raw, bytes | bytearray | memoryview):
+        return bytes(raw)
+    if isinstance(raw, str):
+        try:
+            return base64.b64decode(raw.encode("ascii"), validate=True)
+        except (BinasciiError, UnicodeEncodeError) as exc:
+            raise ValueError("Invalid base64 payload") from exc
+    raise TypeError(f"Unsupported value for bytes field: {type(raw)!r}")
+
+
+def encode_bytes_sequence(values: Iterable[BytesLike | None] | None) -> list[str | None] | None:
+    if values is None:
+        return None
+    encoded: list[str | None] = []
+    for value in values:
+        if value is None:
+            encoded.append(None)
+            continue
+        if not isinstance(value, bytes | bytearray | memoryview):
+            raise TypeError(f"Unsupported value for bytes field sequence: {type(value)!r}")
+        encoded.append(encode_bytes_base64(value))
+    return encoded or None
+
+
+def decode_bytes_sequence(raw: object) -> list[bytes | None] | None:
+    if not isinstance(raw, list):
+        return None
+    decoded: list[bytes | None] = []
+    for item in raw:
+        if item is None:
+            decoded.append(None)
+            continue
+        decoded.append(decode_bytes_base64(item))
+    return decoded or None
 
 
 def encode_model_sequence(values: Iterable[object] | None) -> list[dict[str, object]] | None:

docs/markdown/autoapi/algokit_utils/accounts/kmd_account_manager/index.md

@@ -24,7 +24,7 @@ Provides an account implementation that can be used to sign transactions using k
 
 Provides abstractions over KMD that makes it easier to get and manage accounts.
 
-#### kmd() → algosdk.kmd.KMDClient
+#### kmd() → algokit_kmd_client.client.KmdClient
 
 Returns the KMD client, initializing it if needed.
 
@@ -46,6 +46,8 @@ if no predicate is provided.
   * **sender** – Optional sender address to use this signer for (aka a rekeyed account)
 * **Returns:**
   The signing account or None if no matching wallet or account was found
+* **Raises:**
+  **Exception** – If error received while exporting the private key from KMD
 
 #### get_or_create_wallet_account(name: str, fund_with: [algokit_utils.models.amount.AlgoAmount](../../models/amount/index.md#algokit_utils.models.amount.AlgoAmount) | None = None) → [KmdAccount](#algokit_utils.accounts.kmd_account_manager.KmdAccount)
 
@@ -58,6 +60,8 @@ Provides idempotent access to accounts from LocalNet without specifying the priv
   * **fund_with** – The number of Algos to fund the account with when created
 * **Returns:**
   An Algorand account with private key loaded
+* **Raises:**
+  **Exception** – If error received while creating the wallet or funding the account
 
 #### get_localnet_dispenser_account() → [KmdAccount](#algokit_utils.accounts.kmd_account_manager.KmdAccount)
 

docs/markdown/autoapi/algokit_utils/algorand/index.md

@@ -242,7 +242,7 @@ Returns an AlgorandClient pointing at MainNet using AlgoNode.
   algorand = AlgorandClient.mainnet()
   ```
 
-#### *static* from_clients(algod: algosdk.v2client.algod.AlgodClient, indexer: algosdk.v2client.indexer.IndexerClient | None = None, kmd: algosdk.kmd.KMDClient | None = None) → [AlgorandClient](#algokit_utils.algorand.AlgorandClient)
+#### *static* from_clients(algod: algosdk.v2client.algod.AlgodClient, indexer: algokit_indexer_client.IndexerClient | None = None, kmd: algokit_kmd_client.client.KmdClient | None = None) → [AlgorandClient](#algokit_utils.algorand.AlgorandClient)
 
 Returns an AlgorandClient pointing to the given client(s).
 

docs/markdown/autoapi/algokit_utils/applications/app_deployer/index.md

@@ -151,7 +151,7 @@ The update result
 
 The delete result
 
-### *class* algokit_utils.applications.app_deployer.AppDeployer(app_manager: [algokit_utils.applications.app_manager.AppManager](../app_manager/index.md#algokit_utils.applications.app_manager.AppManager), transaction_sender: [algokit_utils.transactions.transaction_sender.AlgorandClientTransactionSender](../../transactions/transaction_sender/index.md#algokit_utils.transactions.transaction_sender.AlgorandClientTransactionSender), indexer: algosdk.v2client.indexer.IndexerClient | None = None)
+### *class* algokit_utils.applications.app_deployer.AppDeployer(app_manager: [algokit_utils.applications.app_manager.AppManager](../app_manager/index.md#algokit_utils.applications.app_manager.AppManager), transaction_sender: [algokit_utils.transactions.transaction_sender.AlgorandClientTransactionSender](../../transactions/transaction_sender/index.md#algokit_utils.transactions.transaction_sender.AlgorandClientTransactionSender), indexer: algokit_indexer_client.IndexerClient | None = None)
 
 Manages deployment and deployment metadata of applications
 

docs/markdown/autoapi/algokit_utils/clients/client_manager/index.md

@@ -9,7 +9,7 @@
 
 ## Module Contents
 
-### *class* algokit_utils.clients.client_manager.AlgoSdkClients(algod: algosdk.v2client.algod.AlgodClient, indexer: algosdk.v2client.indexer.IndexerClient | None = None, kmd: algosdk.kmd.KMDClient | None = None)
+### *class* algokit_utils.clients.client_manager.AlgoSdkClients(algod: algosdk.v2client.algod.AlgodClient, indexer: algokit_indexer_client.IndexerClient | None = None, kmd: algokit_kmd_client.KmdClient | None = None)
 
 Container for Algorand SDK client instances.
 
@@ -81,25 +81,25 @@ Returns an algosdk Algod API client.
 * **Returns:**
   Algod client instance
 
-#### *property* indexer *: algosdk.v2client.indexer.IndexerClient*
+#### *property* indexer *: algokit_indexer_client.IndexerClient*
 
-Returns an algosdk Indexer API client.
+Returns an Indexer API client.
 
 * **Raises:**
   **ValueError** – If no Indexer client is configured
 * **Returns:**
   Indexer client instance
 
-#### *property* indexer_if_present *: algosdk.v2client.indexer.IndexerClient | None*
+#### *property* indexer_if_present *: algokit_indexer_client.IndexerClient | None*
 
 Returns the Indexer client if configured, otherwise None.
 
 * **Returns:**
   Indexer client instance or None
 
-#### *property* kmd *: algosdk.kmd.KMDClient*
+#### *property* kmd *: algokit_kmd_client.KmdClient*
 
-Returns an algosdk KMD API client.
+Returns a KMD-compatible API client.
 
 * **Raises:**
   **ValueError** – If no KMD client is configured
@@ -231,7 +231,7 @@ Get an Algod client from environment variables.
 * **Returns:**
   Algod client instance
 
-#### *static* get_kmd_client(config: [algokit_utils.models.network.AlgoClientNetworkConfig](../../models/network/index.md#algokit_utils.models.network.AlgoClientNetworkConfig)) → algosdk.kmd.KMDClient
+#### *static* get_kmd_client(config: [algokit_utils.models.network.AlgoClientNetworkConfig](../../models/network/index.md#algokit_utils.models.network.AlgoClientNetworkConfig)) → algokit_kmd_client.KmdClient
 
 Get a KMD client from config or environment.
 
@@ -240,14 +240,14 @@ Get a KMD client from config or environment.
 * **Returns:**
   KMD client instance
 
-#### *static* get_kmd_client_from_environment() → algosdk.kmd.KMDClient
+#### *static* get_kmd_client_from_environment() → algokit_kmd_client.KmdClient
 
 Get a KMD client from environment variables.
 
 * **Returns:**
   KMD client instance
 
-#### *static* get_indexer_client(config: [algokit_utils.models.network.AlgoClientNetworkConfig](../../models/network/index.md#algokit_utils.models.network.AlgoClientNetworkConfig)) → algosdk.v2client.indexer.IndexerClient
+#### *static* get_indexer_client(config: [algokit_utils.models.network.AlgoClientNetworkConfig](../../models/network/index.md#algokit_utils.models.network.AlgoClientNetworkConfig)) → algokit_indexer_client.IndexerClient
 
 Get an Indexer client from config or environment.
 
@@ -256,7 +256,7 @@ Get an Indexer client from config or environment.
 * **Returns:**
   Indexer client instance
 
-#### *static* get_indexer_client_from_environment() → algosdk.v2client.indexer.IndexerClient
+#### *static* get_indexer_client_from_environment() → algokit_indexer_client.IndexerClient
 
 Get an Indexer client from environment variables.
 

src/algokit_algod_client/models/_account_participation.py

@@ -5,6 +5,8 @@
 
 from algokit_common.serde import wire
 
+from ._serde_helpers import decode_bytes_base64, encode_bytes_base64
+
 
 @dataclass(slots=True)
 class AccountParticipation:
@@ -14,7 +16,11 @@ class AccountParticipation:
     """
 
     selection_participation_key: bytes = field(
-        metadata=wire("selection-participation-key"),
+        metadata=wire(
+            "selection-participation-key",
+            encode=encode_bytes_base64,
+            decode=decode_bytes_base64,
+        ),
     )
     vote_first_valid: int = field(
         metadata=wire("vote-first-valid"),
@@ -26,9 +32,17 @@ class AccountParticipation:
         metadata=wire("vote-last-valid"),
     )
     vote_participation_key: bytes = field(
-        metadata=wire("vote-participation-key"),
+        metadata=wire(
+            "vote-participation-key",
+            encode=encode_bytes_base64,
+            decode=decode_bytes_base64,
+        ),
     )
     state_proof_key: bytes | None = field(
         default=None,
-        metadata=wire("state-proof-key"),
+        metadata=wire(
+            "state-proof-key",
+            encode=encode_bytes_base64,
+            decode=decode_bytes_base64,
+        ),
     )

src/algokit_algod_client/models/_app_call_logs.py

@@ -5,6 +5,8 @@
 
 from algokit_common.serde import wire
 
+from ._serde_helpers import decode_bytes_sequence, encode_bytes_sequence
+
 
 @dataclass(slots=True)
 class AppCallLogs:
@@ -17,7 +19,11 @@ class AppCallLogs:
         metadata=wire("app_id"),
     )
     logs: list[bytes] = field(
-        metadata=wire("logs"),
+        metadata=wire(
+            "logs",
+            encode=encode_bytes_sequence,
+            decode=decode_bytes_sequence,
+        ),
     )
     tx_id: str = field(
         metadata=wire("txId"),