From b07e5ae9b4c68dd2a84fede899f302189019ba1a Mon Sep 17 00:00:00 2001 From: Max Wang Date: Mon, 10 Nov 2025 09:54:42 -0800 Subject: [PATCH 1/7] stash --- README.md | 23 +++--- examples/quickstart.py | 150 +++++++++++++++++++++++++++++++++++- src/dataverse_sdk/client.py | 57 +++++++++++--- src/dataverse_sdk/odata.py | 50 +++++++++++- 4 files changed, 254 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e3bbfe4..874cdb3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A Python package allowing developers to connect to Dataverse environments for DDL / DML operations. - Read (SQL) — Execute constrained read-only SQL via the Dataverse Web API `?sql=` parameter. Returns `list[dict]`. -- OData CRUD — Unified methods `create(logical_name, record|records)`, `update(logical_name, id|ids, patch|patches)`, `delete(logical_name, id|ids)` plus `get` with record id or filters. +- OData CRUD — Unified methods `create(logical_name, record|records)`, `update(logical_name, id|ids, patch|patches)`, `delete(logical_name, id|ids)` plus `get` with record id or filters and `delete_async` for better multi-record delete performance. - Bulk create — Pass a list of records to `create(...)` to invoke the bound `CreateMultiple` action; returns `list[str]` of GUIDs. If any payload omits `@odata.type` the SDK resolves and stamps it (cached). - Bulk update — Provide a list of IDs with a single patch (broadcast) or a list of per‑record patches to `update(...)`; internally uses the bound `UpdateMultiple` action; returns nothing. Each record must include the primary key attribute when sent to UpdateMultiple. - Retrieve multiple (paging) — Generator-based `get(...)` that yields pages, supports `$top` and Prefer: `odata.maxpagesize` (`page_size`). @@ -39,7 +39,9 @@ Auth: | `update` | `update(logical_name, list[id], patch)` | `None` | Broadcast; same patch applied to all IDs (UpdateMultiple). | | `update` | `update(logical_name, list[id], list[patch])` | `None` | 1:1 patches; lengths must match (UpdateMultiple). | | `delete` | `delete(logical_name, id)` | `None` | Delete one record. | -| `delete` | `delete(logical_name, list[id], use_bulk_delete=True)` | `Optional[str]` | Delete many with async BulkDelete or sequential single-record delete. | +| `delete` | `delete(logical_name, list[id])` | `None` | Sequential deletes (loops over single-record delete). | +| `delete_async` | `delete_async(logical_name, id)` | `str` | Async single-record delete. | +| `delete_async` | `delete_async(logical_name, list[id])` | `str` | Async multi-record delete. | | `query_sql` | `query_sql(sql)` | `list[dict]` | Constrained read-only SELECT via `?sql=`. | | `create_table` | `create_table(tablename, schema, solution_unique_name=None)` | `dict` | Creates custom table + columns. Friendly name (e.g. `SampleItem`) becomes schema `new_SampleItem`; explicit schema name (contains `_`) used as-is. Pass `solution_unique_name` to attach the table to a specific solution instead of the default solution. | | `create_column` | `create_column(tablename, columns)` | `list[str]` | Adds columns using a `{name: type}` mapping (same shape as `create_table` schema). Returns schema names for the created columns. | @@ -54,10 +56,10 @@ Auth: Guidelines: - `create` always returns a list of GUIDs (1 for single, N for bulk). -- `update` always returns `None`. +- `update` and `delete` always returns `None`. - Bulk update chooses broadcast vs per-record by the type of `changes` (dict vs list). -- `delete` returns `None` for single-record delete and sequential multi-record delete, and the BulkDelete async job ID for multi-record BulkDelete. -- BulkDelete doesn't wait for the delete job to complete. It returns once the async delete job is scheduled. +- `delete_async` returns the BulkDelete async job ID and doesn't wait for completion. +- It's recommended to use delete_async for multi-record delete for better performance. - Paging and SQL operations never mutate inputs. - Metadata lookups for logical name stamping cached per entity set (in-memory). @@ -143,8 +145,11 @@ print({"multi_update": "ok"}) # Delete (single) client.delete("account", account_id) -# Bulk delete (schedules BulkDelete and returns job id) -job_id = client.delete("account", ids) +# Delete multiple sequentially +client.delete("account", ids) + +# Or queue a async bulk delete job +job_id = client.delete_async("account", ids) # SQL (read-only) via Web API `?sql=` rows = client.query_sql("SELECT TOP 3 accountid, name FROM account ORDER BY createdon DESC") @@ -334,8 +339,8 @@ client.delete_table("SampleItem") # delete table (friendly name or explici Notes: - `create` always returns a list of GUIDs (length 1 for single input). -- `update` returns `None`. -- `delete` returns `None` for single-record delete/sequential multi-record delete, and the BulkDelete async job ID for BulkDelete. +- `update` and `delete` returns `None`. +- `delete_async` returns the BulkDelete async job ID. - Passing a list of payloads to `create` triggers bulk create and returns `list[str]` of IDs. - `get` supports single record retrieval with record id or paging through result sets (prefer `select` to limit columns). - For CRUD methods that take a record id, pass the GUID string (36-char hyphenated). Parentheses around the GUID are accepted but not required. diff --git a/examples/quickstart.py b/examples/quickstart.py index 7b6d713..d2dc3f6 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -33,6 +33,10 @@ # Create a credential we can reuse (for DataverseClient) credential = InteractiveBrowserCredential() client = DataverseClient(base_url=base_url, credential=credential) +elastic_table_schema = "new_ElasticDeleteDemo" +elastic_table_created_this_run = False +elastic_table_logical_name: Optional[str] = None +elastic_table_metadata_id: Optional[str] = None # Small helpers: call logging and step pauses def log_call(call: str) -> None: @@ -69,6 +73,129 @@ def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403 break if last_exc: raise last_exc + +def run_elastic_delete_demo() -> None: + global elastic_table_created_this_run, elastic_table_logical_name, elastic_table_metadata_id + print("Elastic DeleteMultiple demo (elastic table setup):") + odata_client = client._get_odata() + schema_name = elastic_table_schema + publisher_prefix = schema_name.split("_", 1)[0] if "_" in schema_name else schema_name + primary_attr = f"{publisher_prefix}_Name" + count_attr = f"{publisher_prefix}_Count" + flag_attr = f"{publisher_prefix}_Flag" + + def _fetch_metadata() -> Optional[dict]: + url = f"{odata_client.api}/EntityDefinitions" + params = { + "$select": "MetadataId,LogicalName,EntitySetName,SchemaName,TableType", + "$filter": f"SchemaName eq '{schema_name}'", + } + r = odata_client._request("get", url, params=params) + try: + body = r.json() if r.text else {} + except ValueError: + return None + items = body.get("value") if isinstance(body, dict) else None + if isinstance(items, list) and items: + md = items[0] + return md if isinstance(md, dict) else None + return None + + try: + metadata = _fetch_metadata() + if metadata and str(metadata.get("TableType", "")).lower() != "elastic": + print({ + "elastic_table": schema_name, + "skipped": True, + "reason": "Existing table is not elastic; DeleteMultiple demo not run", + "table_type": metadata.get("TableType"), + }) + return + if not metadata: + log_call(f"POST EntityDefinitions (TableType=Elastic) for {schema_name}") + attributes = [ + odata_client._attribute_payload(primary_attr, "string", is_primary_name=True), + odata_client._attribute_payload(count_attr, "int"), + odata_client._attribute_payload(flag_attr, "bool"), + ] + attrs = [a for a in attributes if a] + payload = { + "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", + "SchemaName": schema_name, + "DisplayName": odata_client._label("Elastic Delete Demo"), + "DisplayCollectionName": odata_client._label("Elastic Delete Demos"), + "Description": odata_client._label("Elastic table for DeleteMultiple quickstart validation"), + "OwnershipType": "UserOwned", + "HasActivities": False, + "HasNotes": True, + "IsEnabledForCharts": False, + "IsVirtualEntityReportingEnabled": False, + "IsActivity": False, + "TableType": "Elastic", + "Attributes": attrs, + } + def _create_elastic(): + odata_client._request("post", f"{odata_client.api}/EntityDefinitions", json=payload) + return True + try: + backoff_retry(_create_elastic) + except Exception as create_exc: + print({ + "elastic_table": schema_name, + "skipped": True, + "reason": "Elastic table creation failed", + "error": str(create_exc), + }) + return + ready = backoff_retry(lambda: odata_client._wait_for_entity_ready(schema_name), retry_http_statuses=()) + if not ready or not ready.get("MetadataId"): + raise RuntimeError("Elastic demo table metadata not ready") + metadata = _fetch_metadata() + elastic_table_created_this_run = True + if not metadata: + print({"elastic_table": schema_name, "skipped": True, "reason": "Metadata unavailable"}) + return + elastic_table_logical_name = metadata.get("LogicalName") + elastic_table_metadata_id = metadata.get("MetadataId") + odata_client._elastic_table_cache.pop(elastic_table_logical_name, None) + odata_client._logical_to_entityset_cache.pop(elastic_table_logical_name, None) + odata_client._logical_primaryid_cache.pop(elastic_table_logical_name, None) + is_elastic = odata_client._is_elastic_table(elastic_table_logical_name) if elastic_table_logical_name else False + print({ + "elastic_table": schema_name, + "logical_name": elastic_table_logical_name, + "table_type_reported": metadata.get("TableType"), + "is_elastic": bool(is_elastic), + }) + logical = elastic_table_logical_name + if not logical: + print({"elastic_table": schema_name, "skipped": True, "reason": "Logical name missing"}) + return + prefix = logical.split("_", 1)[0] if "_" in logical else logical + name_key = f"{prefix}_name" + count_key = f"{prefix}_count" + flag_key = f"{prefix}_flag" + records = [ + {name_key: "Elastic Demo A", count_key: 10, flag_key: True}, + {name_key: "Elastic Demo B", count_key: 11, flag_key: False}, + {name_key: "Elastic Demo C", count_key: 12, flag_key: True}, + ] + log_call(f"client.create('{logical}', <{len(records)} elastic records>)") + created_ids = backoff_retry(lambda: client.create(logical, records)) + valid_ids = [rid for rid in created_ids if isinstance(rid, str)] if isinstance(created_ids, list) else [] + if not valid_ids: + raise RuntimeError("Elastic demo record creation returned no GUIDs") + print({"elastic_create_ids": valid_ids}) + log_call(f"client.delete('{logical}', <{len(valid_ids)} elastic ids>)") + backoff_retry(lambda: client.delete(logical, valid_ids)) + print({ + "elastic_delete_multiple": { + "requested": len(valid_ids), + "succeeded": True, + } + }) + except Exception as exc: + print({"elastic_demo_error": str(exc)}) # Enum demonstrating local option set creation with multilingual labels (for French labels to work, enable French language in the environment first) class Status(IntEnum): @@ -88,6 +215,10 @@ class Status(IntEnum): } } +pause("Run elastic DeleteMultiple demo") +run_elastic_delete_demo() +sys.exit(0) + print("Ensure custom table exists (Metadata):") table_info = None created_this_run = False @@ -512,16 +643,16 @@ def run_paging_demo(label: str, *, top: Optional[int], page_size: Optional[int]) # Fire-and-forget bulk delete for the first portion try: - log_call(f"client.delete('{logical}', <{len(bulk_targets)} ids>, use_bulk_delete=True)") - bulk_job_id = client.delete(logical, bulk_targets) + log_call(f"client.delete_async('{logical}', <{len(bulk_targets)} ids>)") + bulk_job_id = client.delete_async(logical, bulk_targets) except Exception as ex: bulk_error = str(ex) # Sequential deletes for the remainder try: - log_call(f"client.delete('{logical}', <{len(sequential_targets)} ids>, use_bulk_delete=False)") + log_call(f"client.delete('{logical}', <{len(sequential_targets)} ids>)") for rid in sequential_targets: - backoff_retry(lambda rid=rid: client.delete(logical, rid, use_bulk_delete=False)) + backoff_retry(lambda rid=rid: client.delete(logical, rid)) except Exception as ex: sequential_error = str(ex) @@ -655,5 +786,16 @@ def _ensure_removed(): print({"table_deleted": False, "reason": "not found"}) except Exception as e: print(f"Delete table failed: {e}") + if elastic_table_created_this_run: + try: + log_call(f"client.delete_table('{elastic_table_schema}')") + client.delete_table(elastic_table_schema) + print({"elastic_table_deleted": True}) + except Exception as e: + print({"elastic_table_delete_error": str(e)}) + elif elastic_table_logical_name: + print({"elastic_table_deleted": False, "reason": "skipped (table existed before run)"}) else: print({"table_deleted": False, "reason": "user opted to keep table"}) + if elastic_table_created_this_run: + print({"elastic_table_deleted": False, "reason": "user opted to keep table"}) diff --git a/src/dataverse_sdk/client.py b/src/dataverse_sdk/client.py index 055622e..6f34866 100644 --- a/src/dataverse_sdk/client.py +++ b/src/dataverse_sdk/client.py @@ -11,7 +11,6 @@ from .config import DataverseConfig from .odata import ODataClient - class DataverseClient: """ High-level client for Microsoft Dataverse operations. @@ -208,8 +207,7 @@ def delete( self, logical_name: str, ids: Union[str, List[str]], - use_bulk_delete: bool = True, - ) -> Optional[str]: + ) -> None: """ Delete one or more records by GUID. @@ -217,15 +215,12 @@ def delete( :type logical_name: str :param ids: Single GUID string or list of GUID strings to delete. :type ids: str or list[str] - :param use_bulk_delete: When ``True`` (default) and ``ids`` is a list, execute the BulkDelete action and - return its async job identifier. When ``False`` each record is deleted sequentially. - :type use_bulk_delete: bool :raises TypeError: If ``ids`` is not str or list[str]. :raises HttpError: If the underlying Web API delete request fails. - - :return: BulkDelete job ID when deleting multiple records via BulkDelete; otherwise ``None``. - :rtype: str or None + + :return: ``None`` once the requested records have been deleted sequentially. + :rtype: None Example: Delete a single record:: @@ -234,7 +229,7 @@ def delete( Delete multiple records:: - job_id = client.delete("account", [id1, id2, id3]) + client.delete("account", [id1, id2, id3]) """ od = self._get_odata() if isinstance(ids, str): @@ -246,12 +241,50 @@ def delete( return None if not all(isinstance(rid, str) for rid in ids): raise TypeError("ids must contain string GUIDs") - if use_bulk_delete: - return od._delete_multiple(logical_name, ids) + if od._is_elastic_table(logical_name): + od._delete_multiple(logical_name, ids) + return None for rid in ids: od._delete(logical_name, rid) return None + def delete_async( + self, + logical_name: str, + ids: Union[str, List[str]], + ) -> str: + """ + Issue an asynchronous BulkDelete job for one or more records. + + :param logical_name: Logical (singular) entity name, e.g. ``"account"``. + :type logical_name: str + :param ids: Single GUID string or list of GUID strings to delete. + :type ids: str or list[str] + + :raises TypeError: If ``ids`` is not str or list[str]. + :raises HttpError: If the BulkDelete request fails. + + :return: BulkDelete job identifier, a dummy if ids is empty. + :rtype: str + + Example: + Queue a bulk delete:: + + job_id = client.delete_async("account", [id1, id2, id3]) + """ + od = self._get_odata() + if isinstance(ids, str): + return od._delete_async(logical_name, [ids]) + elif isinstance(ids, list): + if not ids: + noop_bulkdelete_job_id = "00000000-0000-0000-0000-000000000000" + return noop_bulkdelete_job_id + if not all(isinstance(rid, str) for rid in ids): + raise TypeError("ids must contain string GUIDs") + return od._delete_async(logical_name, ids) + else: + raise TypeError("ids must be str or list[str]") + def get( self, logical_name: str, diff --git a/src/dataverse_sdk/odata.py b/src/dataverse_sdk/odata.py index 1e6f133..dcc41db 100644 --- a/src/dataverse_sdk/odata.py +++ b/src/dataverse_sdk/odata.py @@ -50,6 +50,8 @@ def __init__( self._logical_to_entityset_cache: dict[str, str] = {} # Cache: logical name -> primary id attribute (e.g. accountid) self._logical_primaryid_cache: dict[str, str] = {} + # Cache: logical name -> whether the table is elastic + self._elastic_table_cache: dict[str, bool] = {} # Picklist label cache: (logical_name, attribute_logical) -> {'map': {...}, 'ts': epoch_seconds} self._picklist_label_cache = {} self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL @@ -285,7 +287,22 @@ def _update_by_ids(self, logical_name: str, ids: List[str], changes: Union[Dict[ self._update_multiple(entity_set, logical_name, batch) return None - def _delete_multiple( + def _delete_multiple(self, logical_name: str, ids: List[str]) -> None: + """Delete many records synchronously via the DeleteMultiple collection action.""" + entity_set = self._entity_set_from_logical(logical_name) + pk_attr = self._primary_id_attr(logical_name) + targets: List[Dict[str, Any]] = [] + for rid in ids: + targets.append({ + "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}", + pk_attr: rid, + }) + payload = {"Targets": targets} + url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.DeleteMultiple" + self._request("post", url, json=payload) + return None + + def _delete_async( self, logical_name: str, ids: List[str], @@ -656,6 +673,37 @@ def _entity_set_from_logical(self, logical: str) -> str: self._logical_primaryid_cache[logical] = primary_id_attr return es + def _is_elastic_table(self, logical: str) -> bool: + """Return True when the target table is configured as an elastic table.""" + if not logical: + raise ValueError("logical name required") + cached = self._elastic_table_cache.get(logical) + if cached is not None: + return cached + url = f"{self.api}/EntityDefinitions" + logical_escaped = self._escape_odata_quotes(logical) + params = { + "$select": "LogicalName,TableType", + "$filter": f"LogicalName eq '{logical_escaped}'", + } + r = self._request("get", url, params=params) + try: + body = r.json() + items = body.get("value", []) if isinstance(body, dict) else [] + except ValueError: + items = [] + is_elastic = False + if items: + md = items[0] + if isinstance(md, dict): + table_type = md.get("TableType") + if isinstance(table_type, str): + is_elastic = table_type.strip().lower() == "elastic" + else: + is_elastic = False + self._elastic_table_cache[logical] = is_elastic + return is_elastic + # ---------------------- Table metadata helpers ---------------------- def _label(self, text: str) -> Dict[str, Any]: lang = int(self.config.language_code) From 59ef366e6862c6ce9d933f31e6c2a81bc276d5c8 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Mon, 10 Nov 2025 13:15:11 -0800 Subject: [PATCH 2/7] clean up changes --- README.md | 3 +- examples/quickstart.py | 131 ------------------------------------ src/dataverse_sdk/client.py | 4 +- src/dataverse_sdk/odata.py | 15 ++++- 4 files changed, 17 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 874cdb3..b08cd71 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ A Python package allowing developers to connect to Dataverse environments for DD - OData CRUD — Unified methods `create(logical_name, record|records)`, `update(logical_name, id|ids, patch|patches)`, `delete(logical_name, id|ids)` plus `get` with record id or filters and `delete_async` for better multi-record delete performance. - Bulk create — Pass a list of records to `create(...)` to invoke the bound `CreateMultiple` action; returns `list[str]` of GUIDs. If any payload omits `@odata.type` the SDK resolves and stamps it (cached). - Bulk update — Provide a list of IDs with a single patch (broadcast) or a list of per‑record patches to `update(...)`; internally uses the bound `UpdateMultiple` action; returns nothing. Each record must include the primary key attribute when sent to UpdateMultiple. +- Bulk delete - Provide a list of IDs to `delete(...)` or `delete_async(...)`. `delete` internally uses `DeleteMultiple` for elastic tables, for standard tables it is a loop over single-record delete. `delete_async` interally uses async BulkDelete. - Retrieve multiple (paging) — Generator-based `get(...)` that yields pages, supports `$top` and Prefer: `odata.maxpagesize` (`page_size`). - Upload files — Call `upload_file(logical_name, ...)` and an upload method will be auto picked (you can override the mode). See https://learn.microsoft.com/en-us/power-apps/developer/data-platform/file-column-data?tabs=sdk#upload-files - Metadata helpers — Create/inspect/delete tables and create/delete columns (EntityDefinitions + Attributes). @@ -145,7 +146,7 @@ print({"multi_update": "ok"}) # Delete (single) client.delete("account", account_id) -# Delete multiple sequentially +# Delete (multiple) client.delete("account", ids) # Or queue a async bulk delete job diff --git a/examples/quickstart.py b/examples/quickstart.py index d2dc3f6..b0c89b4 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -33,10 +33,6 @@ # Create a credential we can reuse (for DataverseClient) credential = InteractiveBrowserCredential() client = DataverseClient(base_url=base_url, credential=credential) -elastic_table_schema = "new_ElasticDeleteDemo" -elastic_table_created_this_run = False -elastic_table_logical_name: Optional[str] = None -elastic_table_metadata_id: Optional[str] = None # Small helpers: call logging and step pauses def log_call(call: str) -> None: @@ -73,129 +69,6 @@ def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403 break if last_exc: raise last_exc - -def run_elastic_delete_demo() -> None: - global elastic_table_created_this_run, elastic_table_logical_name, elastic_table_metadata_id - print("Elastic DeleteMultiple demo (elastic table setup):") - odata_client = client._get_odata() - schema_name = elastic_table_schema - publisher_prefix = schema_name.split("_", 1)[0] if "_" in schema_name else schema_name - primary_attr = f"{publisher_prefix}_Name" - count_attr = f"{publisher_prefix}_Count" - flag_attr = f"{publisher_prefix}_Flag" - - def _fetch_metadata() -> Optional[dict]: - url = f"{odata_client.api}/EntityDefinitions" - params = { - "$select": "MetadataId,LogicalName,EntitySetName,SchemaName,TableType", - "$filter": f"SchemaName eq '{schema_name}'", - } - r = odata_client._request("get", url, params=params) - try: - body = r.json() if r.text else {} - except ValueError: - return None - items = body.get("value") if isinstance(body, dict) else None - if isinstance(items, list) and items: - md = items[0] - return md if isinstance(md, dict) else None - return None - - try: - metadata = _fetch_metadata() - if metadata and str(metadata.get("TableType", "")).lower() != "elastic": - print({ - "elastic_table": schema_name, - "skipped": True, - "reason": "Existing table is not elastic; DeleteMultiple demo not run", - "table_type": metadata.get("TableType"), - }) - return - if not metadata: - log_call(f"POST EntityDefinitions (TableType=Elastic) for {schema_name}") - attributes = [ - odata_client._attribute_payload(primary_attr, "string", is_primary_name=True), - odata_client._attribute_payload(count_attr, "int"), - odata_client._attribute_payload(flag_attr, "bool"), - ] - attrs = [a for a in attributes if a] - payload = { - "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", - "SchemaName": schema_name, - "DisplayName": odata_client._label("Elastic Delete Demo"), - "DisplayCollectionName": odata_client._label("Elastic Delete Demos"), - "Description": odata_client._label("Elastic table for DeleteMultiple quickstart validation"), - "OwnershipType": "UserOwned", - "HasActivities": False, - "HasNotes": True, - "IsEnabledForCharts": False, - "IsVirtualEntityReportingEnabled": False, - "IsActivity": False, - "TableType": "Elastic", - "Attributes": attrs, - } - def _create_elastic(): - odata_client._request("post", f"{odata_client.api}/EntityDefinitions", json=payload) - return True - try: - backoff_retry(_create_elastic) - except Exception as create_exc: - print({ - "elastic_table": schema_name, - "skipped": True, - "reason": "Elastic table creation failed", - "error": str(create_exc), - }) - return - ready = backoff_retry(lambda: odata_client._wait_for_entity_ready(schema_name), retry_http_statuses=()) - if not ready or not ready.get("MetadataId"): - raise RuntimeError("Elastic demo table metadata not ready") - metadata = _fetch_metadata() - elastic_table_created_this_run = True - if not metadata: - print({"elastic_table": schema_name, "skipped": True, "reason": "Metadata unavailable"}) - return - elastic_table_logical_name = metadata.get("LogicalName") - elastic_table_metadata_id = metadata.get("MetadataId") - odata_client._elastic_table_cache.pop(elastic_table_logical_name, None) - odata_client._logical_to_entityset_cache.pop(elastic_table_logical_name, None) - odata_client._logical_primaryid_cache.pop(elastic_table_logical_name, None) - is_elastic = odata_client._is_elastic_table(elastic_table_logical_name) if elastic_table_logical_name else False - print({ - "elastic_table": schema_name, - "logical_name": elastic_table_logical_name, - "table_type_reported": metadata.get("TableType"), - "is_elastic": bool(is_elastic), - }) - logical = elastic_table_logical_name - if not logical: - print({"elastic_table": schema_name, "skipped": True, "reason": "Logical name missing"}) - return - prefix = logical.split("_", 1)[0] if "_" in logical else logical - name_key = f"{prefix}_name" - count_key = f"{prefix}_count" - flag_key = f"{prefix}_flag" - records = [ - {name_key: "Elastic Demo A", count_key: 10, flag_key: True}, - {name_key: "Elastic Demo B", count_key: 11, flag_key: False}, - {name_key: "Elastic Demo C", count_key: 12, flag_key: True}, - ] - log_call(f"client.create('{logical}', <{len(records)} elastic records>)") - created_ids = backoff_retry(lambda: client.create(logical, records)) - valid_ids = [rid for rid in created_ids if isinstance(rid, str)] if isinstance(created_ids, list) else [] - if not valid_ids: - raise RuntimeError("Elastic demo record creation returned no GUIDs") - print({"elastic_create_ids": valid_ids}) - log_call(f"client.delete('{logical}', <{len(valid_ids)} elastic ids>)") - backoff_retry(lambda: client.delete(logical, valid_ids)) - print({ - "elastic_delete_multiple": { - "requested": len(valid_ids), - "succeeded": True, - } - }) - except Exception as exc: - print({"elastic_demo_error": str(exc)}) # Enum demonstrating local option set creation with multilingual labels (for French labels to work, enable French language in the environment first) class Status(IntEnum): @@ -215,10 +88,6 @@ class Status(IntEnum): } } -pause("Run elastic DeleteMultiple demo") -run_elastic_delete_demo() -sys.exit(0) - print("Ensure custom table exists (Metadata):") table_info = None created_this_run = False diff --git a/src/dataverse_sdk/client.py b/src/dataverse_sdk/client.py index 6f34866..175c670 100644 --- a/src/dataverse_sdk/client.py +++ b/src/dataverse_sdk/client.py @@ -11,6 +11,7 @@ from .config import DataverseConfig from .odata import ODataClient + class DataverseClient: """ High-level client for Microsoft Dataverse operations. @@ -219,9 +220,6 @@ def delete( :raises TypeError: If ``ids`` is not str or list[str]. :raises HttpError: If the underlying Web API delete request fails. - :return: ``None`` once the requested records have been deleted sequentially. - :rtype: None - Example: Delete a single record:: diff --git a/src/dataverse_sdk/odata.py b/src/dataverse_sdk/odata.py index dcc41db..bdb6f92 100644 --- a/src/dataverse_sdk/odata.py +++ b/src/dataverse_sdk/odata.py @@ -288,7 +288,20 @@ def _update_by_ids(self, logical_name: str, ids: List[str], changes: Union[Dict[ return None def _delete_multiple(self, logical_name: str, ids: List[str]) -> None: - """Delete many records synchronously via the DeleteMultiple collection action.""" + """Delete records using the collection-bound DeleteMultiple action. + + Parameters + ---------- + logical_name : str + Singular logical entity name. + ids : list[str] + GUIDs for the records to remove. + + Returns + ------- + None + No representation is returned. + """ entity_set = self._entity_set_from_logical(logical_name) pk_attr = self._primary_id_attr(logical_name) targets: List[Dict[str, Any]] = [] From 7e8c1942df70ff67a544c3f683179ee5a434d329 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Mon, 10 Nov 2025 13:20:15 -0800 Subject: [PATCH 3/7] more cleanup --- README.md | 2 +- examples/quickstart.py | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index b08cd71..0e790f0 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Auth: | `update` | `update(logical_name, list[id], patch)` | `None` | Broadcast; same patch applied to all IDs (UpdateMultiple). | | `update` | `update(logical_name, list[id], list[patch])` | `None` | 1:1 patches; lengths must match (UpdateMultiple). | | `delete` | `delete(logical_name, id)` | `None` | Delete one record. | -| `delete` | `delete(logical_name, list[id])` | `None` | Sequential deletes (loops over single-record delete). | +| `delete` | `delete(logical_name, list[id])` | `None` | DeleteMultiple for elastic tables or loops over single-record delete for standard tables. | | `delete_async` | `delete_async(logical_name, id)` | `str` | Async single-record delete. | | `delete_async` | `delete_async(logical_name, list[id])` | `str` | Async multi-record delete. | | `query_sql` | `query_sql(sql)` | `list[dict]` | Constrained read-only SELECT via `?sql=`. | diff --git a/examples/quickstart.py b/examples/quickstart.py index b0c89b4..37b3c37 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -655,16 +655,5 @@ def _ensure_removed(): print({"table_deleted": False, "reason": "not found"}) except Exception as e: print(f"Delete table failed: {e}") - if elastic_table_created_this_run: - try: - log_call(f"client.delete_table('{elastic_table_schema}')") - client.delete_table(elastic_table_schema) - print({"elastic_table_deleted": True}) - except Exception as e: - print({"elastic_table_delete_error": str(e)}) - elif elastic_table_logical_name: - print({"elastic_table_deleted": False, "reason": "skipped (table existed before run)"}) else: - print({"table_deleted": False, "reason": "user opted to keep table"}) - if elastic_table_created_this_run: - print({"elastic_table_deleted": False, "reason": "user opted to keep table"}) + print({"table_deleted": False, "reason": "user opted to keep table"}) \ No newline at end of file From afe676e234741ac61e1ab6aaf8fc5b24501abfb9 Mon Sep 17 00:00:00 2001 From: zhaodongwang-msft Date: Mon, 10 Nov 2025 13:21:48 -0800 Subject: [PATCH 4/7] Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e790f0..13ef9b3 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Auth: Guidelines: - `create` always returns a list of GUIDs (1 for single, N for bulk). -- `update` and `delete` always returns `None`. +- `update` and `delete` always return `None`. - Bulk update chooses broadcast vs per-record by the type of `changes` (dict vs list). - `delete_async` returns the BulkDelete async job ID and doesn't wait for completion. - It's recommended to use delete_async for multi-record delete for better performance. From 32d0b5fe527125e171c98adf423dc1436bc0fa2c Mon Sep 17 00:00:00 2001 From: zhaodongwang-msft Date: Mon, 10 Nov 2025 13:22:09 -0800 Subject: [PATCH 5/7] Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 13ef9b3..12f19f8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Python package allowing developers to connect to Dataverse environments for DD - OData CRUD — Unified methods `create(logical_name, record|records)`, `update(logical_name, id|ids, patch|patches)`, `delete(logical_name, id|ids)` plus `get` with record id or filters and `delete_async` for better multi-record delete performance. - Bulk create — Pass a list of records to `create(...)` to invoke the bound `CreateMultiple` action; returns `list[str]` of GUIDs. If any payload omits `@odata.type` the SDK resolves and stamps it (cached). - Bulk update — Provide a list of IDs with a single patch (broadcast) or a list of per‑record patches to `update(...)`; internally uses the bound `UpdateMultiple` action; returns nothing. Each record must include the primary key attribute when sent to UpdateMultiple. -- Bulk delete - Provide a list of IDs to `delete(...)` or `delete_async(...)`. `delete` internally uses `DeleteMultiple` for elastic tables, for standard tables it is a loop over single-record delete. `delete_async` interally uses async BulkDelete. +- Bulk delete - Provide a list of IDs to `delete(...)` or `delete_async(...)`. `delete` internally uses `DeleteMultiple` for elastic tables, for standard tables it is a loop over single-record delete. `delete_async` internally uses async BulkDelete. - Retrieve multiple (paging) — Generator-based `get(...)` that yields pages, supports `$top` and Prefer: `odata.maxpagesize` (`page_size`). - Upload files — Call `upload_file(logical_name, ...)` and an upload method will be auto picked (you can override the mode). See https://learn.microsoft.com/en-us/power-apps/developer/data-platform/file-column-data?tabs=sdk#upload-files - Metadata helpers — Create/inspect/delete tables and create/delete columns (EntityDefinitions + Attributes). From 4f9c19d1bb002d750238dd8f4039c088ded4d0e7 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Mon, 10 Nov 2025 13:36:21 -0800 Subject: [PATCH 6/7] copilot suggestions --- src/dataverse_sdk/client.py | 9 +++++---- src/dataverse_sdk/odata.py | 13 ++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/dataverse_sdk/client.py b/src/dataverse_sdk/client.py index 175c670..64526b1 100644 --- a/src/dataverse_sdk/client.py +++ b/src/dataverse_sdk/client.py @@ -274,12 +274,13 @@ def delete_async( if isinstance(ids, str): return od._delete_async(logical_name, [ids]) elif isinstance(ids, list): - if not ids: - noop_bulkdelete_job_id = "00000000-0000-0000-0000-000000000000" - return noop_bulkdelete_job_id if not all(isinstance(rid, str) for rid in ids): raise TypeError("ids must contain string GUIDs") - return od._delete_async(logical_name, ids) + sanitized = [rid.strip() for rid in ids if isinstance(rid, str) and rid.strip()] + if not sanitized: + noop_bulkdelete_job_id = "00000000-0000-0000-0000-000000000000" + return noop_bulkdelete_job_id + return od._delete_async(logical_name, sanitized) else: raise TypeError("ids must be str or list[str]") diff --git a/src/dataverse_sdk/odata.py b/src/dataverse_sdk/odata.py index bdb6f92..d71f17a 100644 --- a/src/dataverse_sdk/odata.py +++ b/src/dataverse_sdk/odata.py @@ -319,15 +319,13 @@ def _delete_async( self, logical_name: str, ids: List[str], - ) -> Optional[str]: + ) -> str: """Delete many records by GUID list. Returns the asynchronous job identifier reported by the BulkDelete action. """ - targets = [rid for rid in ids if rid] - if not targets: - return None - value_objects = [{"Value": rid, "Type": "System.Guid"} for rid in targets] + noop_job_id = "00000000-0000-0000-0000-000000000000" + value_objects = [{"Value": rid, "Type": "System.Guid"} for rid in ids] pk_attr = self._primary_id_attr(logical_name) timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") @@ -368,15 +366,16 @@ def _delete_async( url = f"{self.api}/BulkDelete" response = self._request("post", url, json=payload, expected=(200, 202, 204)) - job_id = None try: body = response.json() if response.text else {} except ValueError: body = {} if isinstance(body, dict): job_id = body.get("JobId") + if isinstance(job_id, str) and job_id.strip(): + return job_id - return job_id + return noop_job_id def _format_key(self, key: str) -> str: k = key.strip() From eabbea432079385518c87475a02c9d1707fc14b2 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Fri, 14 Nov 2025 10:56:51 -0800 Subject: [PATCH 7/7] update for merge --- README.md | 7 +- examples/advanced/walkthrough.py | 18 +- examples/basic/quickstart.py | 659 ---------------------- src/PowerPlatform/Dataverse/data/odata.py | 48 +- 4 files changed, 41 insertions(+), 691 deletions(-) delete mode 100644 examples/basic/quickstart.py diff --git a/README.md b/README.md index 1d34165..88c9bd7 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ A Python client library for Microsoft Dataverse that provides a unified interfac ## Key features - **🔄 CRUD Operations**: Create, read, update, and delete records with support for bulk operations and automatic retry -- **⚡ True Bulk Operations**: Automatically uses Dataverse's native `CreateMultiple`, `UpdateMultiple`, and `BulkDelete` Web API operations for maximum performance and transactional integrity +- **⚡ True Bulk Operations**: Automatically uses Dataverse's native `CreateMultiple`, `UpdateMultiple`, `DeleteMultiple` (elastic tables only), and `BulkDelete` Web API operations for maximum performance and transactional integrity - **📊 SQL Queries**: Execute read-only SQL queries via the Dataverse Web API `?sql=` parameter - **🏗️ Table Management**: Create, inspect, and delete custom tables and columns programmatically - **📎 File Operations**: Upload files to Dataverse file columns with automatic chunking for large files @@ -162,7 +162,10 @@ ids = client.create("account", payloads) client.update("account", ids, {"industry": "Technology"}) # Bulk delete -client.delete("account", ids, use_bulk_delete=True) +client.delete("account", ids) + +# Bulk delete async +client.delete_async("account", ids) ``` ### Query data diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 5311592..a0c3aba 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -281,11 +281,19 @@ def main(): client.delete(table_name, id1) print(f"✓ Deleted single record: {id1}") - # Multiple delete (delete the paging demo records) - log_call(f"client.delete('{table_name}', [{len(paging_ids)} IDs])") - job_id = client.delete(table_name, paging_ids) - print(f"✓ Bulk delete job started: {job_id}") - print(f" (Deleting {len(paging_ids)} paging demo records)") + # Multiple delete (demonstrate async bulk job and synchronous fallback) + midpoint = len(paging_ids) // 2 + async_ids = paging_ids[:midpoint] + sync_ids = paging_ids[midpoint:] + + log_call(f"client.delete_async('{table_name}', [{len(async_ids)} IDs])") + job_id = client.delete_async(table_name, async_ids) + print(f"✓ Bulk delete job queued: {job_id}") + print(f" (Deleting {len(async_ids)} paging demo records asynchronously)") + + log_call(f"client.delete('{table_name}', [{len(sync_ids)} IDs])") + client.delete(table_name, sync_ids) + print(f"✓ Synchronously deleted {len(sync_ids)} paging demo records") # ============================================================================ # 11. CLEANUP diff --git a/examples/basic/quickstart.py b/examples/basic/quickstart.py deleted file mode 100644 index 103826b..0000000 --- a/examples/basic/quickstart.py +++ /dev/null @@ -1,659 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import sys -from pathlib import Path -import os -from typing import Optional - -# Add src to PYTHONPATH for local runs -sys.path.append(str(Path(__file__).resolve().parents[1] / "src")) - -from dataverse_sdk import DataverseClient -from dataverse_sdk.core.errors import MetadataError -from enum import IntEnum -from azure.identity import InteractiveBrowserCredential -import traceback -import requests -import time -from datetime import date, timedelta - - -entered = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() -if not entered: - print("No URL entered; exiting.") - sys.exit(1) - -base_url = entered.rstrip('/') -delete_choice = input("Delete the new_SampleItem table at end? (Y/n): ").strip() or "y" -delete_table_at_end = (str(delete_choice).lower() in ("y", "yes", "true", "1")) -# Ask once whether to pause between steps during this run -pause_choice = input("Pause between test steps? (y/N): ").strip() or "n" -pause_between_steps = (str(pause_choice).lower() in ("y", "yes", "true", "1")) -# Create a credential we can reuse (for DataverseClient) -credential = InteractiveBrowserCredential() -client = DataverseClient(base_url=base_url, credential=credential) - -# Small helpers: call logging and step pauses -def log_call(call: str) -> None: - print({"call": call}) - -def pause(next_step: str) -> None: - if pause_between_steps: - try: - input(f"\nNext: {next_step} — press Enter to continue...") - except EOFError: - # If stdin is not available, just proceed - pass - -# Small generic backoff helper used only in this quickstart -# Include common transient statuses like 429/5xx to improve resilience. -def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403, 404, 409, 412, 429, 500, 502, 503, 504), retry_if=None): - last_exc = None - for delay in delays: - if delay: - time.sleep(delay) - try: - return op() - except Exception as ex: - print(f'Request failed: {ex}') - last_exc = ex - if retry_if and retry_if(ex): - print("Retrying operation...") - continue - if isinstance(ex, requests.exceptions.HTTPError): - code = getattr(getattr(ex, 'response', None), 'status_code', None) - if code in retry_http_statuses: - print("Retrying operation...") - continue - break - if last_exc: - raise last_exc - -# Enum demonstrating local option set creation with multilingual labels (for French labels to work, enable French language in the environment first) -class Status(IntEnum): - Active = 1 - Inactive = 2 - Archived = 5 - __labels__ = { - 1033: { - "Active": "Active", - "Inactive": "Inactive", - "Archived": "Archived", - }, - 1036: { - "Active": "Actif", - "Inactive": "Inactif", - "Archived": "Archivé", - } - } - -print("Ensure custom table exists (Metadata):") -table_info = None -created_this_run = False - -# Check for existing table using list_tables -log_call("client.list_tables()") -tables = client.list_tables() -existing_table = next((t for t in tables if t.get("SchemaName") == "new_SampleItem"), None) -if existing_table: - table_info = client.get_table_info("new_SampleItem") - created_this_run = False - print({ - "table": table_info.get("entity_schema"), - "existed": True, - "entity_set": table_info.get("entity_set_name"), - "logical": table_info.get("entity_logical_name"), - "metadata_id": table_info.get("metadata_id"), - }) - -else: - # Create it since it doesn't exist - try: - log_call("client.create_table('new_SampleItem', schema={code,count,amount,when,active,status})") - table_info = client.create_table( - "new_SampleItem", - { - "code": "string", - "count": "int", - "amount": "decimal", - "when": "datetime", - "active": "bool", - "status": Status, - }, - ) - created_this_run = True if table_info and table_info.get("columns_created") else False - print({ - "table": table_info.get("entity_schema") if table_info else None, - "existed": False, - "entity_set": table_info.get("entity_set_name") if table_info else None, - "logical": table_info.get("entity_logical_name") if table_info else None, - "metadata_id": table_info.get("metadata_id") if table_info else None, - }) - except Exception as e: - # Print full stack trace and any HTTP response details if present - print("Create table failed:") - traceback.print_exc() - resp = getattr(e, 'response', None) - if resp is not None: - try: - print({ - "status": resp.status_code, - "url": getattr(resp, 'url', None), - "body": resp.text[:2000] if getattr(resp, 'text', None) else None, - }) - except Exception: - pass - # Fail fast: all operations must use the custom table - sys.exit(1) -entity_schema = table_info.get("entity_schema") or "new_SampleItem" -logical = table_info.get("entity_logical_name") -metadata_id = table_info.get("metadata_id") -if not metadata_id: - refreshed_info = client.get_table_info(entity_schema) or {} - metadata_id = refreshed_info.get("metadata_id") - if metadata_id: - table_info["metadata_id"] = metadata_id - -# Derive attribute logical name prefix from the entity logical name (segment before first underscore) -attr_prefix = logical.split("_", 1)[0] if "_" in logical else logical -code_key = f"{attr_prefix}_code" -count_key = f"{attr_prefix}_count" -amount_key = f"{attr_prefix}_amount" -when_key = f"{attr_prefix}_when" -status_key = f"{attr_prefix}_status" -id_key = f"{logical}id" - -def summary_from_record(rec: dict) -> dict: - return { - "code": rec.get(code_key), - "count": rec.get(count_key), - "amount": rec.get(amount_key), - "when": rec.get(when_key), - } - -def print_line_summaries(label: str, summaries: list[dict]) -> None: - print(label) - for s in summaries: - print( - f" - id={s.get('id')} code={s.get('code')} " - f"count={s.get('count')} amount={s.get('amount')} when={s.get('when')}" - ) - -def _has_installed_language(base_url: str, credential, lcid: int) -> bool: - try: - token = credential.get_token(f"{base_url}/.default").token - url = f"{base_url}/api/data/v9.2/RetrieveAvailableLanguages()" - headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} - resp = requests.get(url, headers=headers, timeout=15) - if not resp.ok: - return False - data = resp.json() if resp.content else {} - langs: list[int] = [] - for val in data.values(): - if isinstance(val, list) and val and all(isinstance(x, int) for x in val): - langs = val - break - print({"lang_check": {"endpoint": url, "status": resp.status_code, "found": langs, "using": lcid in langs}}) - return lcid in langs - except Exception: - return False - -# if French language (1036) is installed, we use labels in both English and French -use_french_labels = _has_installed_language(base_url, credential, 1036) -if use_french_labels: - print({"labels_language": "fr", "note": "French labels in use."}) -else: - print({"labels_language": "en", "note": "Using English (and numeric values)."}) - -# 2) Create a record in the new table -print("Create records (OData) demonstrating single create and bound CreateMultiple (multi):") - -# Define base payloads -single_payload = { - f"{attr_prefix}_name": "Sample A", - code_key: "X001", - count_key: 42, - amount_key: 123.45, - when_key: "2025-01-01", - f"{attr_prefix}_active": True, - status_key: ("Actif" if use_french_labels else Status.Active.value), -} -# Generate multiple payloads -# Distribution update: roughly one-third English labels, one-third French labels, one-third raw integer values. -# We cycle per record: index % 3 == 1 -> English label, == 2 -> French label (if available, else English), == 0 -> integer value. -multi_payloads: list[dict] = [] -base_date = date(2025, 1, 2) -# Fixed 6-step cycle pattern encapsulated in helper: Active, Inactive, Actif, Inactif, 1, 2 (repeat) -def _status_value_for_index(idx: int, use_french: bool): - pattern = [ - ("label", "Active"), - ("label", "Inactive"), - ("fr_label", "Actif"), - ("fr_label", "Inactif"), - ("int", Status.Active.value), - ("int", Status.Inactive.value), - ] - kind, raw = pattern[(idx - 1) % len(pattern)] - if kind == "label": - return raw - if kind == "fr_label": - if use_french: - return raw - return "Active" if raw == "Actif" else "Inactive" - return raw - -for i in range(1, 16): - multi_payloads.append({ - f"{attr_prefix}_name": f"Sample {i:02d}", - code_key: f"X{200 + i:03d}", - count_key: 5 * i, - amount_key: round(10.0 * i, 2), - when_key: (base_date + timedelta(days=i - 1)).isoformat(), - f"{attr_prefix}_active": True, - status_key: _status_value_for_index(i, use_french_labels), - }) - -record_ids: list[str] = [] - -try: - # Single create returns list[str] (length 1) - log_call(f"client.create('{logical}', single_payload)") - single_ids = backoff_retry(lambda: client.create(logical, single_payload)) - if not (isinstance(single_ids, list) and len(single_ids) == 1): - raise RuntimeError("Unexpected single create return shape (expected one-element list)") - record_ids.extend(single_ids) - - # Multi create returns list[str] - log_call(f"client.create('{logical}', multi_payloads)") - multi_ids = backoff_retry(lambda: client.create(logical, multi_payloads)) - if isinstance(multi_ids, list): - record_ids.extend([mid for mid in multi_ids if isinstance(mid, str)]) - else: - print({"multi_unexpected_type": type(multi_ids).__name__, "value_preview": str(multi_ids)[:300]}) - - print({"entity": logical, "created_ids": record_ids}) - print_line_summaries("Created record summaries (IDs only; representation not fetched):", [{"id": rid} for rid in record_ids[:1]]) -except Exception as e: - # Surface detailed info for debugging (especially multi-create failures) - print(f"Create failed: {e}") - resp = getattr(e, 'response', None) - if resp is not None: - try: - print({ - 'status': resp.status_code, - 'url': getattr(resp, 'url', None), - 'body': resp.text[:2000] if getattr(resp, 'text', None) else None, - 'headers': {k: v for k, v in getattr(resp, 'headers', {}).items() if k.lower() in ('request-id','activityid','dataverse-instanceversion','content-type')} - }) - except Exception: - pass - sys.exit(1) - -pause("Next: Read record") - -# 3) Read record via OData -print("Read (OData):") -try: - if record_ids: - # Read only the first record and move on - target = record_ids[0] - log_call(f"client.get('{logical}', '{target}')") - rec = backoff_retry(lambda: client.get(logical, target)) - print_line_summaries("Read record summary:", [{"id": target, **summary_from_record(rec)}]) - else: - raise RuntimeError("No record created; skipping read.") -except Exception as e: - print(f"Get failed: {e}") -# 3.5) Update record, then read again and verify -print("Update (OData) and verify:") -# Show what will be updated and planned update calls, then pause -try: - if not record_ids: - raise RuntimeError("No record created; skipping update.") - - update_data = { - f"{attr_prefix}_code": "X002", - f"{attr_prefix}_count": 99, - f"{attr_prefix}_amount": 543.21, - f"{attr_prefix}_when": "2025-02-02", - f"{attr_prefix}_active": False, - status_key: ("Inactif" if use_french_labels else Status.Inactive.value), - } - expected_checks = { - f"{attr_prefix}_code": "X002", - f"{attr_prefix}_count": 99, - f"{attr_prefix}_active": False, - status_key: Status.Inactive.value, - } - amount_key = f"{attr_prefix}_amount" - - # Describe what is changing - print( - { - "updating_to": { - code_key: update_data[code_key], - count_key: update_data[count_key], - amount_key: update_data[amount_key], - when_key: update_data[when_key], - } - } - ) - - # Choose a single target to update to keep other records different - target_id = record_ids[0] - pause("Execute Update") - - # Update only the chosen record and summarize - log_call(f"client.update('{logical}', '{target_id}', update_data)") - # Perform update (returns None); follow-up read to verify - backoff_retry(lambda: client.update(logical, target_id, update_data)) - verify_rec = backoff_retry(lambda: client.get(logical, target_id)) - for k, v in expected_checks.items(): - assert verify_rec.get(k) == v, f"Field {k} expected {v}, got {verify_rec.get(k)}" - got = verify_rec.get(amount_key) - got_f = float(got) if got is not None else None - assert got_f is not None and abs(got_f - 543.21) < 1e-6, f"Field {amount_key} expected 543.21, got {got}" - print({"entity": logical, "updated": True}) - print_line_summaries("Updated record summary:", [{"id": target_id, **summary_from_record(verify_rec)}]) -except Exception as e: - print(f"Update/verify failed: {e}") - sys.exit(1) - -# 3.6) Bulk update (UpdateMultiple) demo: update count field on up to first 5 remaining records -print("Bulk update (UpdateMultiple) demo:") -try: - if len(record_ids) > 1: - # Prepare a small subset to update (skip the first already updated one) - subset = record_ids[1:6] - bulk_updates = [] - for idx, rid in enumerate(subset, start=1): - # Simple deterministic changes so user can observe - bulk_updates.append({ - id_key: rid, - count_key: 100 + idx, # new count values - }) - log_call(f"client.update('{logical}', <{len(bulk_updates)} ids>, )") - # Unified update handles multiple via list of patches (returns None) - backoff_retry(lambda: client.update(logical, subset, bulk_updates)) - print({"bulk_update_requested": len(bulk_updates), "bulk_update_completed": True}) - # Verify the updated count values by refetching the subset - verification = [] - # Small delay to reduce risk of any brief replication delay - time.sleep(1) - for rid in subset: - rec = backoff_retry(lambda rid=rid: client.get(logical, rid)) - verification.append({ - "id": rid, - "count": rec.get(count_key), - }) - print({"bulk_update_verification": verification}) - else: - print({"bulk_update_skipped": True, "reason": "not enough records"}) -except Exception as e: - print(f"Bulk update failed: {e}") - -# 4) Query records via SQL (?sql parameter)) -print("Query (SQL via ?sql query parameter):") -try: - import time - pause("Execute SQL Query") - - def _run_query(): - cols = f"{id_key}, {code_key}, {amount_key}, {when_key}" - query = f"SELECT TOP 2 {cols} FROM {logical} ORDER BY {attr_prefix}_amount DESC" - log_call(f"client.query_sql(\"{query}\") (Web API ?sql=)") - return client.query_sql(query) - - def _retry_if(ex: Exception) -> bool: - msg = str(ex) if ex else "" - return ("Invalid table name" in msg) or ("Invalid object name" in msg) - - rows = backoff_retry(_run_query, delays=(0, 2, 5), retry_http_statuses=(), retry_if=_retry_if) - id_key = f"{logical}id" - ids = [r.get(id_key) for r in rows if isinstance(r, dict) and r.get(id_key)] - print({"entity": logical, "rows": len(rows) if isinstance(rows, list) else 0, "ids": ids}) - record_summaries = [] - for row in rows if isinstance(rows, list) else []: - record_summaries.append( - { - "id": row.get(id_key), - "code": row.get(code_key), - "count": row.get(count_key), - "amount": row.get(amount_key), - "when": row.get(when_key), - } - ) - print_line_summaries("SQL record summaries (top 2 by amount):", record_summaries) -except Exception as e: - print(f"SQL query failed: {e}") - -# Pause between SQL query and retrieve-multiple demos -pause("Retrieve multiple (OData paging demos)") - -# 4.5) Retrieve multiple via OData paging (scenarios) -def run_paging_demo(label: str, *, top: Optional[int], page_size: Optional[int]) -> None: - print("") - print({"paging_demo": label, "top": top, "page_size": page_size}) - total = 0 - page_index = 0 - _select = [id_key, code_key, amount_key, when_key, status_key] - _orderby = [f"{code_key} asc"] - for page in client.get( - logical, - select=_select, - filter=None, - orderby=_orderby, - top=top, - expand=None, - page_size=page_size, - ): - page_index += 1 - total += len(page) - print({ - "page": page_index, - "page_size": len(page), - "sample": [ - { - "id": r.get(id_key), - "code": r.get(code_key), - "amount": r.get(amount_key), - "when": r.get(when_key), - "status": r.get(status_key), - } - for r in page[:5] - ], - }) - print({"paging_demo_done": label, "pages": page_index, "total_rows": total}) - print("") - -print("") -print("==============================") -print("Retrieve multiple (OData paging demos)") -print("==============================") -try: - # 1) Tiny page size, no top: force multiple pages - run_paging_demo("page_size=2 (no top)", top=None, page_size=2) - pause("Next paging demo: top=3, page_size=2") - - # 2) Limit total results while keeping small pages - run_paging_demo("top=3, page_size=2", top=3, page_size=2) - pause("Next paging demo: top=2 (default page size)") - - # 3) Limit total results with default server page size (likely one page) - run_paging_demo("top=2 (default page size)", top=2, page_size=None) -except Exception as e: - print(f"Retrieve multiple demos failed: {e}") -# 5) Delete record -print("Delete (OData):") -# Show deletes to be executed (single + bulk) -if 'record_ids' in locals() and record_ids: - print({"delete_count": len(record_ids)}) -pause("Execute Delete (single then bulk)") -try: - if record_ids: - single_target = record_ids[0] - rest_targets = record_ids[1:] - single_error: Optional[str] = None - bulk_job_id: Optional[str] = None - bulk_error: Optional[str] = None - - try: - log_call(f"client.delete('{logical}', '{single_target}')") - backoff_retry(lambda: client.delete(logical, single_target)) - except Exception as ex: - single_error = str(ex) - - half = max(1, len(rest_targets) // 2) - bulk_targets = rest_targets[:half] - sequential_targets = rest_targets[half:] - bulk_error = None - sequential_error = None - - # Fire-and-forget bulk delete for the first portion - try: - log_call(f"client.delete_async('{logical}', <{len(bulk_targets)} ids>)") - bulk_job_id = client.delete_async(logical, bulk_targets) - except Exception as ex: - bulk_error = str(ex) - - # Sequential deletes for the remainder - try: - log_call(f"client.delete('{logical}', <{len(sequential_targets)} ids>)") - for rid in sequential_targets: - backoff_retry(lambda rid=rid: client.delete(logical, rid)) - except Exception as ex: - sequential_error = str(ex) - - print({ - "entity": logical, - "delete_single": { - "id": single_target, - "error": single_error, - }, - "delete_bulk": { - "count": len(bulk_targets), - "job_id": bulk_job_id, - "error": bulk_error, - }, - "delete_sequential": { - "count": len(sequential_targets), - "error": sequential_error, - }, - }) - else: - raise RuntimeError("No record created; skipping delete.") -except Exception as e: - print(f"Delete failed: {e}") - -pause("Next: column metadata helpers") - -# 6) Column metadata helpers: column create/delete -print("Column metadata helpers (create/delete column):") -scratch_column = f"scratch_{int(time.time())}" -column_payload = {scratch_column: "string"} -try: - log_call(f"client.create_column('{entity_schema}', {repr(column_payload)})") - column_create = client.create_columns(entity_schema, column_payload) - if not isinstance(column_create, list) or not column_create: - raise RuntimeError("create_column did not return schema list") - created_details = column_create - if not all(isinstance(item, str) for item in created_details): - raise RuntimeError("create_column entries were not schema strings") - attribute_schema = created_details[0] - odata_client = client._get_odata() - exists_after_create = None - exists_after_delete = None - attr_type_before = None - if metadata_id and attribute_schema: - _ready_message = "Column metadata not yet available" - def _metadata_after_create(): - meta = odata_client._get_attribute_metadata( - metadata_id, - attribute_schema, - extra_select="@odata.type,AttributeType", - ) - if not meta or not meta.get("MetadataId"): - raise RuntimeError(_ready_message) - return meta - - ready_meta = backoff_retry( - _metadata_after_create, - delays=(0, 1, 2, 4, 8), - retry_http_statuses=(), - retry_if=lambda exc: isinstance(exc, RuntimeError) and str(exc) == _ready_message, - ) - exists_after_create = bool(ready_meta) - raw_type = ready_meta.get("@odata.type") or ready_meta.get("AttributeType") - if isinstance(raw_type, str): - attr_type_before = raw_type - lowered = raw_type.lower() - delete_target = attribute_schema or scratch_column - log_call(f"client.delete_column('{entity_schema}', '{delete_target}')") - - def _delete_column(): - return client.delete_columns(entity_schema, delete_target) - - column_delete = backoff_retry( - _delete_column, - delays=(0, 1, 2, 4, 8), - retry_http_statuses=(), - retry_if=lambda exc: ( - isinstance(exc, MetadataError) - or "not found" in str(exc).lower() - or "not yet available" in str(exc).lower() - ), - ) - if not isinstance(column_delete, list) or not column_delete: - raise RuntimeError("delete_column did not return schema list") - deleted_details = column_delete - if not all(isinstance(item, str) for item in deleted_details): - raise RuntimeError("delete_column entries were not schema strings") - if attribute_schema not in deleted_details: - raise RuntimeError("delete_column response missing expected schema name") - if metadata_id and attribute_schema: - _delete_message = "Column metadata still present after delete" - def _ensure_removed(): - meta = odata_client._get_attribute_metadata(metadata_id, attribute_schema) - if meta: - raise RuntimeError(_delete_message) - return True - - removed = backoff_retry( - _ensure_removed, - delays=(0, 1, 2, 4, 8), - retry_http_statuses=(), - retry_if=lambda exc: isinstance(exc, RuntimeError) and str(exc) == _delete_message, - ) - exists_after_delete = not removed - print({ - "created_column": scratch_column, - "create_summary": created_details, - "delete_summary": deleted_details, - "attribute_type_before_delete": attr_type_before, - "exists_after_create": exists_after_create, - "exists_after_delete": exists_after_delete, - }) -except MetadataError as meta_err: - print({"column_metadata_error": str(meta_err)}) -except Exception as exc: - print({"column_metadata_unexpected": str(exc)}) - -pause("Next: Cleanup table") - -# 7) Cleanup: delete the custom table if it exists -print("Cleanup (Metadata):") -if delete_table_at_end: - try: - log_call("client.get_table_info('new_SampleItem')") - info = client.get_table_info("new_SampleItem") - if info: - log_call("client.delete_table('new_SampleItem')") - client.delete_table("new_SampleItem") - print({"table_deleted": True}) - else: - print({"table_deleted": False, "reason": "not found"}) - except Exception as e: - print(f"Delete table failed: {e}") -else: - print({"table_deleted": False, "reason": "user opted to keep table"}) \ No newline at end of file diff --git a/src/PowerPlatform/Dataverse/data/odata.py b/src/PowerPlatform/Dataverse/data/odata.py index 5a7b8fe..7e80cd8 100644 --- a/src/PowerPlatform/Dataverse/data/odata.py +++ b/src/PowerPlatform/Dataverse/data/odata.py @@ -336,23 +336,19 @@ def _update_by_ids(self, table_schema_name: str, ids: List[str], changes: Union[ self._update_multiple(entity_set, table_schema_name, batch) return None - def _delete_multiple(self, logical_name: str, ids: List[str]) -> None: - """Delete records using the collection-bound DeleteMultiple action. - - Parameters - ---------- - logical_name : str - Singular logical entity name. - ids : list[str] - GUIDs for the records to remove. - - Returns - ------- - None - No representation is returned. + def _delete_multiple(self, table_schema_name: str, ids: List[str]) -> None: + """Delete records using the collection-bound ``DeleteMultiple`` action. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param ids: GUIDs for the records to remove. + :type ids: ``list[str]`` + :return: ``None``; the service does not return a representation. + :rtype: ``None`` """ - entity_set = self._entity_set_from_logical(logical_name) - pk_attr = self._primary_id_attr(logical_name) + entity_set = self._entity_set_from_schema_name(table_schema_name) + pk_attr = self._primary_id_attr(table_schema_name) + logical_name = table_schema_name.lower() targets: List[Dict[str, Any]] = [] for rid in ids: targets.append({ @@ -371,8 +367,8 @@ def _delete_async( ) -> str: """Delete many records by GUID list via the ``BulkDelete`` action. - :param logical_name: Logical (singular) entity name. - :type logical_name: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` :param ids: GUIDs of records to delete. :type ids: ``list[str]`` @@ -738,15 +734,17 @@ def _entity_set_from_schema_name(self, table_schema_name: str) -> str: self._logical_primaryid_cache[cache_key] = primary_id_attr return es - def _is_elastic_table(self, logical: str) -> bool: - """Return True when the target table is configured as an elastic table.""" - if not logical: - raise ValueError("logical name required") - cached = self._elastic_table_cache.get(logical) + def _is_elastic_table(self, table_schema_name: str) -> bool: + """Return ``True`` when the target table is elastic.""" + if not table_schema_name: + raise ValueError("table schema name required") + + logical_name = table_schema_name.lower() + cached = self._elastic_table_cache.get(logical_name) if cached is not None: return cached url = f"{self.api}/EntityDefinitions" - logical_escaped = self._escape_odata_quotes(logical) + logical_escaped = self._escape_odata_quotes(logical_name) params = { "$select": "LogicalName,TableType", "$filter": f"LogicalName eq '{logical_escaped}'", @@ -766,7 +764,7 @@ def _is_elastic_table(self, logical: str) -> bool: is_elastic = table_type.strip().lower() == "elastic" else: is_elastic = False - self._elastic_table_cache[logical] = is_elastic + self._elastic_table_cache[logical_name] = is_elastic return is_elastic # ---------------------- Table metadata helpers ----------------------