From 41329274baf0a3528514493773e0076b6febf670 Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 28 Sep 2025 00:00:06 -0500 Subject: [PATCH 1/6] Initial push for policy management on client --- client/python/cli/command/__init__.py | 18 ++ client/python/cli/command/policies.py | 268 +++++++++++++++++++++++ client/python/cli/constants.py | 23 ++ client/python/cli/options/option_tree.py | 44 +++- 4 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 client/python/cli/command/policies.py diff --git a/client/python/cli/command/__init__.py b/client/python/cli/command/__init__.py index 53216f3162..df0337f532 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/cli/command/__init__.py @@ -167,6 +167,24 @@ def options_get(key, f=lambda x: x): command = ProfilesCommand( subcommand, profile_name=options_get(Arguments.PROFILE) ) + elif options.command == Commands.POLICIES: + from cli.command.policies import PoliciesCommand + + subcommand = options_get(f"{Commands.POLICIES}_subcommand") + command = PoliciesCommand( + subcommand, + catalog_name=options_get(Arguments.CATALOG), + namespace=options_get(Arguments.NAMESPACE), + policy_name=options_get(Arguments.POLICY), + policy_file=options_get(Arguments.POLICY_FILE), + policy_type=options_get(Arguments.POLICY_TYPE), + policy_description=options_get(Arguments.POLICY_DESCRIPTION), + target_name=options_get(Arguments.TARGET_NAME), + parameters=Parser.parse_properties(options_get(Arguments.PARAMETERS)), + detach_all=options_get(Arguments.DETACH_ALL), + applicable=options_get(Arguments.APPLICABLE), + attach_target=options_get(Arguments.ATTACH_TARGET), + ) if command is not None: command.validate() diff --git a/client/python/cli/command/policies.py b/client/python/cli/command/policies.py new file mode 100644 index 0000000000..704a9696a1 --- /dev/null +++ b/client/python/cli/command/policies.py @@ -0,0 +1,268 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import json +from dataclasses import dataclass +from typing import List, Optional, Dict + +from cli.command import Command +from cli.constants import Subcommands, Arguments +from polaris.management import PolarisDefaultApi +from polaris.catalog.api_client import ApiClient as CatalogApiClient + +from polaris.catalog.configuration import Configuration as CatalogConfiguration +from polaris.catalog.api.policy_api import PolicyAPI +from polaris.catalog.models.create_policy_request import CreatePolicyRequest +from polaris.catalog.models.update_policy_request import UpdatePolicyRequest +from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget +from polaris.catalog.models.attach_policy_request import AttachPolicyRequest +from polaris.catalog.models.detach_policy_request import DetachPolicyRequest + + + +@dataclass +class PoliciesCommand(Command): + """ + A Command implementation to represent `polaris policies`. + """ + + policies_subcommand: str + catalog_name: str + namespace: str + policy_name: str + policy_file: str + policy_type: str + policy_description: Optional[str] + target_name: Optional[str] + parameters: Optional[Dict[str, str]] + detach_all: Optional[bool] + applicable: Optional[bool] + attach_target: str + + + + def validate(self): + if not self.catalog_name: + raise Exception(f"Missing required argument: {Arguments.CATALOG}") + if self.policies_subcommand in [Subcommands.CREATE, Subcommands.UPDATE]: + if not self.policy_file: + raise Exception(f"Missing required argument: {Arguments.POLICY_FILE}") + if self.policies_subcommand in [Subcommands.ATTACH, Subcommands.DETACH]: + if not self.attach_target: + raise Exception(f"Missing required argument: {Arguments.ATTACH_TARGET}") + + def execute(self, api: PolarisDefaultApi) -> None: + mgmt_config = api.api_client.configuration + catalog_host = mgmt_config.host.replace("/management/v1", "/catalog") + + catalog_config = CatalogConfiguration( + host=catalog_host, + access_token=mgmt_config.access_token, + ) + catalog_config.proxy = mgmt_config.proxy + catalog_config.proxy_headers = mgmt_config.proxy_headers + + catalog_api_client = CatalogApiClient(catalog_config) + policy_api = PolicyAPI(catalog_api_client) + + namespace_str = None + if self.namespace: + namespace_list = self.namespace.split('.') + namespace_str = "\x1F".join(namespace_list) + if self.policies_subcommand == Subcommands.CREATE: + with open(self.policy_file, "r") as f: + policy = json.load(f) + policy_api.create_policy( + prefix=self.catalog_name, + namespace=namespace_str, + create_policy_request=CreatePolicyRequest( + name=self.policy_name, + type=self.policy_type, + description=self.policy_description, + content=json.dumps(policy) + ) + ) + elif self.policies_subcommand == Subcommands.DELETE: + policy_api.drop_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + detach_all=self.detach_all + ) + elif self.policies_subcommand == Subcommands.GET: + print(policy_api.load_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name + ).to_json()) + elif self.policies_subcommand == Subcommands.LIST: + if self.applicable: + api_namespace = None + api_target_name = None + applicable_policies_list = [] + + if self.target_name: + if not self.namespace: + raise Exception("Missing required argument: --namespace when --target-name is set.") + api_namespace = namespace_str + api_target_name = self.target_name + applicable_policies_list = policy_api.get_applicable_policies( + prefix=self.catalog_name, + namespace=api_namespace, + target_name=api_target_name + ).applicable_policies + elif self.namespace: + api_namespace = namespace_str + api_target_name = None + + # Get policies applicable to the namespace (inherited) + applicable_policies_list = policy_api.get_applicable_policies( + prefix=self.catalog_name, + namespace=api_namespace, + target_name=api_target_name + ).applicable_policies + + # Get policies defined directly in the namespace + defined_policies_response = policy_api.list_policies( + prefix=self.catalog_name, + namespace=namespace_str + ).identifiers + + combined_policies = [] + # Add applicable policies (which are full Policy objects) + for policy in applicable_policies_list: + combined_policies.append(policy.to_json()) + + # Add defined policies (which are PolicyIdentifier objects) + # Need to load each defined policy to get its full details + for policy_identifier in defined_policies_response: + # Check if this policy is already in applicable_policies by name + # This is a simplification; a more robust check might compare full policy content + is_duplicate = False + for existing_policy_json in combined_policies: + existing_policy = json.loads(existing_policy_json) + if existing_policy.get("name") == policy_identifier.name: + is_duplicate = True + break + if not is_duplicate: + loaded_policy = policy_api.load_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=policy_identifier.name + ).policy + if loaded_policy: + combined_policies.append(loaded_policy.to_json()) + + for policy_json in combined_policies: + print(policy_json) + return # Exit after printing combined policies + else: + # Catalog-level policies + api_namespace = None + api_target_name = None + applicable_policies_list = policy_api.get_applicable_policies( + prefix=self.catalog_name, + namespace=api_namespace, + target_name=api_target_name + ).applicable_policies + + for policy in applicable_policies_list: + print(policy.to_json()) + elif self.policies_subcommand == Subcommands.UPDATE: + with open(self.policy_file, "r") as f: + policy_document = json.load(f) + + # Fetch the current policy to get its version + loaded_policy_response = policy_api.load_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name + ) + if loaded_policy_response and loaded_policy_response.policy: + current_policy_version = loaded_policy_response.policy.version + else: + raise Exception(f"Could not retrieve current policy version for {self.policy_name}") + + policy_api.update_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + update_policy_request=UpdatePolicyRequest( + description=self.policy_description, + content=json.dumps(policy_document), + current_policy_version=current_policy_version + ) + ) + elif self.policies_subcommand == Subcommands.ATTACH: + target_parts = self.attach_target.split(":") + if len(target_parts) != 2: + raise Exception("Invalid attach target format. Expected 'type:name'") + target_type, target_name = target_parts + + attachment_type = target_type + if target_type in ["table", "view"]: + attachment_type = "table-like" + + attachment_path = [] + if target_type == "catalog": + attachment_path = [] + else: + attachment_path = target_name.split(".") + + policy_api.attach_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + attach_policy_request=AttachPolicyRequest( + target=PolicyAttachmentTarget( + type=attachment_type, + path=attachment_path + ), + parameters=self.parameters + ) + ) + elif self.policies_subcommand == Subcommands.DETACH: + target_parts = self.attach_target.split(":") + if len(target_parts) != 2: + raise Exception("Invalid attach target format. Expected 'type:name'") + target_type, target_name = target_parts + + attachment_type = target_type + if target_type in ["table", "view"]: + attachment_type = "table-like" + + attachment_path = [] + if target_type == "catalog": + attachment_path = [] + else: + attachment_path = target_name.split(".") + + policy_api.detach_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + detach_policy_request=DetachPolicyRequest( + target=PolicyAttachmentTarget( + type=attachment_type, + path=attachment_path + ), + parameters=self.parameters + ) + ) + else: + raise Exception(f"{self.policies_subcommand} is not supported in the CLI") diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 82a4f5d01e..004e196b09 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -88,6 +88,7 @@ class Commands: PRIVILEGES = "privileges" NAMESPACES = "namespaces" PROFILES = "profiles" + POLICIES = "policies" class Subcommands: @@ -109,6 +110,8 @@ class Subcommands: GRANT = "grant" REVOKE = "revoke" ACCESS = "access" + ATTACH = "attach" + DETACH = "detach" class Actions: @@ -186,6 +189,15 @@ class Arguments: CATALOG_EXTERNAL_ID = "catalog_external_id" CATALOG_SIGNING_REGION = "catalog_signing_region" CATALOG_SIGNING_NAME = "catalog_signing_name" + POLICY = "policy" + POLICY_FILE = "policy_file" + POLICY_TYPE = "policy_type" + POLICY_DESCRIPTION = "policy_description" + TARGET_NAME = "target_name" + PARAMETERS = "parameters" + DETACH_ALL = "detach_all" + APPLICABLE = "applicable" + ATTACH_TARGET = "attach_target" class Hints: @@ -364,6 +376,17 @@ class Namespaces: LOCATION = "If specified, the location at which to store the namespace and entities inside it" PARENT = "If specified, list namespaces inside this parent namespace" + class Policies: + POLICY = "The name of a policy" + POLICY_FILE = "The path to a JSON file containing the policy definition" + POLICY_TYPE = "The type of the policy, e.g., 'system.data-compaction'" + POLICY_DESCRIPTION = "An optional description for the policy." + TARGET_NAME = "The name of the target entity (e.g., table name, namespace name)." + PARAMETERS = "Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times." + DETACH_ALL = "When set to true, the policy will be deleted along with all its attached mappings." + APPLICABLE = "When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the namespace." + ATTACH_TARGET = "The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1'" + UNIT_SEPARATOR = chr(0x1F) CLIENT_ID_ENV = "CLIENT_ID" diff --git a/client/python/cli/options/option_tree.py b/client/python/cli/options/option_tree.py index 5b95741f23..118f6af0e9 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/cli/options/option_tree.py @@ -273,5 +273,47 @@ def get_tree() -> List[Option]: Option(Subcommands.UPDATE, input_name=Arguments.PROFILE), Option(Subcommands.GET, input_name=Arguments.PROFILE), Option(Subcommands.LIST), - ]) + ]), + Option(Commands.POLICIES, 'manage policies', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.POLICY_FILE, str, Hints.Policies.POLICY_FILE), + Argument(Arguments.POLICY_TYPE, str, Hints.Policies.POLICY_TYPE), + Argument(Arguments.POLICY_DESCRIPTION, str, Hints.Policies.POLICY_DESCRIPTION), + ], input_name=Arguments.POLICY), + Option(Subcommands.DELETE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.DETACH_ALL, bool, Hints.Policies.DETACH_ALL), + ], input_name=Arguments.POLICY), + Option(Subcommands.GET, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + ], input_name=Arguments.POLICY), + Option(Subcommands.LIST, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.TARGET_NAME, str, Hints.Policies.TARGET_NAME), + Argument(Arguments.APPLICABLE, bool, Hints.Policies.APPLICABLE), + ]), + Option(Subcommands.UPDATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.POLICY_FILE, str, Hints.Policies.POLICY_FILE), + Argument(Arguments.POLICY_DESCRIPTION, str, Hints.Policies.POLICY_DESCRIPTION), + ], input_name=Arguments.POLICY), + Option(Subcommands.ATTACH, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.ATTACH_TARGET, str, Hints.Policies.ATTACH_TARGET), + Argument(Arguments.PARAMETERS, str, Hints.Policies.PARAMETERS, allow_repeats=True), + ], input_name=Arguments.POLICY), + Option(Subcommands.DETACH, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.ATTACH_TARGET, str, Hints.Policies.ATTACH_TARGET), + Argument(Arguments.PARAMETERS, str, Hints.Policies.PARAMETERS, allow_repeats=True), + ], input_name=Arguments.POLICY), + ]), ] From 304ca3ebd55bc005d62f3e6c26e234d18ff5c558 Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 28 Sep 2025 15:24:53 -0500 Subject: [PATCH 2/6] Initial push for policy management on client --- client/python/cli/command/__init__.py | 3 +- client/python/cli/command/namespaces.py | 22 +- client/python/cli/command/policies.py | 157 ++++--------- client/python/cli/command/utils.py | 44 ++++ client/python/cli/constants.py | 2 +- client/python/cli/options/option_tree.py | 1 + client/python/cli/polaris_cli.py | 42 ++++ client/python/integration_tests/conftest.py | 63 ++++- .../integration_tests/test_catalog_apis.py | 218 ++++++++++++++++++ 9 files changed, 423 insertions(+), 129 deletions(-) create mode 100644 client/python/cli/command/utils.py diff --git a/client/python/cli/command/__init__.py b/client/python/cli/command/__init__.py index df0337f532..e081227591 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/cli/command/__init__.py @@ -40,6 +40,7 @@ def options_get(key, f=lambda x: x): set_properties = Parser.parse_properties(options_get(Arguments.SET_PROPERTY)) remove_properties = options_get(Arguments.REMOVE_PROPERTY) catalog_client_scopes = options_get(Arguments.CATALOG_CLIENT_SCOPE) + parameters = Parser.parse_properties(options_get(Arguments.PARAMETERS)), command = None if options.command == Commands.CATALOGS: @@ -180,7 +181,7 @@ def options_get(key, f=lambda x: x): policy_type=options_get(Arguments.POLICY_TYPE), policy_description=options_get(Arguments.POLICY_DESCRIPTION), target_name=options_get(Arguments.TARGET_NAME), - parameters=Parser.parse_properties(options_get(Arguments.PARAMETERS)), + parameters={} if parameters is None else parameters, detach_all=options_get(Arguments.DETACH_ALL), applicable=options_get(Arguments.APPLICABLE), attach_target=options_get(Arguments.ATTACH_TARGET), diff --git a/client/python/cli/command/namespaces.py b/client/python/cli/command/namespaces.py index 3528a6ba12..e3bf193f56 100644 --- a/client/python/cli/command/namespaces.py +++ b/client/python/cli/command/namespaces.py @@ -17,16 +17,16 @@ # under the License. # import json -import re from dataclasses import dataclass from typing import Dict, Optional, List from pydantic import StrictStr from cli.command import Command +from cli.command.utils import get_catalog_api_client from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR from cli.options.option_tree import Argument -from polaris.catalog import IcebergCatalogAPI, CreateNamespaceRequest, ApiClient, Configuration +from polaris.catalog import IcebergCatalogAPI, CreateNamespaceRequest from polaris.management import PolarisDefaultApi @@ -55,23 +55,9 @@ def validate(self): f"Missing required argument: {Argument.to_flag_name(Arguments.CATALOG)}" ) - def _get_catalog_api(self, api: PolarisDefaultApi): - """ - Convert a management API to a catalog API - """ - catalog_host = re.match( - r"(https?://.+)/api/management", api.api_client.configuration.host - ).group(1) - configuration = Configuration( - host=f"{catalog_host}/api/catalog", - username=api.api_client.configuration.username, - password=api.api_client.configuration.password, - access_token=api.api_client.configuration.access_token, - ) - return IcebergCatalogAPI(ApiClient(configuration)) - def execute(self, api: PolarisDefaultApi) -> None: - catalog_api = self._get_catalog_api(api) + catalog_api_client = get_catalog_api_client(api) + catalog_api = IcebergCatalogAPI(catalog_api_client) if self.namespaces_subcommand == Subcommands.CREATE: req_properties = self.properties or {} if self.location: diff --git a/client/python/cli/command/policies.py b/client/python/cli/command/policies.py index 704a9696a1..384b55c611 100644 --- a/client/python/cli/command/policies.py +++ b/client/python/cli/command/policies.py @@ -16,16 +16,15 @@ # specific language governing permissions and limitations # under the License. # +import os import json from dataclasses import dataclass -from typing import List, Optional, Dict - +from typing import Optional, Dict from cli.command import Command -from cli.constants import Subcommands, Arguments +from cli.command.utils import get_catalog_api_client +from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR +from cli.options.option_tree import Argument from polaris.management import PolarisDefaultApi -from polaris.catalog.api_client import ApiClient as CatalogApiClient - -from polaris.catalog.configuration import Configuration as CatalogConfiguration from polaris.catalog.api.policy_api import PolicyAPI from polaris.catalog.models.create_policy_request import CreatePolicyRequest from polaris.catalog.models.update_policy_request import UpdatePolicyRequest @@ -46,7 +45,7 @@ class PoliciesCommand(Command): namespace: str policy_name: str policy_file: str - policy_type: str + policy_type: Optional[str] policy_description: Optional[str] target_name: Optional[str] parameters: Optional[Dict[str, str]] @@ -54,36 +53,35 @@ class PoliciesCommand(Command): applicable: Optional[bool] attach_target: str - - def validate(self): if not self.catalog_name: - raise Exception(f"Missing required argument: {Arguments.CATALOG}") + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.CATALOG)}") if self.policies_subcommand in [Subcommands.CREATE, Subcommands.UPDATE]: if not self.policy_file: - raise Exception(f"Missing required argument: {Arguments.POLICY_FILE}") + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.POLICY_FILE)}") + if not os.path.exists(self.policy_file): + raise Exception(f"Policy file not found: {self.policy_file}") if self.policies_subcommand in [Subcommands.ATTACH, Subcommands.DETACH]: if not self.attach_target: - raise Exception(f"Missing required argument: {Arguments.ATTACH_TARGET}") + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.ATTACH_TARGET)}") + if self.policies_subcommand == Subcommands.LIST and self.applicable and self.target_name: + if not self.namespace: + raise Exception( + f"Missing required argument: {Argument.to_flag_name(Arguments.NAMESPACE)}" + f" when {Argument.to_flag_name(Arguments.TARGET_NAME)} is set." + ) + if self.policies_subcommand == Subcommands.LIST and not self.applicable: + if not self.namespace: + raise Exception( + f"Missing required argument: {Argument.to_flag_name(Arguments.NAMESPACE)}" + f" when listing policies without {Argument.to_flag_name(Arguments.APPLICABLE)} flag." + ) def execute(self, api: PolarisDefaultApi) -> None: - mgmt_config = api.api_client.configuration - catalog_host = mgmt_config.host.replace("/management/v1", "/catalog") - - catalog_config = CatalogConfiguration( - host=catalog_host, - access_token=mgmt_config.access_token, - ) - catalog_config.proxy = mgmt_config.proxy - catalog_config.proxy_headers = mgmt_config.proxy_headers - - catalog_api_client = CatalogApiClient(catalog_config) + catalog_api_client = get_catalog_api_client(api) policy_api = PolicyAPI(catalog_api_client) - namespace_str = None - if self.namespace: - namespace_list = self.namespace.split('.') - namespace_str = "\x1F".join(namespace_list) + namespace_str = self.namespace.replace('.', UNIT_SEPARATOR) if self.namespace else "" if self.policies_subcommand == Subcommands.CREATE: with open(self.policy_file, "r") as f: policy = json.load(f) @@ -112,81 +110,42 @@ def execute(self, api: PolarisDefaultApi) -> None: ).to_json()) elif self.policies_subcommand == Subcommands.LIST: if self.applicable: - api_namespace = None - api_target_name = None applicable_policies_list = [] if self.target_name: - if not self.namespace: - raise Exception("Missing required argument: --namespace when --target-name is set.") - api_namespace = namespace_str - api_target_name = self.target_name + # Table-like-level policies applicable_policies_list = policy_api.get_applicable_policies( prefix=self.catalog_name, - namespace=api_namespace, - target_name=api_target_name + namespace=namespace_str, + target_name=self.target_name, + policy_type=self.policy_type ).applicable_policies elif self.namespace: - api_namespace = namespace_str - api_target_name = None - - # Get policies applicable to the namespace (inherited) + # Namespace-level policies applicable_policies_list = policy_api.get_applicable_policies( prefix=self.catalog_name, - namespace=api_namespace, - target_name=api_target_name + namespace=namespace_str, + policy_type=self.policy_type ).applicable_policies - - # Get policies defined directly in the namespace - defined_policies_response = policy_api.list_policies( - prefix=self.catalog_name, - namespace=namespace_str - ).identifiers - - combined_policies = [] - # Add applicable policies (which are full Policy objects) - for policy in applicable_policies_list: - combined_policies.append(policy.to_json()) - - # Add defined policies (which are PolicyIdentifier objects) - # Need to load each defined policy to get its full details - for policy_identifier in defined_policies_response: - # Check if this policy is already in applicable_policies by name - # This is a simplification; a more robust check might compare full policy content - is_duplicate = False - for existing_policy_json in combined_policies: - existing_policy = json.loads(existing_policy_json) - if existing_policy.get("name") == policy_identifier.name: - is_duplicate = True - break - if not is_duplicate: - loaded_policy = policy_api.load_policy( - prefix=self.catalog_name, - namespace=namespace_str, - policy_name=policy_identifier.name - ).policy - if loaded_policy: - combined_policies.append(loaded_policy.to_json()) - - for policy_json in combined_policies: - print(policy_json) - return # Exit after printing combined policies else: # Catalog-level policies - api_namespace = None - api_target_name = None applicable_policies_list = policy_api.get_applicable_policies( prefix=self.catalog_name, - namespace=api_namespace, - target_name=api_target_name + policy_type=self.policy_type ).applicable_policies - for policy in applicable_policies_list: print(policy.to_json()) + else: + # List all policy identifiers in the namespace + policies_response = policy_api.list_policies( + prefix=self.catalog_name, + namespace=namespace_str, + policy_type=self.policy_type + ).to_json() + print(policies_response) elif self.policies_subcommand == Subcommands.UPDATE: with open(self.policy_file, "r") as f: policy_document = json.load(f) - # Fetch the current policy to get its version loaded_policy_response = policy_api.load_policy( prefix=self.catalog_name, @@ -211,18 +170,9 @@ def execute(self, api: PolarisDefaultApi) -> None: elif self.policies_subcommand == Subcommands.ATTACH: target_parts = self.attach_target.split(":") if len(target_parts) != 2: - raise Exception("Invalid attach target format. Expected 'type:name'") - target_type, target_name = target_parts - - attachment_type = target_type - if target_type in ["table", "view"]: - attachment_type = "table-like" - - attachment_path = [] - if target_type == "catalog": - attachment_path = [] - else: - attachment_path = target_name.split(".") + raise Exception("Invalid attach target format. Expected 'type:path'") + attachment_type, target_path = target_parts + attachment_path = [] if attachment_type == "catalog" else target_path.split(".") policy_api.attach_policy( prefix=self.catalog_name, @@ -233,24 +183,15 @@ def execute(self, api: PolarisDefaultApi) -> None: type=attachment_type, path=attachment_path ), - parameters=self.parameters + parameters=self.parameters if isinstance(self.parameters, dict) else None ) ) elif self.policies_subcommand == Subcommands.DETACH: target_parts = self.attach_target.split(":") if len(target_parts) != 2: - raise Exception("Invalid attach target format. Expected 'type:name'") - target_type, target_name = target_parts - - attachment_type = target_type - if target_type in ["table", "view"]: - attachment_type = "table-like" - - attachment_path = [] - if target_type == "catalog": - attachment_path = [] - else: - attachment_path = target_name.split(".") + raise Exception("Invalid attach target format. Expected 'type:path'") + attachment_type, target_path = target_parts + attachment_path = [] if attachment_type == "catalog" else target_path.split(".") policy_api.detach_policy( prefix=self.catalog_name, @@ -261,7 +202,7 @@ def execute(self, api: PolarisDefaultApi) -> None: type=attachment_type, path=attachment_path ), - parameters=self.parameters + parameters=self.parameters if isinstance(self.parameters, dict) else None ) ) else: diff --git a/client/python/cli/command/utils.py b/client/python/cli/command/utils.py new file mode 100644 index 0000000000..b19b562829 --- /dev/null +++ b/client/python/cli/command/utils.py @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import re + +from polaris.catalog.api_client import ApiClient +from polaris.catalog.configuration import Configuration +from polaris.management import PolarisDefaultApi + + +def get_catalog_api_client(api: PolarisDefaultApi) -> ApiClient: + """ + Convert a management API to a catalog API client + """ + mgmt_config = api.api_client.configuration + catalog_host = re.sub(r"/api/management(?:/v1)?", "/api/catalog", mgmt_config.host) + configuration = Configuration( + host=catalog_host, + username=mgmt_config.username, + password=mgmt_config.password, + access_token=mgmt_config.access_token, + ) + + if hasattr(mgmt_config, "proxy"): + configuration.proxy = mgmt_config.proxy + if hasattr(mgmt_config, "proxy_headers"): + configuration.proxy_headers = mgmt_config.proxy_headers + + return ApiClient(configuration) diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 004e196b09..8f097afd8e 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -384,7 +384,7 @@ class Policies: TARGET_NAME = "The name of the target entity (e.g., table name, namespace name)." PARAMETERS = "Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times." DETACH_ALL = "When set to true, the policy will be deleted along with all its attached mappings." - APPLICABLE = "When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the namespace." + APPLICABLE = "When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the target." ATTACH_TARGET = "The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1'" diff --git a/client/python/cli/options/option_tree.py b/client/python/cli/options/option_tree.py index 118f6af0e9..c10925cded 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/cli/options/option_tree.py @@ -296,6 +296,7 @@ def get_tree() -> List[Option]: Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), Argument(Arguments.TARGET_NAME, str, Hints.Policies.TARGET_NAME), Argument(Arguments.APPLICABLE, bool, Hints.Policies.APPLICABLE), + Argument(Arguments.POLICY_TYPE, str, Hints.Policies.POLICY_TYPE), ]), Option(Subcommands.UPDATE, args=[ Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), diff --git a/client/python/cli/polaris_cli.py b/client/python/cli/polaris_cli.py index 63268462ee..a7be1bdb18 100644 --- a/client/python/cli/polaris_cli.py +++ b/client/python/cli/polaris_cli.py @@ -58,8 +58,50 @@ class PolarisCli: # Can be enabled if the client is able to authenticate directly without first fetching a token DIRECT_AUTHENTICATION_ENABLED = False + @staticmethod + def _patch_generated_models() -> None: + """ + The OpenAPI generator creates an `api_client` that dynamically looks up + model classes from the `polaris.catalog.models` module using `getattr()`. + For example, when a response for a `create_policy` call is received, the + deserializer tries to find the `LoadPolicyResponse` class by looking for + `polaris.catalog.models.LoadPolicyResponse`. + + However, the generator fails to add the necessary `import` statements + to the `polaris/catalog/models/__init__.py` file. This means that even + though the model files exist (e.g., `load_policy_response.py`), the classes + are not part of the `polaris.catalog.models` namespace. + + This method works around the bug in the generated code without modifying + the source files. It runs once per CLI execution, before any commands, and + manually injects the missing response-side model classes into the + `polaris.catalog.models` namespace, allowing the deserializer to find them. + """ + import polaris.catalog.models + from polaris.catalog.models.applicable_policy import ApplicablePolicy + from polaris.catalog.models.get_applicable_policies_response import GetApplicablePoliciesResponse + from polaris.catalog.models.list_policies_response import ListPoliciesResponse + from polaris.catalog.models.load_policy_response import LoadPolicyResponse + from polaris.catalog.models.policy import Policy + from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget + from polaris.catalog.models.policy_identifier import PolicyIdentifier + + models_to_patch = { + "ApplicablePolicy": ApplicablePolicy, + "GetApplicablePoliciesResponse": GetApplicablePoliciesResponse, + "ListPoliciesResponse": ListPoliciesResponse, + "LoadPolicyResponse": LoadPolicyResponse, + "Policy": Policy, + "PolicyAttachmentTarget": PolicyAttachmentTarget, + "PolicyIdentifier": PolicyIdentifier, + } + + for name, model_class in models_to_patch.items(): + setattr(polaris.catalog.models, name, model_class) + @staticmethod def execute(args=None): + PolarisCli._patch_generated_models() options = Parser.parse(args) if options.command == Commands.PROFILES: from cli.command import Command diff --git a/client/python/integration_tests/conftest.py b/client/python/integration_tests/conftest.py index 5ad5165a00..824c64d77e 100644 --- a/client/python/integration_tests/conftest.py +++ b/client/python/integration_tests/conftest.py @@ -28,8 +28,10 @@ from polaris.catalog import OAuthTokenResponse from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API +from polaris.catalog.api.policy_api import PolicyAPI from polaris.catalog.api_client import ApiClient as CatalogApiClient from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration + from polaris.management import ( Catalog, ApiClient, @@ -195,13 +197,28 @@ def test_catalog_client( return IcebergCatalogAPI( CatalogApiClient( - Configuration( + CatalogApiClientConfiguration( access_token=test_principal_token.access_token, host=polaris_catalog_url ) ) ) +@pytest.fixture +def test_policy_api( + polaris_catalog_url: str, + test_principal_token: OAuthTokenResponse, +) -> PolicyAPI: + return PolicyAPI( + CatalogApiClient( + CatalogApiClientConfiguration( + access_token=test_principal_token.access_token, + host=polaris_catalog_url, + ) + ) + ) + + @pytest.fixture def test_catalog( management_client: PolarisDefaultApi, @@ -354,3 +371,47 @@ def clear_namespace( def format_namespace(namespace: List[str]) -> str: return codecs.decode("1F", "hex").decode("UTF-8").join(namespace) + + +@pytest.fixture(scope="session", autouse=True) +def _patch_generated_models() -> None: + """ + The OpenAPI generator creates an `api_client` that dynamically looks up + model classes from the `polaris.catalog.models` module using `getattr()`. + For example, when a response for a `create_policy` call is received, the + deserializer tries to find the `LoadPolicyResponse` class by looking for + `polaris.catalog.models.LoadPolicyResponse`. + + However, the generator fails to add the necessary `import` statements + to the `polaris/catalog/models/__init__.py` file. This means that even + though the model files exist (e.g., `load_policy_response.py`), the classes + are not part of the `polaris.catalog.models` namespace. + + This fixture works around the bug in the generated code without modifying + the source files. It runs once per test session, before any tests, and + manually injects the missing response-side model classes into the + `polaris.catalog.models` namespace, allowing the deserializer to find them. + """ + import polaris.catalog.models + from polaris.catalog.models.applicable_policy import ApplicablePolicy + from polaris.catalog.models.get_applicable_policies_response import ( + GetApplicablePoliciesResponse, + ) + from polaris.catalog.models.list_policies_response import ListPoliciesResponse + from polaris.catalog.models.load_policy_response import LoadPolicyResponse + from polaris.catalog.models.policy import Policy + from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget + from polaris.catalog.models.policy_identifier import PolicyIdentifier + + models_to_patch = { + "ApplicablePolicy": ApplicablePolicy, + "GetApplicablePoliciesResponse": GetApplicablePoliciesResponse, + "ListPoliciesResponse": ListPoliciesResponse, + "LoadPolicyResponse": LoadPolicyResponse, + "Policy": Policy, + "PolicyAttachmentTarget": PolicyAttachmentTarget, + "PolicyIdentifier": PolicyIdentifier, + } + + for name, model_class in models_to_patch.items(): + setattr(polaris.catalog.models, name, model_class) diff --git a/client/python/integration_tests/test_catalog_apis.py b/client/python/integration_tests/test_catalog_apis.py index 44bc162640..064073e0ab 100644 --- a/client/python/integration_tests/test_catalog_apis.py +++ b/client/python/integration_tests/test_catalog_apis.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json import os.path import time @@ -28,6 +29,13 @@ TableIdentifier, IcebergCatalogAPI, ) +from polaris.catalog.models.attach_policy_request import AttachPolicyRequest +from polaris.catalog.models.create_policy_request import CreatePolicyRequest +from polaris.catalog.models.detach_policy_request import DetachPolicyRequest +from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget +from polaris.catalog.models.update_policy_request import UpdatePolicyRequest +from polaris.catalog.exceptions import NotFoundException +from polaris.catalog.api.policy_api import PolicyAPI from pyiceberg.schema import Schema from polaris.management import Catalog from pyiceberg.catalog import Catalog as PyIcebergCatalog @@ -166,3 +174,213 @@ def test_drop_table_purge( if os.path.exists(metadata_location): pytest.fail(f"Metadata file {metadata_location} still exists after 60 seconds.") + + +def test_policies( + test_catalog: Catalog, + test_catalog_client: IcebergCatalogAPI, + test_policy_api: PolicyAPI, + test_table_schema: Schema, +) -> None: + # Resource identifiers + namespace_name = "POLNS1" + sub_namespace = "POLNS2" + sub_namespace_path = [namespace_name, sub_namespace] + policy_name = "test_policy" + table_name = "test_table_for_policy" + table_path = sub_namespace_path + [table_name] + + policy_content = { + "version": "2025-02-03", + "enable": True, + "config": {"target_file_size_bytes": 134217728}, + } + policy_type = "system.data-compaction" + policy_description = "A test policy" + + # Create resources + test_catalog_client.create_namespace( + prefix=test_catalog.name, + create_namespace_request=CreateNamespaceRequest(namespace=[namespace_name]), + ) + test_catalog_client.create_namespace( + prefix=test_catalog.name, + create_namespace_request=CreateNamespaceRequest(namespace=sub_namespace_path), + ) + test_catalog_client.create_table( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + create_table_request=CreateTableRequest( + name=table_name, + var_schema=ModelSchema.from_dict(test_table_schema.model_dump()), + ), + ) + test_policy_api.create_policy( + prefix=test_catalog.name, + namespace=namespace_name, + create_policy_request=CreatePolicyRequest( + name=policy_name, + type=policy_type, + description=policy_description, + content=json.dumps(policy_content), + ), + ) + + try: + # GET + loaded_policy = test_policy_api.load_policy( + prefix=test_catalog.name, namespace=namespace_name, policy_name=policy_name + ) + assert loaded_policy.policy.name == policy_name + assert loaded_policy.policy.policy_type == policy_type + assert loaded_policy.policy.description == policy_description + assert json.loads(loaded_policy.policy.content) == policy_content + + # LIST + policies = test_policy_api.list_policies( + prefix=test_catalog.name, namespace=namespace_name + ) + assert len(policies.identifiers) == 1 + assert policies.identifiers[0].name == policy_name + assert policies.identifiers[0].namespace == [namespace_name] + + # UPDATE + updated_policy_content = { + "version": "2025-02-03", + "enable": False, + "config": {"target_file_size_bytes": 134217728}, + } + updated_policy_description = "An updated test policy" + test_policy_api.update_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + update_policy_request=UpdatePolicyRequest( + description=updated_policy_description, + content=json.dumps(updated_policy_content), + current_policy_version=loaded_policy.policy.version, + ), + ) + + # GET after UPDATE + updated_policy = test_policy_api.load_policy( + prefix=test_catalog.name, namespace=namespace_name, policy_name=policy_name + ) + assert updated_policy.policy.description == updated_policy_description + assert json.loads(updated_policy.policy.content) == updated_policy_content + + # ATTACH to namespace + test_policy_api.attach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + attach_policy_request=AttachPolicyRequest( + target=PolicyAttachmentTarget(type="namespace", path=sub_namespace_path) + ), + ) + + # GET APPLICABLE on namespace + applicable_policies = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, namespace=format_namespace(sub_namespace_path) + ) + assert len(applicable_policies.applicable_policies) == 1 + assert applicable_policies.applicable_policies[0].name == policy_name + + # DETACH from namespace + test_policy_api.detach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_policy_request=DetachPolicyRequest( + target=PolicyAttachmentTarget(type="namespace", path=sub_namespace_path) + ), + ) + + # GET APPLICABLE on namespace after DETACH + applicable_policies_after_detach = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, namespace=format_namespace(sub_namespace_path) + ) + assert len(applicable_policies_after_detach.applicable_policies) == 0 + + # ATTACH to table + test_policy_api.attach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + attach_policy_request=AttachPolicyRequest( + target=PolicyAttachmentTarget(type="table-like", path=table_path) + ), + ) + + # GET APPLICABLE on table + applicable_for_table = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + target_name=table_name, + ) + assert len(applicable_for_table.applicable_policies) == 1 + assert applicable_for_table.applicable_policies[0].name == policy_name + + # DETACH from table + test_policy_api.detach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_policy_request=DetachPolicyRequest( + target=PolicyAttachmentTarget(type="table-like", path=table_path) + ), + ) + + # GET APPLICABLE on table after DETACH + applicable_for_table_after_detach = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + target_name=table_name, + ) + assert len(applicable_for_table_after_detach.applicable_policies) == 0 + + # DELETE + test_policy_api.drop_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_all=True, + ) + + # LIST after DELETE + policies_after_delete = test_policy_api.list_policies( + prefix=test_catalog.name, namespace=namespace_name + ) + assert len(policies_after_delete.identifiers) == 0 + + finally: + # Cleanup + try: + test_policy_api.drop_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_all=True, + ) + except NotFoundException: + pass + try: + test_catalog_client.drop_table( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + table=table_name, + ) + except NotFoundException: + pass + try: + test_catalog_client.drop_namespace( + prefix=test_catalog.name, namespace=format_namespace(sub_namespace_path) + ) + except NotFoundException: + pass + try: + test_catalog_client.drop_namespace( + prefix=test_catalog.name, namespace=namespace_name + ) + except NotFoundException: + pass From cc41a7697afaa83d45cae98f47c9a4a8953dfee4 Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 28 Sep 2025 17:16:43 -0500 Subject: [PATCH 3/6] Initial push for policy management on client --- client/python/cli/command/policies.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/client/python/cli/command/policies.py b/client/python/cli/command/policies.py index 384b55c611..f19cc3fb51 100644 --- a/client/python/cli/command/policies.py +++ b/client/python/cli/command/policies.py @@ -59,8 +59,6 @@ def validate(self): if self.policies_subcommand in [Subcommands.CREATE, Subcommands.UPDATE]: if not self.policy_file: raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.POLICY_FILE)}") - if not os.path.exists(self.policy_file): - raise Exception(f"Policy file not found: {self.policy_file}") if self.policies_subcommand in [Subcommands.ATTACH, Subcommands.DETACH]: if not self.attach_target: raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.ATTACH_TARGET)}") @@ -113,7 +111,7 @@ def execute(self, api: PolarisDefaultApi) -> None: applicable_policies_list = [] if self.target_name: - # Table-like-level policies + # Table-like level policies applicable_policies_list = policy_api.get_applicable_policies( prefix=self.catalog_name, namespace=namespace_str, @@ -121,14 +119,14 @@ def execute(self, api: PolarisDefaultApi) -> None: policy_type=self.policy_type ).applicable_policies elif self.namespace: - # Namespace-level policies + # Namespace level policies applicable_policies_list = policy_api.get_applicable_policies( prefix=self.catalog_name, namespace=namespace_str, policy_type=self.policy_type ).applicable_policies else: - # Catalog-level policies + # Catalog level policies applicable_policies_list = policy_api.get_applicable_policies( prefix=self.catalog_name, policy_type=self.policy_type From 4d9fb73249795415fac08877381735db28c59a4e Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 28 Sep 2025 17:22:06 -0500 Subject: [PATCH 4/6] Initial push for policy management on client --- client/python/integration_tests/test_catalog_apis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/python/integration_tests/test_catalog_apis.py b/client/python/integration_tests/test_catalog_apis.py index 064073e0ab..0f73e9fb0d 100644 --- a/client/python/integration_tests/test_catalog_apis.py +++ b/client/python/integration_tests/test_catalog_apis.py @@ -183,8 +183,8 @@ def test_policies( test_table_schema: Schema, ) -> None: # Resource identifiers - namespace_name = "POLNS1" - sub_namespace = "POLNS2" + namespace_name = "POLICY_NS1" + sub_namespace = "POLICY_NS2" sub_namespace_path = [namespace_name, sub_namespace] policy_name = "test_policy" table_name = "test_table_for_policy" From f04ec88087cf978bffe454c85457a105891a83ec Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 28 Sep 2025 17:47:30 -0500 Subject: [PATCH 5/6] Initial push for policy management on client --- .../unreleased/command-line-interface.md | 189 +++++++++++++++++- 1 file changed, 188 insertions(+), 1 deletion(-) diff --git a/site/content/in-dev/unreleased/command-line-interface.md b/site/content/in-dev/unreleased/command-line-interface.md index c191fe960c..a2aee31194 100644 --- a/site/content/in-dev/unreleased/command-line-interface.md +++ b/site/content/in-dev/unreleased/command-line-interface.md @@ -37,6 +37,7 @@ options: --client-secret --access-token --profile +--proxy ``` `COMMAND` must be one of the following: @@ -47,7 +48,8 @@ options: 5. namespaces 6. privileges 7. profiles -8. repair +8. policies +9. repair Each _command_ supports several _subcommands_, and some _subcommands_ have _actions_ that come after the subcommand in turn. Finally, _arguments_ follow to form a full invocation. Within a set of named arguments at the end of an invocation ordering is generally not important. Many invocations also have a required positional argument of the type that the _command_ refers to. Again, the ordering of this positional argument relative to named arguments is not important. @@ -60,6 +62,7 @@ polaris catalogs update --property foo=bar some_other_catalog polaris catalogs update another_catalog --property k=v polaris privileges namespace grant --namespace some.schema --catalog fourth_catalog --catalog-role some_catalog_role TABLE_READ_DATA polaris profiles list +polaris policies list --catalog some_catalog --namespace some.schema polaris repair ``` @@ -1214,6 +1217,190 @@ options: polaris profiles update dev ``` +### Policies + +The `policies` command is used to manage policies within Polaris. + +`policies` supports the following subcommands: + +1. attach +2. create +3. delete +4. detach +5. get +6. list +7. update + +#### attach + +The `attach` subcommand is used to create a mapping between a policy and a resource entity. + +``` +input: polaris policies attach --help +options: + attach + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --attach-target The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1' + --parameters Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies attach --catalog some_catalog --namespace some.schema --attach-target "namespace:some.schema " some_policy + +polaris policies attach --catalog some_catalog --namespace some.schema --attach-target "table-like:some.schema.t" some_table_policy +``` + +#### create + +The `create` subcommand is used to create a policy. + +``` +input: polaris policies create --help +options: + create + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --policy-file The path to a JSON file containing the policy definition + --policy-type The type of the policy, e.g., 'system.data-compaction' + --policy-description An optional description for the policy. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies create --catalog some_catalog --namespace some.schema --policy-file some_policy.json --policy-type system.data-compaction some_policy + +polaris policies create --catalog some_catalog --namespace some.schema --policy-file some_snapshot_expiry_policy.json --policy-type system.snapshot-expiry some_snapshot_expiry_policy +``` + +#### delete + +The `delete` subcommand is used to delete a policy. + +``` +input: polaris policies delete --help +options: + delete + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --detach-all When set to true, the policy will be deleted along with all its attached mappings. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies delete --catalog some_catalog --namespace some.schema some_policy + +polaris policies delete --catalog some_catalog --namespace some.schema --detach-all some_policy +``` + +#### detach + +The `detach` subcommand is used to remove a mapping between a policy and a target entity + +``` +input: polaris policies detach --help +options: + detach + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --attach-target The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1' + --parameters Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies detach --catalog some_catalog --namespace some.schema --attach-target "namespace:some.schema" some_policy + +polaris policies detach --catalog some_catalog --namespace some.schema --attach-target "catalog:some_catalog" some_policy +``` + +#### get + +The `get` subcommand is used to load a policy from the catalog. + +``` +input: polaris policies get --help +options: + get + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies get --catalog some_catalog --namespace some.schema some_policy +``` + +#### list + +The `list` subcommand is used to get all policy identifiers under this namespace and all applicable policies for a specified entity. + +``` +input: polaris policies list --help +options: + list + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --target-name The name of the target entity (e.g., table name, namespace name). + --applicable When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the target. + --policy-type The type of the policy, e.g., 'system.data-compaction' +``` + +##### Examples + +``` +polaris policies list --catalog some_catalog + +polaris policies list --catalog some_catalog --applicable +``` + +#### update + +The `update` subcommand is used to update a policy. + +``` +input: polaris policies update --help +options: + update + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --policy-file The path to a JSON file containing the policy definition + --policy-description An optional description for the policy. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies update --catalog some_catalog --namespace some.schema --policy-file my_updated_policy.json my_policy + +polaris policies update --catalog some_catalog --namespace some.schema --policy-file my_updated_policy.json --policy-description "Updated policy description" my_policy +``` + ### repair The `repair` command is a bash script wrapper used to regenerate Python client code and update necessary dependencies, ensuring the Polaris client remains up-to-date and functional. **Please note that this command does not support any options and its usage information is not available via a `--help` flag.** From fe84057a10f6bd695f2b3b072a7fd8fdc376e3b4 Mon Sep 17 00:00:00 2001 From: Yong Date: Wed, 1 Oct 2025 23:37:54 -0500 Subject: [PATCH 6/6] Split ATTACH_TARGET into ATTACHMENT_TYPE and ATTACHMENT_PATH --- client/python/cli/command/__init__.py | 3 +- client/python/cli/command/policies.py | 29 ++++++++----------- client/python/cli/constants.py | 6 ++-- client/python/cli/options/option_tree.py | 6 ++-- .../unreleased/command-line-interface.md | 14 +++++---- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/client/python/cli/command/__init__.py b/client/python/cli/command/__init__.py index 51d5ae9ef1..76cc0bca3d 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/cli/command/__init__.py @@ -186,7 +186,8 @@ def options_get(key, f=lambda x: x): parameters={} if parameters is None else parameters, detach_all=options_get(Arguments.DETACH_ALL), applicable=options_get(Arguments.APPLICABLE), - attach_target=options_get(Arguments.ATTACH_TARGET), + attachment_type=options_get(Arguments.ATTACHMENT_TYPE), + attachment_path=options_get(Arguments.ATTACHMENT_PATH), ) if command is not None: diff --git a/client/python/cli/command/policies.py b/client/python/cli/command/policies.py index f19cc3fb51..78ada1768a 100644 --- a/client/python/cli/command/policies.py +++ b/client/python/cli/command/policies.py @@ -51,7 +51,8 @@ class PoliciesCommand(Command): parameters: Optional[Dict[str, str]] detach_all: Optional[bool] applicable: Optional[bool] - attach_target: str + attachment_type: Optional[str] + attachment_path: Optional[str] def validate(self): if not self.catalog_name: @@ -60,8 +61,10 @@ def validate(self): if not self.policy_file: raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.POLICY_FILE)}") if self.policies_subcommand in [Subcommands.ATTACH, Subcommands.DETACH]: - if not self.attach_target: - raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.ATTACH_TARGET)}") + if not self.attachment_type: + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.ATTACHMENT_TYPE)}") + if self.attachment_type != 'catalog' and not self.attachment_path: + raise Exception(f"'{Argument.to_flag_name(Arguments.ATTACHMENT_PATH)}' is required when attachment type is not 'catalog'") if self.policies_subcommand == Subcommands.LIST and self.applicable and self.target_name: if not self.namespace: raise Exception( @@ -166,11 +169,7 @@ def execute(self, api: PolarisDefaultApi) -> None: ) ) elif self.policies_subcommand == Subcommands.ATTACH: - target_parts = self.attach_target.split(":") - if len(target_parts) != 2: - raise Exception("Invalid attach target format. Expected 'type:path'") - attachment_type, target_path = target_parts - attachment_path = [] if attachment_type == "catalog" else target_path.split(".") + attachment_path_list = [] if self.attachment_type == "catalog" else self.attachment_path.split('.') policy_api.attach_policy( prefix=self.catalog_name, @@ -178,18 +177,14 @@ def execute(self, api: PolarisDefaultApi) -> None: policy_name=self.policy_name, attach_policy_request=AttachPolicyRequest( target=PolicyAttachmentTarget( - type=attachment_type, - path=attachment_path + type=self.attachment_type, + path=attachment_path_list ), parameters=self.parameters if isinstance(self.parameters, dict) else None ) ) elif self.policies_subcommand == Subcommands.DETACH: - target_parts = self.attach_target.split(":") - if len(target_parts) != 2: - raise Exception("Invalid attach target format. Expected 'type:path'") - attachment_type, target_path = target_parts - attachment_path = [] if attachment_type == "catalog" else target_path.split(".") + attachment_path_list = [] if self.attachment_type == "catalog" else self.attachment_path.split('.') policy_api.detach_policy( prefix=self.catalog_name, @@ -197,8 +192,8 @@ def execute(self, api: PolarisDefaultApi) -> None: policy_name=self.policy_name, detach_policy_request=DetachPolicyRequest( target=PolicyAttachmentTarget( - type=attachment_type, - path=attachment_path + type=self.attachment_type, + path=attachment_path_list ), parameters=self.parameters if isinstance(self.parameters, dict) else None ) diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 641a57e0a5..98d191e1ce 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -200,7 +200,8 @@ class Arguments: PARAMETERS = "parameters" DETACH_ALL = "detach_all" APPLICABLE = "applicable" - ATTACH_TARGET = "attach_target" + ATTACHMENT_TYPE = "attachment_type" + ATTACHMENT_PATH = "attachment_path" class Hints: @@ -392,7 +393,8 @@ class Policies: PARAMETERS = "Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times." DETACH_ALL = "When set to true, the policy will be deleted along with all its attached mappings." APPLICABLE = "When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the target." - ATTACH_TARGET = "The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1'" + ATTACHMENT_TYPE = "The type of entity to attach the policy to, e.g., 'catalog', 'namespace', or table-like." + ATTACHMENT_PATH = "The path of the entity to attach the policy to, e.g., 'ns1.tb1'. Not required for catalog-level attachment." UNIT_SEPARATOR = chr(0x1F) diff --git a/client/python/cli/options/option_tree.py b/client/python/cli/options/option_tree.py index f1ba7f029f..78f15b96e3 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/cli/options/option_tree.py @@ -311,13 +311,15 @@ def get_tree() -> List[Option]: Option(Subcommands.ATTACH, args=[ Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), - Argument(Arguments.ATTACH_TARGET, str, Hints.Policies.ATTACH_TARGET), + Argument(Arguments.ATTACHMENT_TYPE, str, Hints.Policies.ATTACHMENT_TYPE), + Argument(Arguments.ATTACHMENT_PATH, str, Hints.Policies.ATTACHMENT_PATH), Argument(Arguments.PARAMETERS, str, Hints.Policies.PARAMETERS, allow_repeats=True), ], input_name=Arguments.POLICY), Option(Subcommands.DETACH, args=[ Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), - Argument(Arguments.ATTACH_TARGET, str, Hints.Policies.ATTACH_TARGET), + Argument(Arguments.ATTACHMENT_TYPE, str, Hints.Policies.ATTACHMENT_TYPE), + Argument(Arguments.ATTACHMENT_PATH, str, Hints.Policies.ATTACHMENT_PATH), Argument(Arguments.PARAMETERS, str, Hints.Policies.PARAMETERS, allow_repeats=True), ], input_name=Arguments.POLICY), ]), diff --git a/site/content/in-dev/unreleased/command-line-interface.md b/site/content/in-dev/unreleased/command-line-interface.md index 2123d3d8b5..ab0452703a 100644 --- a/site/content/in-dev/unreleased/command-line-interface.md +++ b/site/content/in-dev/unreleased/command-line-interface.md @@ -1246,7 +1246,8 @@ options: Named arguments: --catalog The name of an existing catalog --namespace A period-delimited namespace - --attach-target The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1' + --attachment-type The type of entity to attach the policy to, e.g., 'catalog', 'namespace', or table-like. + --attachment-path The path of the entity to attach the policy to, e.g., 'ns1.tb1'. Not required for catalog-level attachment. --parameters Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times. Positional arguments: policy @@ -1255,9 +1256,9 @@ options: ##### Examples ``` -polaris policies attach --catalog some_catalog --namespace some.schema --attach-target "namespace:some.schema " some_policy +polaris policies attach --catalog some_catalog --namespace some.schema --attachment-type namespace --attachment-path some.schema some_policy -polaris policies attach --catalog some_catalog --namespace some.schema --attach-target "table-like:some.schema.t" some_table_policy +polaris policies attach --catalog some_catalog --namespace some.schema --attachment-type table-like --attachment-path some.schema.t some_table_policy ``` #### create @@ -1321,7 +1322,8 @@ options: Named arguments: --catalog The name of an existing catalog --namespace A period-delimited namespace - --attach-target The target to attach the policy to, e.g. 'namespace:ns1' or 'table:ns1.tb1' + --attachment-type The type of entity to attach the policy to, e.g., 'catalog', 'namespace', or table-like. + --attachment-path The path of the entity to attach the policy to, e.g., 'ns1.tb1'. Not required for catalog-level attachment. --parameters Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times. Positional arguments: policy @@ -1330,9 +1332,9 @@ options: ##### Examples ``` -polaris policies detach --catalog some_catalog --namespace some.schema --attach-target "namespace:some.schema" some_policy +polaris policies detach --catalog some_catalog --namespace some.schema --attachment-type namespace --attachment-path some.schema some_policy -polaris policies detach --catalog some_catalog --namespace some.schema --attach-target "catalog:some_catalog" some_policy +polaris policies detach --catalog some_catalog --namespace some.schema --attachment-type catalog --attachment-path some_catalog some_policy ``` #### get