diff --git a/client/python/cli/command/__init__.py b/client/python/cli/command/__init__.py index d9c29f2153..76cc0bca3d 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/cli/command/__init__.py @@ -40,6 +40,7 @@ def options_get(key, f=lambda x: x): set_properties = Parser.parse_properties(options_get(Arguments.SET_PROPERTY)) remove_properties = options_get(Arguments.REMOVE_PROPERTY) catalog_client_scopes = options_get(Arguments.CATALOG_CLIENT_SCOPE) + parameters = Parser.parse_properties(options_get(Arguments.PARAMETERS)), command = None if options.command == Commands.CATALOGS: @@ -169,6 +170,25 @@ def options_get(key, f=lambda x: x): command = ProfilesCommand( subcommand, profile_name=options_get(Arguments.PROFILE) ) + elif options.command == Commands.POLICIES: + from cli.command.policies import PoliciesCommand + + subcommand = options_get(f"{Commands.POLICIES}_subcommand") + command = PoliciesCommand( + subcommand, + catalog_name=options_get(Arguments.CATALOG), + namespace=options_get(Arguments.NAMESPACE), + policy_name=options_get(Arguments.POLICY), + policy_file=options_get(Arguments.POLICY_FILE), + policy_type=options_get(Arguments.POLICY_TYPE), + policy_description=options_get(Arguments.POLICY_DESCRIPTION), + target_name=options_get(Arguments.TARGET_NAME), + parameters={} if parameters is None else parameters, + detach_all=options_get(Arguments.DETACH_ALL), + applicable=options_get(Arguments.APPLICABLE), + attachment_type=options_get(Arguments.ATTACHMENT_TYPE), + attachment_path=options_get(Arguments.ATTACHMENT_PATH), + ) if command is not None: command.validate() diff --git a/client/python/cli/command/namespaces.py b/client/python/cli/command/namespaces.py index 3528a6ba12..e3bf193f56 100644 --- a/client/python/cli/command/namespaces.py +++ b/client/python/cli/command/namespaces.py @@ -17,16 +17,16 @@ # under the License. # import json -import re from dataclasses import dataclass from typing import Dict, Optional, List from pydantic import StrictStr from cli.command import Command +from cli.command.utils import get_catalog_api_client from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR from cli.options.option_tree import Argument -from polaris.catalog import IcebergCatalogAPI, CreateNamespaceRequest, ApiClient, Configuration +from polaris.catalog import IcebergCatalogAPI, CreateNamespaceRequest from polaris.management import PolarisDefaultApi @@ -55,23 +55,9 @@ def validate(self): f"Missing required argument: {Argument.to_flag_name(Arguments.CATALOG)}" ) - def _get_catalog_api(self, api: PolarisDefaultApi): - """ - Convert a management API to a catalog API - """ - catalog_host = re.match( - r"(https?://.+)/api/management", api.api_client.configuration.host - ).group(1) - configuration = Configuration( - host=f"{catalog_host}/api/catalog", - username=api.api_client.configuration.username, - password=api.api_client.configuration.password, - access_token=api.api_client.configuration.access_token, - ) - return IcebergCatalogAPI(ApiClient(configuration)) - def execute(self, api: PolarisDefaultApi) -> None: - catalog_api = self._get_catalog_api(api) + catalog_api_client = get_catalog_api_client(api) + catalog_api = IcebergCatalogAPI(catalog_api_client) if self.namespaces_subcommand == Subcommands.CREATE: req_properties = self.properties or {} if self.location: diff --git a/client/python/cli/command/policies.py b/client/python/cli/command/policies.py new file mode 100644 index 0000000000..78ada1768a --- /dev/null +++ b/client/python/cli/command/policies.py @@ -0,0 +1,202 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os +import json +from dataclasses import dataclass +from typing import Optional, Dict +from cli.command import Command +from cli.command.utils import get_catalog_api_client +from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR +from cli.options.option_tree import Argument +from polaris.management import PolarisDefaultApi +from polaris.catalog.api.policy_api import PolicyAPI +from polaris.catalog.models.create_policy_request import CreatePolicyRequest +from polaris.catalog.models.update_policy_request import UpdatePolicyRequest +from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget +from polaris.catalog.models.attach_policy_request import AttachPolicyRequest +from polaris.catalog.models.detach_policy_request import DetachPolicyRequest + + + +@dataclass +class PoliciesCommand(Command): + """ + A Command implementation to represent `polaris policies`. + """ + + policies_subcommand: str + catalog_name: str + namespace: str + policy_name: str + policy_file: str + policy_type: Optional[str] + policy_description: Optional[str] + target_name: Optional[str] + parameters: Optional[Dict[str, str]] + detach_all: Optional[bool] + applicable: Optional[bool] + attachment_type: Optional[str] + attachment_path: Optional[str] + + def validate(self): + if not self.catalog_name: + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.CATALOG)}") + if self.policies_subcommand in [Subcommands.CREATE, Subcommands.UPDATE]: + if not self.policy_file: + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.POLICY_FILE)}") + if self.policies_subcommand in [Subcommands.ATTACH, Subcommands.DETACH]: + if not self.attachment_type: + raise Exception(f"Missing required argument: {Argument.to_flag_name(Arguments.ATTACHMENT_TYPE)}") + if self.attachment_type != 'catalog' and not self.attachment_path: + raise Exception(f"'{Argument.to_flag_name(Arguments.ATTACHMENT_PATH)}' is required when attachment type is not 'catalog'") + if self.policies_subcommand == Subcommands.LIST and self.applicable and self.target_name: + if not self.namespace: + raise Exception( + f"Missing required argument: {Argument.to_flag_name(Arguments.NAMESPACE)}" + f" when {Argument.to_flag_name(Arguments.TARGET_NAME)} is set." + ) + if self.policies_subcommand == Subcommands.LIST and not self.applicable: + if not self.namespace: + raise Exception( + f"Missing required argument: {Argument.to_flag_name(Arguments.NAMESPACE)}" + f" when listing policies without {Argument.to_flag_name(Arguments.APPLICABLE)} flag." + ) + + def execute(self, api: PolarisDefaultApi) -> None: + catalog_api_client = get_catalog_api_client(api) + policy_api = PolicyAPI(catalog_api_client) + + namespace_str = self.namespace.replace('.', UNIT_SEPARATOR) if self.namespace else "" + if self.policies_subcommand == Subcommands.CREATE: + with open(self.policy_file, "r") as f: + policy = json.load(f) + policy_api.create_policy( + prefix=self.catalog_name, + namespace=namespace_str, + create_policy_request=CreatePolicyRequest( + name=self.policy_name, + type=self.policy_type, + description=self.policy_description, + content=json.dumps(policy) + ) + ) + elif self.policies_subcommand == Subcommands.DELETE: + policy_api.drop_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + detach_all=self.detach_all + ) + elif self.policies_subcommand == Subcommands.GET: + print(policy_api.load_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name + ).to_json()) + elif self.policies_subcommand == Subcommands.LIST: + if self.applicable: + applicable_policies_list = [] + + if self.target_name: + # Table-like level policies + applicable_policies_list = policy_api.get_applicable_policies( + prefix=self.catalog_name, + namespace=namespace_str, + target_name=self.target_name, + policy_type=self.policy_type + ).applicable_policies + elif self.namespace: + # Namespace level policies + applicable_policies_list = policy_api.get_applicable_policies( + prefix=self.catalog_name, + namespace=namespace_str, + policy_type=self.policy_type + ).applicable_policies + else: + # Catalog level policies + applicable_policies_list = policy_api.get_applicable_policies( + prefix=self.catalog_name, + policy_type=self.policy_type + ).applicable_policies + for policy in applicable_policies_list: + print(policy.to_json()) + else: + # List all policy identifiers in the namespace + policies_response = policy_api.list_policies( + prefix=self.catalog_name, + namespace=namespace_str, + policy_type=self.policy_type + ).to_json() + print(policies_response) + elif self.policies_subcommand == Subcommands.UPDATE: + with open(self.policy_file, "r") as f: + policy_document = json.load(f) + # Fetch the current policy to get its version + loaded_policy_response = policy_api.load_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name + ) + if loaded_policy_response and loaded_policy_response.policy: + current_policy_version = loaded_policy_response.policy.version + else: + raise Exception(f"Could not retrieve current policy version for {self.policy_name}") + + policy_api.update_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + update_policy_request=UpdatePolicyRequest( + description=self.policy_description, + content=json.dumps(policy_document), + current_policy_version=current_policy_version + ) + ) + elif self.policies_subcommand == Subcommands.ATTACH: + attachment_path_list = [] if self.attachment_type == "catalog" else self.attachment_path.split('.') + + policy_api.attach_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + attach_policy_request=AttachPolicyRequest( + target=PolicyAttachmentTarget( + type=self.attachment_type, + path=attachment_path_list + ), + parameters=self.parameters if isinstance(self.parameters, dict) else None + ) + ) + elif self.policies_subcommand == Subcommands.DETACH: + attachment_path_list = [] if self.attachment_type == "catalog" else self.attachment_path.split('.') + + policy_api.detach_policy( + prefix=self.catalog_name, + namespace=namespace_str, + policy_name=self.policy_name, + detach_policy_request=DetachPolicyRequest( + target=PolicyAttachmentTarget( + type=self.attachment_type, + path=attachment_path_list + ), + parameters=self.parameters if isinstance(self.parameters, dict) else None + ) + ) + else: + raise Exception(f"{self.policies_subcommand} is not supported in the CLI") diff --git a/client/python/cli/command/utils.py b/client/python/cli/command/utils.py new file mode 100644 index 0000000000..b19b562829 --- /dev/null +++ b/client/python/cli/command/utils.py @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import re + +from polaris.catalog.api_client import ApiClient +from polaris.catalog.configuration import Configuration +from polaris.management import PolarisDefaultApi + + +def get_catalog_api_client(api: PolarisDefaultApi) -> ApiClient: + """ + Convert a management API to a catalog API client + """ + mgmt_config = api.api_client.configuration + catalog_host = re.sub(r"/api/management(?:/v1)?", "/api/catalog", mgmt_config.host) + configuration = Configuration( + host=catalog_host, + username=mgmt_config.username, + password=mgmt_config.password, + access_token=mgmt_config.access_token, + ) + + if hasattr(mgmt_config, "proxy"): + configuration.proxy = mgmt_config.proxy + if hasattr(mgmt_config, "proxy_headers"): + configuration.proxy_headers = mgmt_config.proxy_headers + + return ApiClient(configuration) diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 335085d66f..756a47d159 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -88,6 +88,7 @@ class Commands: PRIVILEGES = "privileges" NAMESPACES = "namespaces" PROFILES = "profiles" + POLICIES = "policies" class Subcommands: @@ -110,6 +111,8 @@ class Subcommands: REVOKE = "revoke" ACCESS = "access" RESET = "reset" + ATTACH = "attach" + DETACH = "detach" class Actions: @@ -189,6 +192,16 @@ class Arguments: CATALOG_EXTERNAL_ID = "catalog_external_id" CATALOG_SIGNING_REGION = "catalog_signing_region" CATALOG_SIGNING_NAME = "catalog_signing_name" + POLICY = "policy" + POLICY_FILE = "policy_file" + POLICY_TYPE = "policy_type" + POLICY_DESCRIPTION = "policy_description" + TARGET_NAME = "target_name" + PARAMETERS = "parameters" + DETACH_ALL = "detach_all" + APPLICABLE = "applicable" + ATTACHMENT_TYPE = "attachment_type" + ATTACHMENT_PATH = "attachment_path" REALM = "realm" HEADER = "header" @@ -373,6 +386,18 @@ class Namespaces: LOCATION = "If specified, the location at which to store the namespace and entities inside it" PARENT = "If specified, list namespaces inside this parent namespace" + class Policies: + POLICY = "The name of a policy" + POLICY_FILE = "The path to a JSON file containing the policy definition" + POLICY_TYPE = "The type of the policy, e.g., 'system.data-compaction'" + POLICY_DESCRIPTION = "An optional description for the policy." + TARGET_NAME = "The name of the target entity (e.g., table name, namespace name)." + PARAMETERS = "Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times." + DETACH_ALL = "When set to true, the policy will be deleted along with all its attached mappings." + APPLICABLE = "When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the target." + ATTACHMENT_TYPE = "The type of entity to attach the policy to, e.g., 'catalog', 'namespace', or table-like." + ATTACHMENT_PATH = "The path of the entity to attach the policy to, e.g., 'ns1.tb1'. Not required for catalog-level attachment." + UNIT_SEPARATOR = chr(0x1F) CLIENT_ID_ENV = "CLIENT_ID" diff --git a/client/python/cli/options/option_tree.py b/client/python/cli/options/option_tree.py index f71589d901..78f15b96e3 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/cli/options/option_tree.py @@ -277,5 +277,50 @@ def get_tree() -> List[Option]: Option(Subcommands.UPDATE, input_name=Arguments.PROFILE), Option(Subcommands.GET, input_name=Arguments.PROFILE), Option(Subcommands.LIST), - ]) + ]), + Option(Commands.POLICIES, 'manage policies', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.POLICY_FILE, str, Hints.Policies.POLICY_FILE), + Argument(Arguments.POLICY_TYPE, str, Hints.Policies.POLICY_TYPE), + Argument(Arguments.POLICY_DESCRIPTION, str, Hints.Policies.POLICY_DESCRIPTION), + ], input_name=Arguments.POLICY), + Option(Subcommands.DELETE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.DETACH_ALL, bool, Hints.Policies.DETACH_ALL), + ], input_name=Arguments.POLICY), + Option(Subcommands.GET, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + ], input_name=Arguments.POLICY), + Option(Subcommands.LIST, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.TARGET_NAME, str, Hints.Policies.TARGET_NAME), + Argument(Arguments.APPLICABLE, bool, Hints.Policies.APPLICABLE), + Argument(Arguments.POLICY_TYPE, str, Hints.Policies.POLICY_TYPE), + ]), + Option(Subcommands.UPDATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.POLICY_FILE, str, Hints.Policies.POLICY_FILE), + Argument(Arguments.POLICY_DESCRIPTION, str, Hints.Policies.POLICY_DESCRIPTION), + ], input_name=Arguments.POLICY), + Option(Subcommands.ATTACH, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.ATTACHMENT_TYPE, str, Hints.Policies.ATTACHMENT_TYPE), + Argument(Arguments.ATTACHMENT_PATH, str, Hints.Policies.ATTACHMENT_PATH), + Argument(Arguments.PARAMETERS, str, Hints.Policies.PARAMETERS, allow_repeats=True), + ], input_name=Arguments.POLICY), + Option(Subcommands.DETACH, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.ATTACHMENT_TYPE, str, Hints.Policies.ATTACHMENT_TYPE), + Argument(Arguments.ATTACHMENT_PATH, str, Hints.Policies.ATTACHMENT_PATH), + Argument(Arguments.PARAMETERS, str, Hints.Policies.PARAMETERS, allow_repeats=True), + ], input_name=Arguments.POLICY), + ]), ] diff --git a/client/python/cli/polaris_cli.py b/client/python/cli/polaris_cli.py index c038bedbf2..78d8728e34 100644 --- a/client/python/cli/polaris_cli.py +++ b/client/python/cli/polaris_cli.py @@ -46,8 +46,50 @@ class PolarisCli: # Can be enabled if the client is able to authenticate directly without first fetching a token DIRECT_AUTHENTICATION_ENABLED = False + @staticmethod + def _patch_generated_models() -> None: + """ + The OpenAPI generator creates an `api_client` that dynamically looks up + model classes from the `polaris.catalog.models` module using `getattr()`. + For example, when a response for a `create_policy` call is received, the + deserializer tries to find the `LoadPolicyResponse` class by looking for + `polaris.catalog.models.LoadPolicyResponse`. + + However, the generator fails to add the necessary `import` statements + to the `polaris/catalog/models/__init__.py` file. This means that even + though the model files exist (e.g., `load_policy_response.py`), the classes + are not part of the `polaris.catalog.models` namespace. + + This method works around the bug in the generated code without modifying + the source files. It runs once per CLI execution, before any commands, and + manually injects the missing response-side model classes into the + `polaris.catalog.models` namespace, allowing the deserializer to find them. + """ + import polaris.catalog.models + from polaris.catalog.models.applicable_policy import ApplicablePolicy + from polaris.catalog.models.get_applicable_policies_response import GetApplicablePoliciesResponse + from polaris.catalog.models.list_policies_response import ListPoliciesResponse + from polaris.catalog.models.load_policy_response import LoadPolicyResponse + from polaris.catalog.models.policy import Policy + from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget + from polaris.catalog.models.policy_identifier import PolicyIdentifier + + models_to_patch = { + "ApplicablePolicy": ApplicablePolicy, + "GetApplicablePoliciesResponse": GetApplicablePoliciesResponse, + "ListPoliciesResponse": ListPoliciesResponse, + "LoadPolicyResponse": LoadPolicyResponse, + "Policy": Policy, + "PolicyAttachmentTarget": PolicyAttachmentTarget, + "PolicyIdentifier": PolicyIdentifier, + } + + for name, model_class in models_to_patch.items(): + setattr(polaris.catalog.models, name, model_class) + @staticmethod def execute(args=None): + PolarisCli._patch_generated_models() options = Parser.parse(args) if options.command == Commands.PROFILES: from cli.command import Command diff --git a/client/python/integration_tests/conftest.py b/client/python/integration_tests/conftest.py index eaa98e1e43..adfb9d6cd7 100644 --- a/client/python/integration_tests/conftest.py +++ b/client/python/integration_tests/conftest.py @@ -30,8 +30,10 @@ from polaris.catalog import OAuthTokenResponse from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API +from polaris.catalog.api.policy_api import PolicyAPI from polaris.catalog.api_client import ApiClient as CatalogApiClient from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration + from polaris.management import ( Catalog, ApiClient, @@ -197,13 +199,28 @@ def test_catalog_client( return IcebergCatalogAPI( CatalogApiClient( - Configuration( + CatalogApiClientConfiguration( access_token=test_principal_token.access_token, host=polaris_catalog_url ) ) ) +@pytest.fixture +def test_policy_api( + polaris_catalog_url: str, + test_principal_token: OAuthTokenResponse, +) -> PolicyAPI: + return PolicyAPI( + CatalogApiClient( + CatalogApiClientConfiguration( + access_token=test_principal_token.access_token, + host=polaris_catalog_url, + ) + ) + ) + + @pytest.fixture def test_catalog( management_client: PolarisDefaultApi, @@ -356,3 +373,47 @@ def clear_namespace( def format_namespace(namespace: List[str]) -> str: return codecs.decode("1F", "hex").decode("UTF-8").join(namespace) + + +@pytest.fixture(scope="session", autouse=True) +def _patch_generated_models() -> None: + """ + The OpenAPI generator creates an `api_client` that dynamically looks up + model classes from the `polaris.catalog.models` module using `getattr()`. + For example, when a response for a `create_policy` call is received, the + deserializer tries to find the `LoadPolicyResponse` class by looking for + `polaris.catalog.models.LoadPolicyResponse`. + + However, the generator fails to add the necessary `import` statements + to the `polaris/catalog/models/__init__.py` file. This means that even + though the model files exist (e.g., `load_policy_response.py`), the classes + are not part of the `polaris.catalog.models` namespace. + + This fixture works around the bug in the generated code without modifying + the source files. It runs once per test session, before any tests, and + manually injects the missing response-side model classes into the + `polaris.catalog.models` namespace, allowing the deserializer to find them. + """ + import polaris.catalog.models + from polaris.catalog.models.applicable_policy import ApplicablePolicy + from polaris.catalog.models.get_applicable_policies_response import ( + GetApplicablePoliciesResponse, + ) + from polaris.catalog.models.list_policies_response import ListPoliciesResponse + from polaris.catalog.models.load_policy_response import LoadPolicyResponse + from polaris.catalog.models.policy import Policy + from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget + from polaris.catalog.models.policy_identifier import PolicyIdentifier + + models_to_patch = { + "ApplicablePolicy": ApplicablePolicy, + "GetApplicablePoliciesResponse": GetApplicablePoliciesResponse, + "ListPoliciesResponse": ListPoliciesResponse, + "LoadPolicyResponse": LoadPolicyResponse, + "Policy": Policy, + "PolicyAttachmentTarget": PolicyAttachmentTarget, + "PolicyIdentifier": PolicyIdentifier, + } + + for name, model_class in models_to_patch.items(): + setattr(polaris.catalog.models, name, model_class) diff --git a/client/python/integration_tests/test_catalog_apis.py b/client/python/integration_tests/test_catalog_apis.py index 60109713e3..214329a069 100644 --- a/client/python/integration_tests/test_catalog_apis.py +++ b/client/python/integration_tests/test_catalog_apis.py @@ -17,6 +17,7 @@ # under the License. # +import json import os.path import time @@ -30,6 +31,13 @@ TableIdentifier, IcebergCatalogAPI, ) +from polaris.catalog.models.attach_policy_request import AttachPolicyRequest +from polaris.catalog.models.create_policy_request import CreatePolicyRequest +from polaris.catalog.models.detach_policy_request import DetachPolicyRequest +from polaris.catalog.models.policy_attachment_target import PolicyAttachmentTarget +from polaris.catalog.models.update_policy_request import UpdatePolicyRequest +from polaris.catalog.exceptions import NotFoundException +from polaris.catalog.api.policy_api import PolicyAPI from pyiceberg.schema import Schema from polaris.management import Catalog from pyiceberg.catalog import Catalog as PyIcebergCatalog @@ -168,3 +176,213 @@ def test_drop_table_purge( if os.path.exists(metadata_location): pytest.fail(f"Metadata file {metadata_location} still exists after 60 seconds.") + + +def test_policies( + test_catalog: Catalog, + test_catalog_client: IcebergCatalogAPI, + test_policy_api: PolicyAPI, + test_table_schema: Schema, +) -> None: + # Resource identifiers + namespace_name = "POLICY_NS1" + sub_namespace = "POLICY_NS2" + sub_namespace_path = [namespace_name, sub_namespace] + policy_name = "test_policy" + table_name = "test_table_for_policy" + table_path = sub_namespace_path + [table_name] + + policy_content = { + "version": "2025-02-03", + "enable": True, + "config": {"target_file_size_bytes": 134217728}, + } + policy_type = "system.data-compaction" + policy_description = "A test policy" + + # Create resources + test_catalog_client.create_namespace( + prefix=test_catalog.name, + create_namespace_request=CreateNamespaceRequest(namespace=[namespace_name]), + ) + test_catalog_client.create_namespace( + prefix=test_catalog.name, + create_namespace_request=CreateNamespaceRequest(namespace=sub_namespace_path), + ) + test_catalog_client.create_table( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + create_table_request=CreateTableRequest( + name=table_name, + var_schema=ModelSchema.from_dict(test_table_schema.model_dump()), + ), + ) + test_policy_api.create_policy( + prefix=test_catalog.name, + namespace=namespace_name, + create_policy_request=CreatePolicyRequest( + name=policy_name, + type=policy_type, + description=policy_description, + content=json.dumps(policy_content), + ), + ) + + try: + # GET + loaded_policy = test_policy_api.load_policy( + prefix=test_catalog.name, namespace=namespace_name, policy_name=policy_name + ) + assert loaded_policy.policy.name == policy_name + assert loaded_policy.policy.policy_type == policy_type + assert loaded_policy.policy.description == policy_description + assert json.loads(loaded_policy.policy.content) == policy_content + + # LIST + policies = test_policy_api.list_policies( + prefix=test_catalog.name, namespace=namespace_name + ) + assert len(policies.identifiers) == 1 + assert policies.identifiers[0].name == policy_name + assert policies.identifiers[0].namespace == [namespace_name] + + # UPDATE + updated_policy_content = { + "version": "2025-02-03", + "enable": False, + "config": {"target_file_size_bytes": 134217728}, + } + updated_policy_description = "An updated test policy" + test_policy_api.update_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + update_policy_request=UpdatePolicyRequest( + description=updated_policy_description, + content=json.dumps(updated_policy_content), + current_policy_version=loaded_policy.policy.version, + ), + ) + + # GET after UPDATE + updated_policy = test_policy_api.load_policy( + prefix=test_catalog.name, namespace=namespace_name, policy_name=policy_name + ) + assert updated_policy.policy.description == updated_policy_description + assert json.loads(updated_policy.policy.content) == updated_policy_content + + # ATTACH to namespace + test_policy_api.attach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + attach_policy_request=AttachPolicyRequest( + target=PolicyAttachmentTarget(type="namespace", path=sub_namespace_path) + ), + ) + + # GET APPLICABLE on namespace + applicable_policies = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, namespace=format_namespace(sub_namespace_path) + ) + assert len(applicable_policies.applicable_policies) == 1 + assert applicable_policies.applicable_policies[0].name == policy_name + + # DETACH from namespace + test_policy_api.detach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_policy_request=DetachPolicyRequest( + target=PolicyAttachmentTarget(type="namespace", path=sub_namespace_path) + ), + ) + + # GET APPLICABLE on namespace after DETACH + applicable_policies_after_detach = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, namespace=format_namespace(sub_namespace_path) + ) + assert len(applicable_policies_after_detach.applicable_policies) == 0 + + # ATTACH to table + test_policy_api.attach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + attach_policy_request=AttachPolicyRequest( + target=PolicyAttachmentTarget(type="table-like", path=table_path) + ), + ) + + # GET APPLICABLE on table + applicable_for_table = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + target_name=table_name, + ) + assert len(applicable_for_table.applicable_policies) == 1 + assert applicable_for_table.applicable_policies[0].name == policy_name + + # DETACH from table + test_policy_api.detach_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_policy_request=DetachPolicyRequest( + target=PolicyAttachmentTarget(type="table-like", path=table_path) + ), + ) + + # GET APPLICABLE on table after DETACH + applicable_for_table_after_detach = test_policy_api.get_applicable_policies( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + target_name=table_name, + ) + assert len(applicable_for_table_after_detach.applicable_policies) == 0 + + # DELETE + test_policy_api.drop_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_all=True, + ) + + # LIST after DELETE + policies_after_delete = test_policy_api.list_policies( + prefix=test_catalog.name, namespace=namespace_name + ) + assert len(policies_after_delete.identifiers) == 0 + + finally: + # Cleanup + try: + test_policy_api.drop_policy( + prefix=test_catalog.name, + namespace=namespace_name, + policy_name=policy_name, + detach_all=True, + ) + except NotFoundException: + pass + try: + test_catalog_client.drop_table( + prefix=test_catalog.name, + namespace=format_namespace(sub_namespace_path), + table=table_name, + ) + except NotFoundException: + pass + try: + test_catalog_client.drop_namespace( + prefix=test_catalog.name, namespace=format_namespace(sub_namespace_path) + ) + except NotFoundException: + pass + try: + test_catalog_client.drop_namespace( + prefix=test_catalog.name, namespace=namespace_name + ) + except NotFoundException: + pass diff --git a/site/content/in-dev/unreleased/command-line-interface.md b/site/content/in-dev/unreleased/command-line-interface.md index cfe606ba10..2c7c26563e 100644 --- a/site/content/in-dev/unreleased/command-line-interface.md +++ b/site/content/in-dev/unreleased/command-line-interface.md @@ -39,6 +39,7 @@ options: --realm --header --profile +--proxy ``` `COMMAND` must be one of the following: @@ -49,7 +50,8 @@ options: 5. namespaces 6. privileges 7. profiles -8. repair +8. policies +9. repair Each _command_ supports several _subcommands_, and some _subcommands_ have _actions_ that come after the subcommand in turn. Finally, _arguments_ follow to form a full invocation. Within a set of named arguments at the end of an invocation ordering is generally not important. Many invocations also have a required positional argument of the type that the _command_ refers to. Again, the ordering of this positional argument relative to named arguments is not important. @@ -62,6 +64,7 @@ polaris catalogs update --property foo=bar some_other_catalog polaris catalogs update another_catalog --property k=v polaris privileges namespace grant --namespace some.schema --catalog fourth_catalog --catalog-role some_catalog_role TABLE_READ_DATA polaris profiles list +polaris policies list --catalog some_catalog --namespace some.schema polaris repair ``` @@ -1225,6 +1228,192 @@ options: polaris profiles update dev ``` +### Policies + +The `policies` command is used to manage policies within Polaris. + +`policies` supports the following subcommands: + +1. attach +2. create +3. delete +4. detach +5. get +6. list +7. update + +#### attach + +The `attach` subcommand is used to create a mapping between a policy and a resource entity. + +``` +input: polaris policies attach --help +options: + attach + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --attachment-type The type of entity to attach the policy to, e.g., 'catalog', 'namespace', or table-like. + --attachment-path The path of the entity to attach the policy to, e.g., 'ns1.tb1'. Not required for catalog-level attachment. + --parameters Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies attach --catalog some_catalog --namespace some.schema --attachment-type namespace --attachment-path some.schema some_policy + +polaris policies attach --catalog some_catalog --namespace some.schema --attachment-type table-like --attachment-path some.schema.t some_table_policy +``` + +#### create + +The `create` subcommand is used to create a policy. + +``` +input: polaris policies create --help +options: + create + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --policy-file The path to a JSON file containing the policy definition + --policy-type The type of the policy, e.g., 'system.data-compaction' + --policy-description An optional description for the policy. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies create --catalog some_catalog --namespace some.schema --policy-file some_policy.json --policy-type system.data-compaction some_policy + +polaris policies create --catalog some_catalog --namespace some.schema --policy-file some_snapshot_expiry_policy.json --policy-type system.snapshot-expiry some_snapshot_expiry_policy +``` + +#### delete + +The `delete` subcommand is used to delete a policy. + +``` +input: polaris policies delete --help +options: + delete + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --detach-all When set to true, the policy will be deleted along with all its attached mappings. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies delete --catalog some_catalog --namespace some.schema some_policy + +polaris policies delete --catalog some_catalog --namespace some.schema --detach-all some_policy +``` + +#### detach + +The `detach` subcommand is used to remove a mapping between a policy and a target entity + +``` +input: polaris policies detach --help +options: + detach + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --attachment-type The type of entity to attach the policy to, e.g., 'catalog', 'namespace', or table-like. + --attachment-path The path of the entity to attach the policy to, e.g., 'ns1.tb1'. Not required for catalog-level attachment. + --parameters Optional key-value pairs for the attachment/detachment, e.g., key=value. Can be specified multiple times. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies detach --catalog some_catalog --namespace some.schema --attachment-type namespace --attachment-path some.schema some_policy + +polaris policies detach --catalog some_catalog --namespace some.schema --attachment-type catalog --attachment-path some_catalog some_policy +``` + +#### get + +The `get` subcommand is used to load a policy from the catalog. + +``` +input: polaris policies get --help +options: + get + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies get --catalog some_catalog --namespace some.schema some_policy +``` + +#### list + +The `list` subcommand is used to get all policy identifiers under this namespace and all applicable policies for a specified entity. + +``` +input: polaris policies list --help +options: + list + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --target-name The name of the target entity (e.g., table name, namespace name). + --applicable When set, lists policies applicable to the target entity (considering inheritance) instead of policies defined directly in the target. + --policy-type The type of the policy, e.g., 'system.data-compaction' +``` + +##### Examples + +``` +polaris policies list --catalog some_catalog + +polaris policies list --catalog some_catalog --applicable +``` + +#### update + +The `update` subcommand is used to update a policy. + +``` +input: polaris policies update --help +options: + update + Named arguments: + --catalog The name of an existing catalog + --namespace A period-delimited namespace + --policy-file The path to a JSON file containing the policy definition + --policy-description An optional description for the policy. + Positional arguments: + policy +``` + +##### Examples + +``` +polaris policies update --catalog some_catalog --namespace some.schema --policy-file my_updated_policy.json my_policy + +polaris policies update --catalog some_catalog --namespace some.schema --policy-file my_updated_policy.json --policy-description "Updated policy description" my_policy +``` + ### repair The `repair` command is a bash script wrapper used to regenerate Python client code and update necessary dependencies, ensuring the Polaris client remains up-to-date and functional. **Please note that this command does not support any options and its usage information is not available via a `--help` flag.**