diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index e310f398f821..2754e720d147 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -983,6 +983,90 @@ paths: $ref: '#/components/responses/TableNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + post: + tags: + - partition + summary: Create partitions + operationId: createPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionsRequest' + responses: + "200": + description: Partitions created or found to exist + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionsResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TableNotExistErrorResponse' + "409": + $ref: '#/components/responses/ResourceAlreadyExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}/partitions/drop: + post: + tags: + - partition + summary: Drop partitions + description: Unregisters partitions from the catalog. The server never deletes data files; managed format table data deletion is performed by the client afterwards. + operationId: dropPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DropPartitionsRequest' + responses: + "200": + description: Partitions dropped or found missing + content: + application/json: + schema: + $ref: '#/components/schemas/DropPartitionsResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TableNotExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/tables/{table}/partitions/mark: post: tags: @@ -2352,11 +2436,68 @@ components: type: string DropPartitionsRequest: type: object + required: + - partitionSpecs + properties: + partitionSpecs: + type: array + items: + type: object + additionalProperties: + type: string + ignoreIfNotExists: + type: boolean + default: true + DropPartitionsResponse: + type: object + required: + - dropped + - missing properties: - specs: + dropped: type: array items: type: object + additionalProperties: + type: string + missing: + type: array + items: + type: object + additionalProperties: + type: string + CreatePartitionsRequest: + type: object + required: + - partitionSpecs + properties: + partitionSpecs: + type: array + items: + type: object + additionalProperties: + type: string + ignoreIfExists: + type: boolean + default: true + CreatePartitionsResponse: + type: object + required: + - created + - existed + properties: + created: + type: array + items: + type: object + additionalProperties: + type: string + existed: + type: array + items: + type: object + additionalProperties: + type: string AlterTableRequest: type: object properties: @@ -2398,7 +2539,7 @@ components: resourceType: type: string nullable: true - enum: [ "DATABASE", "TABLE", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "DIALECT", "UNKNOWN" ] + enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "DIALECT", "UNKNOWN" ] resourceName: type: string nullable: true diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 263c4e2c0640..4d321ed2797e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -42,9 +42,11 @@ import org.apache.paimon.rest.requests.CreateBranchRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; @@ -58,6 +60,8 @@ import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -855,6 +859,37 @@ public void markDonePartitions(Identifier identifier, List> restAuthFunction); } + /** Create partitions for table, ignoring partitions which already exist. */ + public CreatePartitionsResponse createPartitions( + Identifier identifier, List> partitions) { + return createPartitions(identifier, partitions, true); + } + + /** Create partitions for table. */ + public CreatePartitionsResponse createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) { + CreatePartitionsRequest request = new CreatePartitionsRequest(partitions, ignoreIfExists); + return client.post( + resourcePaths.partitions(identifier.getDatabaseName(), identifier.getObjectName()), + request, + CreatePartitionsResponse.class, + restAuthFunction); + } + + /** Drop (unregister) partitions for table; the server never deletes data files. */ + public DropPartitionsResponse dropPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfNotExists) { + DropPartitionsRequest request = new DropPartitionsRequest(partitions, ignoreIfNotExists); + return client.post( + resourcePaths.dropPartitions( + identifier.getDatabaseName(), identifier.getObjectName()), + request, + DropPartitionsResponse.class, + restAuthFunction); + } + /** * List partitions for table. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 28f79d040995..0ab36e348c6e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -213,6 +213,18 @@ public String partitions(String databaseName, String objectName) { PARTITIONS); } + public String dropPartitions(String databaseName, String objectName) { + return SLASH.join( + V1, + prefix, + DATABASES, + encodeString(databaseName), + TABLES, + encodeString(objectName), + PARTITIONS, + "drop"); + } + public String markDonePartitions(String databaseName, String objectName) { return SLASH.join( V1, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java new file mode 100644 index 000000000000..506c66406eec --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; + +/** Request for creating partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreatePartitionsRequest implements RESTRequest { + + private static final String FIELD_PARTITION_SPECS = "partitionSpecs"; + private static final String FIELD_IGNORE_IF_EXISTS = "ignoreIfExists"; + + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; + + @JsonProperty(FIELD_IGNORE_IF_EXISTS) + private final boolean ignoreIfExists; + + public CreatePartitionsRequest(List> partitionSpecs) { + this(partitionSpecs, true); + } + + @JsonCreator + public CreatePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs, + @JsonProperty(FIELD_IGNORE_IF_EXISTS) @Nullable Boolean ignoreIfExists) { + this.partitionSpecs = partitionSpecs; + this.ignoreIfExists = ignoreIfExists == null || ignoreIfExists; + } + + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; + } + + @JsonGetter(FIELD_IGNORE_IF_EXISTS) + public boolean ignoreIfExists() { + return ignoreIfExists; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java new file mode 100644 index 000000000000..cdc448034e38 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; + +/** Request for dropping (unregistering) partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionsRequest implements RESTRequest { + + private static final String FIELD_PARTITION_SPECS = "partitionSpecs"; + private static final String FIELD_IGNORE_IF_NOT_EXISTS = "ignoreIfNotExists"; + + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; + + @JsonProperty(FIELD_IGNORE_IF_NOT_EXISTS) + private final boolean ignoreIfNotExists; + + public DropPartitionsRequest(List> partitionSpecs) { + this(partitionSpecs, true); + } + + @JsonCreator + public DropPartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs, + @JsonProperty(FIELD_IGNORE_IF_NOT_EXISTS) @Nullable Boolean ignoreIfNotExists) { + this.partitionSpecs = partitionSpecs; + this.ignoreIfNotExists = ignoreIfNotExists == null || ignoreIfNotExists; + } + + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; + } + + @JsonGetter(FIELD_IGNORE_IF_NOT_EXISTS) + public boolean ignoreIfNotExists() { + return ignoreIfNotExists; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java new file mode 100644 index 000000000000..03a923ed21eb --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** Response for creating partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreatePartitionsResponse implements RESTResponse { + + private static final String FIELD_CREATED = "created"; + private static final String FIELD_EXISTED = "existed"; + + @JsonProperty(FIELD_CREATED) + private final List> created; + + @JsonProperty(FIELD_EXISTED) + private final List> existed; + + @JsonCreator + public CreatePartitionsResponse( + @JsonProperty(FIELD_CREATED) List> created, + @JsonProperty(FIELD_EXISTED) List> existed) { + this.created = created; + this.existed = existed; + } + + @JsonGetter(FIELD_CREATED) + public List> getCreated() { + return created; + } + + @JsonGetter(FIELD_EXISTED) + public List> getExisted() { + return existed; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java new file mode 100644 index 000000000000..b250b6ddb9a4 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** Response for dropping (unregistering) partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionsResponse implements RESTResponse { + + private static final String FIELD_DROPPED = "dropped"; + private static final String FIELD_MISSING = "missing"; + + @JsonProperty(FIELD_DROPPED) + private final List> dropped; + + @JsonProperty(FIELD_MISSING) + private final List> missing; + + @JsonCreator + public DropPartitionsResponse( + @JsonProperty(FIELD_DROPPED) List> dropped, + @JsonProperty(FIELD_MISSING) List> missing) { + this.dropped = dropped; + this.missing = missing; + } + + @JsonGetter(FIELD_DROPPED) + public List> getDropped() { + return dropped; + } + + @JsonGetter(FIELD_MISSING) + public List> getMissing() { + return missing; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java index d4f9662dfa28..4eb52308dd0c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java @@ -35,6 +35,8 @@ public class ErrorResponse implements RESTResponse { public static final String RESOURCE_TYPE_TABLE = "TABLE"; + public static final String RESOURCE_TYPE_PARTITION = "PARTITION"; + public static final String RESOURCE_TYPE_COLUMN = "COLUMN"; public static final String RESOURCE_TYPE_SNAPSHOT = "SNAPSHOT"; diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index afe4ed8ae6de..01f04cbba3ae 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -335,6 +335,25 @@ public List listPartitions(Identifier identifier) throws TableNotExis return result; } + @Override + public void createPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + wrapped.createPartitions(identifier, partitions); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + + @Override + public void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + wrapped.createPartitions(identifier, partitions, ignoreIfExists); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 57fa040a2acd..8e4ad8d51815 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1034,6 +1034,27 @@ void deleteTag(Identifier identifier, String tagName) default void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException {} + /** + * Create partitions of the specify table with explicit existence semantics. + * + * @param identifier path of the table to create partitions + * @param partitions partitions to be created + * @param ignoreIfExists if false, fail when any partition already exists and apply none of the + * batch; if true, behave like {@link #createPartitions(Identifier, List)} + * @throws TableNotExistException if the table does not exist + */ + default void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + if (!ignoreIfExists) { + throw new UnsupportedOperationException( + String.format( + "Catalog %s does not support createPartitions without ignoreIfExists.", + getClass().getName())); + } + createPartitions(identifier, partitions); + } + /** * Drop partitions of the specify table. Ignore non-existent partitions. * diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 0f18f7d04540..290849024fdb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -322,6 +322,13 @@ public void createPartitions(Identifier identifier, List> pa wrapped.createPartitions(identifier, partitions); } + @Override + public void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + wrapped.createPartitions(identifier, partitions, ignoreIfExists); + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 0a76fb5a7848..c44378f6d00f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -59,9 +59,11 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Instant; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; +import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotNotExistException; @@ -725,6 +727,59 @@ public void markDonePartitions(Identifier identifier, List> } } + @Override + public void createPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + createPartitions(identifier, partitions, true); + } + + @Override + public void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + try { + api.createPartitions(identifier, partitions, ignoreIfExists); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } catch (AlreadyExistsException e) { + // Server contract: with ignoreIfExists=false the whole batch is rejected atomically. + throw new IllegalStateException( + String.format( + "Some partitions of table %s already exist: %s", + identifier, e.getMessage())); + } catch (BadRequestException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + + @Override + public void dropPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + // Managed format table: metadata-only unregistration on the server. Data deletion is + // the caller's responsibility (performed with the table FileIO afterwards). + try { + api.dropPartitions(identifier, partitions, true); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } catch (BadRequestException e) { + throw new IllegalArgumentException(e.getMessage()); + } + return; + } + // Non-managed tables keep the default truncate-based data semantics. + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.truncatePartitions(partitions); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Override public List listPartitions(Identifier identifier) throws TableNotExistException { try { @@ -734,8 +789,11 @@ public List listPartitions(Identifier identifier) throws TableNotExis } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } catch (NotImplementedException e) { - // not a metastore partitioned table - return listPartitionsFromFileSystem(getTable(identifier)); + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + throw e; + } + return listPartitionsFromFileSystem(table); } } @@ -753,8 +811,11 @@ public PagedList listPartitionsPaged( } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } catch (NotImplementedException e) { - // not a metastore partitioned table - return new PagedList<>(listPartitionsFromFileSystem(getTable(identifier)), null); + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + throw e; + } + return new PagedList<>(listPartitionsFromFileSystem(table), null); } } @@ -769,7 +830,11 @@ public List listPartitionsByNames( } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } catch (NotImplementedException e) { - return listPartitionsFromFileSystem(getTable(identifier), partitions); + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + throw e; + } + return listPartitionsFromFileSystem(table, partitions); } } @@ -836,6 +901,7 @@ public List listBranches(Identifier identifier) throws TableNotExistExce @Override public boolean supportsPartitionModification() { + // Paimon tables use commit-based partition maintenance. return false; } @@ -1257,6 +1323,11 @@ RESTApi api() { return api; } + private static boolean isCatalogManagedFormatTable(Table table) { + return table instanceof FormatTable + && new CoreOptions(table.options()).partitionedTableInMetastore(); + } + private FileIO fileIOForData(Path path, Identifier identifier) { return CachingFileIO.wrapWithCachingIfNeeded( dataTokenEnabled diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index c28299b79e38..0961050f110d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -56,6 +56,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,6 +66,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.apache.paimon.data.BinaryString.fromString; import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY; @@ -334,6 +336,38 @@ public void testPartitionCache() throws Exception { assertThat(partitionEntryListFromCache).containsAll(partitionEntryList); } + @Test + public void testCreatePartitionsInvalidatesPartitionCache() throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + when(wrapped.listPartitions(identifier)).thenReturn(emptyList(), singletonList(created)); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + catalog.createPartitions(identifier, singletonList(spec)); + + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + + @Test + public void testCreatePartitionsWithIgnoreIfExistsInvalidatesPartitionCache() throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + when(wrapped.listPartitions(identifier)).thenReturn(emptyList(), singletonList(created)); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + catalog.createPartitions(identifier, singletonList(spec), false); + + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + @Test public void testDeadlock() throws Exception { Catalog underlyCatalog = this.catalog; diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index f5045a4360a6..b4af3194b72f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -43,6 +43,7 @@ import org.apache.paimon.rest.auth.DLFTokenLoaderFactory; import org.apache.paimon.rest.auth.RESTAuthParameter; import org.apache.paimon.rest.exceptions.NotAuthorizedException; +import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.types.DataTypes; @@ -214,6 +215,55 @@ void testCreateTableDefaultOptions() throws Exception { "default-value"); } + @Test + void testManagedFormatTablePagedPartitionListingDoesNotFallback() throws Exception { + Identifier identifier = createManagedFormatTable(); + restCatalogServer.setPartitionListingSupported(false); + + assertThatThrownBy(() -> restCatalog.listPartitionsPaged(identifier, null, null, null)) + .isInstanceOf(NotImplementedException.class); + } + + @Test + void testManagedFormatTablePartitionListingByNamesDoesNotFallback() throws Exception { + Identifier identifier = createManagedFormatTable(); + restCatalogServer.setPartitionListingSupported(false); + + assertThatThrownBy( + () -> + restCatalog.listPartitionsByNames( + identifier, + Collections.singletonList( + Collections.singletonMap("dt", "20260717")))) + .isInstanceOf(NotImplementedException.class); + } + + @Test + void testManagedFormatTablePartitionListingDoesNotFallback() throws Exception { + Identifier identifier = createManagedFormatTable(); + restCatalogServer.setPartitionListingSupported(false); + + assertThatThrownBy(() -> restCatalog.listPartitions(identifier)) + .isInstanceOf(NotImplementedException.class); + } + + private Identifier createManagedFormatTable() throws Exception { + Identifier identifier = Identifier.create("db1", "managed_partition_table"); + restCatalog.createDatabase(identifier.getDatabaseName(), true); + restCatalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + false); + return identifier; + } + @Test void testBaseHeadersInRequests() throws Exception { // Set custom headers in options diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java index 1b8b1f71d0c0..bdf2b5f82c1b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java @@ -24,13 +24,17 @@ import org.apache.paimon.rest.requests.AlterViewRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -50,10 +54,12 @@ import org.junit.Test; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** Test for {@link RESTApi} json. */ @@ -204,6 +210,87 @@ public void listPartitionsResponseParseTest() throws Exception { parseData.getPartitions().get(0).fileCount()); } + @Test + public void createPartitionsResponseParseTest() throws Exception { + Map created = new HashMap<>(); + created.put("dt", "20260714"); + created.put("region", "cn"); + Map existed = new HashMap<>(); + existed.put("dt", "20260713"); + existed.put("region", "us"); + CreatePartitionsResponse response = + new CreatePartitionsResponse( + Collections.singletonList(created), Collections.singletonList(existed)); + CreatePartitionsResponse parsed = + RESTApi.fromJson(RESTApi.toJson(response), CreatePartitionsResponse.class); + + assertEquals(Collections.singletonList(created), parsed.getCreated()); + assertEquals(Collections.singletonList(existed), parsed.getExisted()); + } + + @Test + public void createPartitionsRequestParseTest() throws Exception { + String requestWithoutIgnoreIfExists = "{\"partitionSpecs\":[{\"dt\":\"20260715\"}]}"; + CreatePartitionsRequest defaultRequest = + RESTApi.fromJson(requestWithoutIgnoreIfExists, CreatePartitionsRequest.class); + + assertEquals( + Collections.singletonList(Collections.singletonMap("dt", "20260715")), + defaultRequest.getPartitionSpecs()); + assertTrue(defaultRequest.ignoreIfExists()); + + CreatePartitionsRequest explicitRequest = + new CreatePartitionsRequest(defaultRequest.getPartitionSpecs(), false); + String explicitRequestJson = RESTApi.toJson(explicitRequest); + assertTrue(explicitRequestJson.contains("\"partitionSpecs\"")); + assertFalse(explicitRequestJson.contains("\"specs\"")); + assertTrue(explicitRequestJson.contains("\"ignoreIfExists\":false")); + + CreatePartitionsRequest parsedExplicitRequest = + RESTApi.fromJson(explicitRequestJson, CreatePartitionsRequest.class); + assertFalse(parsedExplicitRequest.ignoreIfExists()); + } + + @Test + public void dropPartitionsResponseParseTest() throws Exception { + Map dropped = new HashMap<>(); + dropped.put("dt", "20260714"); + dropped.put("region", "cn"); + Map missing = new HashMap<>(); + missing.put("dt", "20260713"); + missing.put("region", "us"); + DropPartitionsResponse response = + new DropPartitionsResponse( + Collections.singletonList(dropped), Collections.singletonList(missing)); + DropPartitionsResponse parsed = + RESTApi.fromJson(RESTApi.toJson(response), DropPartitionsResponse.class); + + assertEquals(Collections.singletonList(dropped), parsed.getDropped()); + assertEquals(Collections.singletonList(missing), parsed.getMissing()); + } + + @Test + public void dropPartitionsRequestParseTest() throws Exception { + String requestWithoutIgnoreIfNotExists = "{\"partitionSpecs\":[{\"dt\":\"20260715\"}]}"; + DropPartitionsRequest defaultRequest = + RESTApi.fromJson(requestWithoutIgnoreIfNotExists, DropPartitionsRequest.class); + + assertEquals( + Collections.singletonList(Collections.singletonMap("dt", "20260715")), + defaultRequest.getPartitionSpecs()); + assertTrue(defaultRequest.ignoreIfNotExists()); + + DropPartitionsRequest explicitRequest = + new DropPartitionsRequest(defaultRequest.getPartitionSpecs(), false); + String explicitRequestJson = RESTApi.toJson(explicitRequest); + assertTrue(explicitRequestJson.contains("\"partitionSpecs\"")); + assertTrue(explicitRequestJson.contains("\"ignoreIfNotExists\":false")); + + DropPartitionsRequest parsedExplicitRequest = + RESTApi.fromJson(explicitRequestJson, DropPartitionsRequest.class); + assertFalse(parsedExplicitRequest.ignoreIfNotExists()); + } + @Test public void createViewRequestParseTest() throws Exception { CreateViewRequest request = MockRESTMessage.createViewRequest("t1"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index af5d94e3f632..3a0e1539eea1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -56,9 +56,11 @@ import org.apache.paimon.rest.requests.CreateBranchRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -70,6 +72,8 @@ import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -207,6 +211,8 @@ public class RESTCatalogServer { private final List> receivedHeaders = new ArrayList<>(); + private volatile boolean partitionListingSupported = true; + public RESTCatalogServer( String dataPath, AuthProvider authProvider, ConfigResponse config, String warehouse) { this.warehouse = warehouse; @@ -269,6 +275,10 @@ public void removeDataToken(Identifier identifier) { DataTokenStore.removeDataToken(warehouse, identifier.getFullName()); } + public void setPartitionListingSupported(boolean partitionListingSupported) { + this.partitionListingSupported = partitionListingSupported; + } + public void addNoPermissionDatabase(String database) { noPermissionDatabases.add(database); } @@ -449,6 +459,11 @@ && isTableByIdRequest(request.getPath())) { && ResourcePaths.TABLES.equals(resources[1]) && "partitions".equals(resources[3]) && "list-by-names".equals(resources[4]); + boolean isDropPartitions = + resources.length == 5 + && ResourcePaths.TABLES.equals(resources[1]) + && "partitions".equals(resources[3]) + && "drop".equals(resources[4]); boolean isBranches = resources.length >= 4 @@ -479,7 +494,7 @@ && isTableByIdRequest(request.getPath())) { } } // validate partition - if (isPartitions || isMarkDonePartitions) { + if (isPartitions || isMarkDonePartitions || isDropPartitions) { String tableName = RESTUtil.decodeString(resources[2]); Optional error = checkTablePartitioned( @@ -494,9 +509,18 @@ && isTableByIdRequest(request.getPath())) { catalog.markDonePartitions( identifier, markDonePartitionsRequest.getPartitionSpecs()); return new MockResponse().setResponseCode(200); + } else if (!partitionListingSupported + && ((isPartitions && "GET".equals(restAuthParameter.method())) + || isListPartitionsByNames)) { + return mockResponse(new ErrorResponse(null, null, "", 501), 501); + } else if (isDropPartitions) { + return dropPartitionsHandle(restAuthParameter.data(), identifier); } else if (isPartitions) { return partitionsApiHandle( - restAuthParameter.method(), parameters, identifier); + restAuthParameter.method(), + restAuthParameter.data(), + parameters, + identifier); } else if (isListPartitionsByNames) { ListPartitionsByNamesRequest listPartitionsByNamesRequest = RESTApi.fromJson(data, ListPartitionsByNamesRequest.class); @@ -1815,7 +1839,8 @@ private MockResponse renameTableHandle(String data) throws Exception { } private MockResponse partitionsApiHandle( - String method, Map parameters, Identifier tableIdentifier) { + String method, String data, Map parameters, Identifier tableIdentifier) + throws Exception { String partitionNamePattern = parameters.get(PARTITION_NAME_PATTERN); switch (method) { case "GET": @@ -1835,11 +1860,89 @@ private MockResponse partitionsApiHandle( } } return generateFinalListPartitionsResponse(parameters, partitions); + case "POST": + CreatePartitionsRequest request = + RESTApi.fromJson(data, CreatePartitionsRequest.class); + List storedPartitions = + tablePartitionsStore.computeIfAbsent( + tableIdentifier.getFullName(), ignored -> new ArrayList<>()); + Set> existingSpecs = + storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); + if (!request.ignoreIfExists()) { + Set> seenSpecs = new HashSet<>(existingSpecs); + Optional> conflictingSpec = + request.getPartitionSpecs().stream() + .filter(spec -> !seenSpecs.add(spec)) + .findFirst(); + if (conflictingSpec.isPresent()) { + String partitionName = + PartitionUtils.buildPartitionName(conflictingSpec.get()); + ErrorResponse response = + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + partitionName, + String.format( + "Partition %s already exists.", partitionName), + 409); + return mockResponse(response, 409); + } + } + List> created = new ArrayList<>(); + List> existed = new ArrayList<>(); + for (Map spec : request.getPartitionSpecs()) { + if (existingSpecs.add(spec)) { + storedPartitions.add(new Partition(spec, 0, 0, 0, 0, -1, false)); + created.add(spec); + } else { + existed.add(spec); + } + } + return mockResponse(new CreatePartitionsResponse(created, existed), 200); default: return new MockResponse().setResponseCode(404); } } + private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifier) + throws Exception { + DropPartitionsRequest request = RESTApi.fromJson(data, DropPartitionsRequest.class); + List storedPartitions = + tablePartitionsStore.computeIfAbsent( + tableIdentifier.getFullName(), ignored -> new ArrayList<>()); + Set> existingSpecs = + storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); + List> missing = new ArrayList<>(); + for (Map spec : request.getPartitionSpecs()) { + if (!existingSpecs.contains(spec)) { + missing.add(spec); + } + } + if (!request.ignoreIfNotExists() && !missing.isEmpty()) { + List missingNames = + missing.stream() + .map(PartitionUtils::buildPartitionName) + .collect(Collectors.toList()); + ErrorResponse response = + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + missingNames.get(0), + String.format("Partitions %s do not exist.", missingNames), + 404); + return mockResponse(response, 404); + } + List> dropped = new ArrayList<>(); + Set> toDrop = new HashSet<>(request.getPartitionSpecs()); + storedPartitions.removeIf( + partition -> { + if (toDrop.contains(partition.spec())) { + dropped.add(partition.spec()); + return true; + } + return false; + }); + return mockResponse(new DropPartitionsResponse(dropped, missing), 200); + } + private MockResponse listPartitionsByNames( Map parameters, Identifier tableIdentifier, diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 4cac0d13c98e..2de1b1f90be1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -64,9 +64,13 @@ import org.apache.paimon.predicate.UpperTransform; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.rest.auth.DLFToken; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.exceptions.BadRequestException; import org.apache.paimon.rest.exceptions.ForbiddenException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.GetTagResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -106,6 +110,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; import java.io.File; import java.io.IOException; @@ -1530,6 +1535,111 @@ void testListPartitionsFromFile() throws Exception { assertEquals(0, result.size()); } + @Test + void testCreatePartitionsForManagedFormatTable() throws Exception { + Identifier identifier = Identifier.create("format_partition_db", "managed_table"); + catalog.createDatabase(identifier.getDatabaseName(), true); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + false); + + List> partitionSpecs = + Arrays.asList(singletonMap("dt", "20260714"), singletonMap("dt", "20260715")); + CreatePartitionsResponse response = + restCatalog.api().createPartitions(identifier, partitionSpecs); + + assertThat(response.getCreated()).containsExactlyInAnyOrderElementsOf(partitionSpecs); + assertThat(response.getExisted()).isEmpty(); + + catalog.createPartitions(identifier, partitionSpecs); + catalog.createPartitions(identifier, partitionSpecs); + + List> conflictingSpecs = + Arrays.asList(partitionSpecs.get(0), singletonMap("dt", "20260716")); + assertThatThrownBy( + () -> + restCatalog + .api() + .createPartitions(identifier, conflictingSpecs, false)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("dt=20260714"); + + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrderElementsOf(partitionSpecs); + } + + @Test + void testDropPartitionsForManagedFormatTable() throws Exception { + Identifier identifier = Identifier.create("format_partition_db", "managed_drop_table"); + catalog.createDatabase(identifier.getDatabaseName(), true); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + false); + + List> partitionSpecs = + Arrays.asList(singletonMap("dt", "20260714"), singletonMap("dt", "20260715")); + catalog.createPartitions(identifier, partitionSpecs); + + DropPartitionsResponse response = + restCatalog + .api() + .dropPartitions( + identifier, + Arrays.asList( + singletonMap("dt", "20260714"), + singletonMap("dt", "20260799")), + true); + assertThat(response.getDropped()).containsExactly(singletonMap("dt", "20260714")); + assertThat(response.getMissing()).containsExactly(singletonMap("dt", "20260799")); + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactly(singletonMap("dt", "20260715")); + + // Catalog contract: dropPartitions ignores non-existent partitions and unregisters + // metadata only (no data deletion on the server). + catalog.dropPartitions( + identifier, + Arrays.asList(singletonMap("dt", "20260715"), singletonMap("dt", "20260799"))); + assertThat(catalog.listPartitions(identifier)).isEmpty(); + + assertThatThrownBy( + () -> + restCatalog + .api() + .dropPartitions( + identifier, + singletonList(singletonMap("dt", "20260799")), + false)) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("dt=20260799"); + } + + @Test + void testDropPartitionsLoadsNonManagedTableOnce() throws Exception { + Identifier identifier = Identifier.create("test_db", "drop_partition_table"); + createTable(identifier, emptyMap(), singletonList("col1")); + RESTCatalog catalogSpy = Mockito.spy(restCatalog); + + catalogSpy.dropPartitions(identifier, singletonList(singletonMap("col1", "20260717"))); + + Mockito.verify(catalogSpy, Mockito.times(1)).getTable(identifier); + } + @Test void testListPartitions() throws Exception { innerTestListPartitions(true);