From 9679e3cb0779b2cdf887fd884b92c1db5c21036c Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Fri, 17 Jul 2026 00:42:56 +0800 Subject: [PATCH 1/4] [core][rest] Add managed partition REST and catalog contract Define the managed partition wire format, fail-closed catalog APIs, and REST create/drop behavior. Keep server-side DROP metadata-only so data deletion remains a caller responsibility. --- docs/static/rest-catalog-open-api.yaml | 137 +++++++++++++++++- .../java/org/apache/paimon/rest/RESTApi.java | 35 +++++ .../org/apache/paimon/rest/ResourcePaths.java | 12 ++ .../requests/CreatePartitionsRequest.java | 67 +++++++++ .../rest/requests/DropPartitionsRequest.java | 67 +++++++++ .../responses/CreatePartitionsResponse.java | 60 ++++++++ .../responses/DropPartitionsResponse.java | 60 ++++++++ .../paimon/rest/responses/ErrorResponse.java | 2 + .../org/apache/paimon/catalog/Catalog.java | 64 ++++++++ .../paimon/catalog/DelegateCatalog.java | 30 ++++ .../org/apache/paimon/rest/RESTCatalog.java | 90 +++++++++++- .../paimon/rest/MockRESTCatalogTest.java | 9 ++ .../apache/paimon/rest/RESTApiJsonTest.java | 80 ++++++++++ .../apache/paimon/rest/RESTCatalogServer.java | 95 +++++++++++- .../apache/paimon/rest/RESTCatalogTest.java | 98 +++++++++++++ 15 files changed, 898 insertions(+), 8 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index e310f398f821..ae58d24cd1a3 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -983,6 +983,90 @@ paths: $ref: '#/components/responses/TableNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + post: + tags: + - partition + summary: Create partitions + operationId: createPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionsRequest' + responses: + "200": + description: Partitions created or found to exist + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePartitionsResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TableNotExistErrorResponse' + "409": + $ref: '#/components/responses/ResourceAlreadyExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}/partitions/drop: + post: + tags: + - partition + summary: Drop partitions + description: Unregisters partitions from the catalog. The server never deletes data files; managed format table data deletion is performed by the client afterwards. + operationId: dropPartitions + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: table + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DropPartitionsRequest' + responses: + "200": + description: Partitions dropped or found missing + content: + application/json: + schema: + $ref: '#/components/schemas/DropPartitionsResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TableNotExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/tables/{table}/partitions/mark: post: tags: @@ -2352,11 +2436,60 @@ components: type: string DropPartitionsRequest: type: object + required: + - partitionSpecs properties: - specs: + partitionSpecs: type: array items: type: object + additionalProperties: + type: string + ignoreIfNotExists: + type: boolean + default: true + DropPartitionsResponse: + type: object + required: + - dropped + - missing + properties: + dropped: + type: array + items: + type: string + missing: + type: array + items: + type: string + CreatePartitionsRequest: + type: object + required: + - partitionSpecs + properties: + partitionSpecs: + type: array + items: + type: object + additionalProperties: + type: string + ignoreIfExists: + type: boolean + default: true + CreatePartitionsResponse: + type: object + required: + - created + - existed + properties: + created: + type: array + items: + type: string + existed: + type: array + items: + type: string AlterTableRequest: type: object properties: @@ -2398,7 +2531,7 @@ components: resourceType: type: string nullable: true - enum: [ "DATABASE", "TABLE", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "DIALECT", "UNKNOWN" ] + enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "DIALECT", "UNKNOWN" ] resourceName: type: string nullable: true diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 263c4e2c0640..4d321ed2797e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -42,9 +42,11 @@ import org.apache.paimon.rest.requests.CreateBranchRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; @@ -58,6 +60,8 @@ import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -855,6 +859,37 @@ public void markDonePartitions(Identifier identifier, List> restAuthFunction); } + /** Create partitions for table, ignoring partitions which already exist. */ + public CreatePartitionsResponse createPartitions( + Identifier identifier, List> partitions) { + return createPartitions(identifier, partitions, true); + } + + /** Create partitions for table. */ + public CreatePartitionsResponse createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) { + CreatePartitionsRequest request = new CreatePartitionsRequest(partitions, ignoreIfExists); + return client.post( + resourcePaths.partitions(identifier.getDatabaseName(), identifier.getObjectName()), + request, + CreatePartitionsResponse.class, + restAuthFunction); + } + + /** Drop (unregister) partitions for table; the server never deletes data files. */ + public DropPartitionsResponse dropPartitions( + Identifier identifier, + List> partitions, + boolean ignoreIfNotExists) { + DropPartitionsRequest request = new DropPartitionsRequest(partitions, ignoreIfNotExists); + return client.post( + resourcePaths.dropPartitions( + identifier.getDatabaseName(), identifier.getObjectName()), + request, + DropPartitionsResponse.class, + restAuthFunction); + } + /** * List partitions for table. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 28f79d040995..0ab36e348c6e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -213,6 +213,18 @@ public String partitions(String databaseName, String objectName) { PARTITIONS); } + public String dropPartitions(String databaseName, String objectName) { + return SLASH.join( + V1, + prefix, + DATABASES, + encodeString(databaseName), + TABLES, + encodeString(objectName), + PARTITIONS, + "drop"); + } + public String markDonePartitions(String databaseName, String objectName) { return SLASH.join( V1, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java new file mode 100644 index 000000000000..506c66406eec --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreatePartitionsRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; + +/** Request for creating partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreatePartitionsRequest implements RESTRequest { + + private static final String FIELD_PARTITION_SPECS = "partitionSpecs"; + private static final String FIELD_IGNORE_IF_EXISTS = "ignoreIfExists"; + + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; + + @JsonProperty(FIELD_IGNORE_IF_EXISTS) + private final boolean ignoreIfExists; + + public CreatePartitionsRequest(List> partitionSpecs) { + this(partitionSpecs, true); + } + + @JsonCreator + public CreatePartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs, + @JsonProperty(FIELD_IGNORE_IF_EXISTS) @Nullable Boolean ignoreIfExists) { + this.partitionSpecs = partitionSpecs; + this.ignoreIfExists = ignoreIfExists == null || ignoreIfExists; + } + + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; + } + + @JsonGetter(FIELD_IGNORE_IF_EXISTS) + public boolean ignoreIfExists() { + return ignoreIfExists; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java new file mode 100644 index 000000000000..cdc448034e38 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPartitionsRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; + +/** Request for dropping (unregistering) partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionsRequest implements RESTRequest { + + private static final String FIELD_PARTITION_SPECS = "partitionSpecs"; + private static final String FIELD_IGNORE_IF_NOT_EXISTS = "ignoreIfNotExists"; + + @JsonProperty(FIELD_PARTITION_SPECS) + private final List> partitionSpecs; + + @JsonProperty(FIELD_IGNORE_IF_NOT_EXISTS) + private final boolean ignoreIfNotExists; + + public DropPartitionsRequest(List> partitionSpecs) { + this(partitionSpecs, true); + } + + @JsonCreator + public DropPartitionsRequest( + @JsonProperty(FIELD_PARTITION_SPECS) List> partitionSpecs, + @JsonProperty(FIELD_IGNORE_IF_NOT_EXISTS) @Nullable Boolean ignoreIfNotExists) { + this.partitionSpecs = partitionSpecs; + this.ignoreIfNotExists = ignoreIfNotExists == null || ignoreIfNotExists; + } + + @JsonGetter(FIELD_PARTITION_SPECS) + public List> getPartitionSpecs() { + return partitionSpecs; + } + + @JsonGetter(FIELD_IGNORE_IF_NOT_EXISTS) + public boolean ignoreIfNotExists() { + return ignoreIfNotExists; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java new file mode 100644 index 000000000000..ef612f889e68 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** Response for creating partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreatePartitionsResponse implements RESTResponse { + + private static final String FIELD_CREATED = "created"; + private static final String FIELD_EXISTED = "existed"; + + @JsonProperty(FIELD_CREATED) + private final List created; + + @JsonProperty(FIELD_EXISTED) + private final List existed; + + @JsonCreator + public CreatePartitionsResponse( + @JsonProperty(FIELD_CREATED) List created, + @JsonProperty(FIELD_EXISTED) List existed) { + this.created = created; + this.existed = existed; + } + + @JsonGetter(FIELD_CREATED) + public List getCreated() { + return created; + } + + @JsonGetter(FIELD_EXISTED) + public List getExisted() { + return existed; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java new file mode 100644 index 000000000000..57debbd7ee25 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** Response for dropping (unregistering) partitions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPartitionsResponse implements RESTResponse { + + private static final String FIELD_DROPPED = "dropped"; + private static final String FIELD_MISSING = "missing"; + + @JsonProperty(FIELD_DROPPED) + private final List dropped; + + @JsonProperty(FIELD_MISSING) + private final List missing; + + @JsonCreator + public DropPartitionsResponse( + @JsonProperty(FIELD_DROPPED) List dropped, + @JsonProperty(FIELD_MISSING) List missing) { + this.dropped = dropped; + this.missing = missing; + } + + @JsonGetter(FIELD_DROPPED) + public List getDropped() { + return dropped; + } + + @JsonGetter(FIELD_MISSING) + public List getMissing() { + return missing; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java index d4f9662dfa28..4eb52308dd0c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java @@ -35,6 +35,8 @@ public class ErrorResponse implements RESTResponse { public static final String RESOURCE_TYPE_TABLE = "TABLE"; + public static final String RESOURCE_TYPE_PARTITION = "PARTITION"; + public static final String RESOURCE_TYPE_COLUMN = "COLUMN"; public static final String RESOURCE_TYPE_SNAPSHOT = "SNAPSHOT"; diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 57fa040a2acd..344282af52f2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -423,6 +423,24 @@ PagedList listPartitionsPaged( @Nullable String partitionNamePattern) throws TableNotExistException; + /** + * Get a page of partitions without falling back to filesystem discovery. + * + *

This entry point is intended for tables whose partition visibility is owned by the + * catalog. Implementations must fail when the catalog service cannot list partitions. + */ + default PagedList listPartitionsPagedWithoutFallback( + Identifier identifier, + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String partitionNamePattern) + throws TableNotExistException { + throw new UnsupportedOperationException( + String.format( + "Catalog %s does not support managed partition listing.", + getClass().getName())); + } + /** * Get Partition list by partition names of the table. * @@ -436,6 +454,21 @@ List listPartitionsByNames( Identifier identifier, List> partitions) throws TableNotExistException; + /** + * Get partitions by partition names without falling back to filesystem discovery. + * + *

This entry point is intended for tables whose partition visibility is owned by the + * catalog. Implementations must fail when the catalog service cannot list partitions. + */ + default List listPartitionsByNamesWithoutFallback( + Identifier identifier, List> partitions) + throws TableNotExistException { + throw new UnsupportedOperationException( + String.format( + "Catalog %s does not support managed partition listing.", + getClass().getName())); + } + // ======================= view methods =============================== /** @@ -1024,6 +1057,16 @@ void deleteTag(Identifier identifier, String tagName) */ boolean supportsPartitionModification(); + /** + * Whether this catalog supports managed format table partitions: fail-closed catalog-owned + * partition listing plus the {@link #createPartitions(Identifier, List)} and {@link + * #dropPartitions(Identifier, List)} registration DDL for such tables, independent of {@link + * #supportsPartitionModification()}. + */ + default boolean supportsManagedPartitionListing() { + return false; + } + /** * Create partitions of the specify table. Ignore existing partitions. * @@ -1034,6 +1077,27 @@ void deleteTag(Identifier identifier, String tagName) default void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException {} + /** + * Create partitions of the specify table with explicit existence semantics. + * + * @param identifier path of the table to create partitions + * @param partitions partitions to be created + * @param ignoreIfExists if false, fail when any partition already exists and apply none of the + * batch; if true, behave like {@link #createPartitions(Identifier, List)} + * @throws TableNotExistException if the table does not exist + */ + default void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + if (!ignoreIfExists) { + throw new UnsupportedOperationException( + String.format( + "Catalog %s does not support createPartitions without ignoreIfExists.", + getClass().getName())); + } + createPartitions(identifier, partitions); + } + /** * Drop partitions of the specify table. Ignore non-existent partitions. * diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 0f18f7d04540..ead764750b32 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -316,12 +316,24 @@ public boolean supportsPartitionModification() { return wrapped.supportsPartitionModification(); } + @Override + public boolean supportsManagedPartitionListing() { + return wrapped.supportsManagedPartitionListing(); + } + @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { wrapped.createPartitions(identifier, partitions); } + @Override + public void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + wrapped.createPartitions(identifier, partitions, ignoreIfExists); + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { @@ -448,6 +460,17 @@ public PagedList listPartitionsPaged( return wrapped.listPartitionsPaged(identifier, maxResults, pageToken, partitionNamePattern); } + @Override + public PagedList listPartitionsPagedWithoutFallback( + Identifier identifier, + Integer maxResults, + String pageToken, + String partitionNamePattern) + throws TableNotExistException { + return wrapped.listPartitionsPagedWithoutFallback( + identifier, maxResults, pageToken, partitionNamePattern); + } + @Override public List listPartitionsByNames( Identifier identifier, List> partitions) @@ -455,6 +478,13 @@ public List listPartitionsByNames( return wrapped.listPartitionsByNames(identifier, partitions); } + @Override + public List listPartitionsByNamesWithoutFallback( + Identifier identifier, List> partitions) + throws TableNotExistException { + return wrapped.listPartitionsByNamesWithoutFallback(identifier, partitions); + } + @Override public TableQueryAuthResult authTableQuery(Identifier identifier, @Nullable List select) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 0a76fb5a7848..1996d4dad2f4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -59,6 +59,7 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Instant; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; @@ -725,6 +726,56 @@ public void markDonePartitions(Identifier identifier, List> } } + @Override + public void createPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + createPartitions(identifier, partitions, true); + } + + @Override + public void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + try { + api.createPartitions(identifier, partitions, ignoreIfExists); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } catch (AlreadyExistsException e) { + // Server contract: with ignoreIfExists=false the whole batch is rejected atomically. + throw new IllegalStateException( + String.format( + "Some partitions of table %s already exist: %s", + identifier, e.getMessage())); + } catch (BadRequestException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + + @Override + public void dropPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + Table table = getTable(identifier); + if (table instanceof FormatTable + && new CoreOptions(table.options()).partitionedTableInMetastore()) { + // Managed format table: metadata-only unregistration on the server. Data deletion is + // the caller's responsibility (performed with the table FileIO afterwards). + try { + api.dropPartitions(identifier, partitions, true); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } catch (BadRequestException e) { + throw new IllegalArgumentException(e.getMessage()); + } + return; + } + // Non-managed tables keep the default truncate-based data semantics. + Catalog.super.dropPartitions(identifier, partitions); + } + @Override public List listPartitions(Identifier identifier) throws TableNotExistException { try { @@ -746,15 +797,28 @@ public PagedList listPartitionsPaged( @Nullable String pageToken, @Nullable String partitionNamePattern) throws TableNotExistException { + try { + return listPartitionsPagedWithoutFallback( + identifier, maxResults, pageToken, partitionNamePattern); + } catch (NotImplementedException e) { + // not a metastore partitioned table + return new PagedList<>(listPartitionsFromFileSystem(getTable(identifier)), null); + } + } + + @Override + public PagedList listPartitionsPagedWithoutFallback( + Identifier identifier, + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String partitionNamePattern) + throws TableNotExistException { try { return api.listPartitionsPaged(identifier, maxResults, pageToken, partitionNamePattern); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); - } catch (NotImplementedException e) { - // not a metastore partitioned table - return new PagedList<>(listPartitionsFromFileSystem(getTable(identifier)), null); } } @@ -773,6 +837,19 @@ public List listPartitionsByNames( } } + @Override + public List listPartitionsByNamesWithoutFallback( + Identifier identifier, List> partitions) + throws TableNotExistException { + try { + return api.listPartitionsByNames(identifier, partitions); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier); + } catch (ForbiddenException e) { + throw new TableNoPermissionException(identifier, e); + } + } + @Override public void createBranch(Identifier identifier, String branch, @Nullable String fromTag) throws TableNotExistException, BranchAlreadyExistException, TagNotExistException { @@ -836,9 +913,16 @@ public List listBranches(Identifier identifier) throws TableNotExistExce @Override public boolean supportsPartitionModification() { + // Upstream parity: REST paimon tables keep commit-based partition maintenance. Managed + // format table partition DDL is capability-gated by supportsManagedPartitionListing(). return false; } + @Override + public boolean supportsManagedPartitionListing() { + return true; + } + @Override public List listFunctions(String databaseName) throws DatabaseNotExistException { try { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index f5045a4360a6..a980ac4be63a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -214,6 +214,15 @@ void testCreateTableDefaultOptions() throws Exception { "default-value"); } + @Test + void testPartitionCapabilities() { + // Paimon tables keep upstream commit-based partition maintenance; managed format table + // partitions ride their own capability so partition expire never routes through the + // truncate-based Catalog#dropPartitions default (it cannot represent .done markers). + assertThat(restCatalog.supportsPartitionModification()).isFalse(); + assertThat(restCatalog.supportsManagedPartitionListing()).isTrue(); + } + @Test void testBaseHeadersInRequests() throws Exception { // Set custom headers in options diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java index 1b8b1f71d0c0..71c394906013 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java @@ -24,13 +24,17 @@ import org.apache.paimon.rest.requests.AlterViewRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -50,10 +54,13 @@ import org.junit.Test; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** Test for {@link RESTApi} json. */ @@ -204,6 +211,79 @@ public void listPartitionsResponseParseTest() throws Exception { parseData.getPartitions().get(0).fileCount()); } + @Test + public void createPartitionsResponseParseTest() throws Exception { + CreatePartitionsResponse response = + new CreatePartitionsResponse( + Arrays.asList("dt=20260714", "dt=20260715"), + Collections.singletonList("dt=20260713")); + + CreatePartitionsResponse parsed = + RESTApi.fromJson(RESTApi.toJson(response), CreatePartitionsResponse.class); + + assertEquals(response.getCreated(), parsed.getCreated()); + assertEquals(response.getExisted(), parsed.getExisted()); + } + + @Test + public void createPartitionsRequestParseTest() throws Exception { + String requestWithoutIgnoreIfExists = "{\"partitionSpecs\":[{\"dt\":\"20260715\"}]}"; + CreatePartitionsRequest defaultRequest = + RESTApi.fromJson(requestWithoutIgnoreIfExists, CreatePartitionsRequest.class); + + assertEquals( + Collections.singletonList(Collections.singletonMap("dt", "20260715")), + defaultRequest.getPartitionSpecs()); + assertTrue(defaultRequest.ignoreIfExists()); + + CreatePartitionsRequest explicitRequest = + new CreatePartitionsRequest(defaultRequest.getPartitionSpecs(), false); + String explicitRequestJson = RESTApi.toJson(explicitRequest); + assertTrue(explicitRequestJson.contains("\"partitionSpecs\"")); + assertFalse(explicitRequestJson.contains("\"specs\"")); + assertTrue(explicitRequestJson.contains("\"ignoreIfExists\":false")); + + CreatePartitionsRequest parsedExplicitRequest = + RESTApi.fromJson(explicitRequestJson, CreatePartitionsRequest.class); + assertFalse(parsedExplicitRequest.ignoreIfExists()); + } + + @Test + public void dropPartitionsResponseParseTest() throws Exception { + DropPartitionsResponse response = + new DropPartitionsResponse( + Arrays.asList("dt=20260714", "dt=20260715"), + Collections.singletonList("dt=20260713")); + + DropPartitionsResponse parsed = + RESTApi.fromJson(RESTApi.toJson(response), DropPartitionsResponse.class); + + assertEquals(response.getDropped(), parsed.getDropped()); + assertEquals(response.getMissing(), parsed.getMissing()); + } + + @Test + public void dropPartitionsRequestParseTest() throws Exception { + String requestWithoutIgnoreIfNotExists = "{\"partitionSpecs\":[{\"dt\":\"20260715\"}]}"; + DropPartitionsRequest defaultRequest = + RESTApi.fromJson(requestWithoutIgnoreIfNotExists, DropPartitionsRequest.class); + + assertEquals( + Collections.singletonList(Collections.singletonMap("dt", "20260715")), + defaultRequest.getPartitionSpecs()); + assertTrue(defaultRequest.ignoreIfNotExists()); + + DropPartitionsRequest explicitRequest = + new DropPartitionsRequest(defaultRequest.getPartitionSpecs(), false); + String explicitRequestJson = RESTApi.toJson(explicitRequest); + assertTrue(explicitRequestJson.contains("\"partitionSpecs\"")); + assertTrue(explicitRequestJson.contains("\"ignoreIfNotExists\":false")); + + DropPartitionsRequest parsedExplicitRequest = + RESTApi.fromJson(explicitRequestJson, DropPartitionsRequest.class); + assertFalse(parsedExplicitRequest.ignoreIfNotExists()); + } + @Test public void createViewRequestParseTest() throws Exception { CreateViewRequest request = MockRESTMessage.createViewRequest("t1"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index af5d94e3f632..c0929184a6c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -56,9 +56,11 @@ import org.apache.paimon.rest.requests.CreateBranchRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; +import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -70,6 +72,8 @@ import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; @@ -449,6 +453,11 @@ && isTableByIdRequest(request.getPath())) { && ResourcePaths.TABLES.equals(resources[1]) && "partitions".equals(resources[3]) && "list-by-names".equals(resources[4]); + boolean isDropPartitions = + resources.length == 5 + && ResourcePaths.TABLES.equals(resources[1]) + && "partitions".equals(resources[3]) + && "drop".equals(resources[4]); boolean isBranches = resources.length >= 4 @@ -479,7 +488,7 @@ && isTableByIdRequest(request.getPath())) { } } // validate partition - if (isPartitions || isMarkDonePartitions) { + if (isPartitions || isMarkDonePartitions || isDropPartitions) { String tableName = RESTUtil.decodeString(resources[2]); Optional error = checkTablePartitioned( @@ -494,9 +503,14 @@ && isTableByIdRequest(request.getPath())) { catalog.markDonePartitions( identifier, markDonePartitionsRequest.getPartitionSpecs()); return new MockResponse().setResponseCode(200); + } else if (isDropPartitions) { + return dropPartitionsHandle(restAuthParameter.data(), identifier); } else if (isPartitions) { return partitionsApiHandle( - restAuthParameter.method(), parameters, identifier); + restAuthParameter.method(), + restAuthParameter.data(), + parameters, + identifier); } else if (isListPartitionsByNames) { ListPartitionsByNamesRequest listPartitionsByNamesRequest = RESTApi.fromJson(data, ListPartitionsByNamesRequest.class); @@ -1815,7 +1829,8 @@ private MockResponse renameTableHandle(String data) throws Exception { } private MockResponse partitionsApiHandle( - String method, Map parameters, Identifier tableIdentifier) { + String method, String data, Map parameters, Identifier tableIdentifier) + throws Exception { String partitionNamePattern = parameters.get(PARTITION_NAME_PATTERN); switch (method) { case "GET": @@ -1835,11 +1850,85 @@ private MockResponse partitionsApiHandle( } } return generateFinalListPartitionsResponse(parameters, partitions); + case "POST": + CreatePartitionsRequest request = + RESTApi.fromJson(data, CreatePartitionsRequest.class); + List storedPartitions = + tablePartitionsStore.computeIfAbsent( + tableIdentifier.getFullName(), ignored -> new ArrayList<>()); + Set> existingSpecs = + storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); + if (!request.ignoreIfExists()) { + Set> seenSpecs = new HashSet<>(existingSpecs); + Optional> conflictingSpec = + request.getPartitionSpecs().stream() + .filter(spec -> !seenSpecs.add(spec)) + .findFirst(); + if (conflictingSpec.isPresent()) { + String partitionName = + PartitionUtils.buildPartitionName(conflictingSpec.get()); + ErrorResponse response = + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + partitionName, + String.format( + "Partition %s already exists.", partitionName), + 409); + return mockResponse(response, 409); + } + } + List created = new ArrayList<>(); + List existed = new ArrayList<>(); + for (Map spec : request.getPartitionSpecs()) { + if (existingSpecs.add(spec)) { + storedPartitions.add(new Partition(spec, 0, 0, 0, 0, -1, false)); + created.add(PartitionUtils.buildPartitionName(spec)); + } else { + existed.add(PartitionUtils.buildPartitionName(spec)); + } + } + return mockResponse(new CreatePartitionsResponse(created, existed), 200); default: return new MockResponse().setResponseCode(404); } } + private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifier) + throws Exception { + DropPartitionsRequest request = RESTApi.fromJson(data, DropPartitionsRequest.class); + List storedPartitions = + tablePartitionsStore.computeIfAbsent( + tableIdentifier.getFullName(), ignored -> new ArrayList<>()); + Set> existingSpecs = + storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); + List missing = new ArrayList<>(); + for (Map spec : request.getPartitionSpecs()) { + if (!existingSpecs.contains(spec)) { + missing.add(PartitionUtils.buildPartitionName(spec)); + } + } + if (!request.ignoreIfNotExists() && !missing.isEmpty()) { + ErrorResponse response = + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_PARTITION, + missing.get(0), + String.format("Partitions %s do not exist.", missing), + 404); + return mockResponse(response, 404); + } + List dropped = new ArrayList<>(); + Set> toDrop = new HashSet<>(request.getPartitionSpecs()); + storedPartitions.removeIf( + partition -> { + if (toDrop.contains(partition.spec())) { + dropped.add(PartitionUtils.buildPartitionName(partition.spec())); + return true; + } + return false; + }); + return mockResponse(new DropPartitionsResponse(dropped, missing), 200); + } + private MockResponse listPartitionsByNames( Map parameters, Identifier tableIdentifier, diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 4cac0d13c98e..e7783835ebe7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -64,9 +64,13 @@ import org.apache.paimon.predicate.UpperTransform; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.rest.auth.DLFToken; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.exceptions.BadRequestException; import org.apache.paimon.rest.exceptions.ForbiddenException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.GetTagResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -1530,6 +1534,100 @@ void testListPartitionsFromFile() throws Exception { assertEquals(0, result.size()); } + @Test + void testCreatePartitionsForManagedFormatTable() throws Exception { + Identifier identifier = Identifier.create("format_partition_db", "managed_table"); + catalog.createDatabase(identifier.getDatabaseName(), true); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + false); + + List> partitionSpecs = + Arrays.asList(singletonMap("dt", "20260714"), singletonMap("dt", "20260715")); + CreatePartitionsResponse response = + restCatalog.api().createPartitions(identifier, partitionSpecs); + + assertThat(response.getCreated()).containsExactlyInAnyOrder("dt=20260714", "dt=20260715"); + assertThat(response.getExisted()).isEmpty(); + + catalog.createPartitions(identifier, partitionSpecs); + catalog.createPartitions(identifier, partitionSpecs); + + List> conflictingSpecs = + Arrays.asList(partitionSpecs.get(0), singletonMap("dt", "20260716")); + assertThatThrownBy( + () -> + restCatalog + .api() + .createPartitions(identifier, conflictingSpecs, false)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("dt=20260714"); + + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactlyInAnyOrderElementsOf(partitionSpecs); + } + + @Test + void testDropPartitionsForManagedFormatTable() throws Exception { + Identifier identifier = Identifier.create("format_partition_db", "managed_drop_table"); + catalog.createDatabase(identifier.getDatabaseName(), true); + catalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + false); + + List> partitionSpecs = + Arrays.asList(singletonMap("dt", "20260714"), singletonMap("dt", "20260715")); + catalog.createPartitions(identifier, partitionSpecs); + + DropPartitionsResponse response = + restCatalog + .api() + .dropPartitions( + identifier, + Arrays.asList( + singletonMap("dt", "20260714"), + singletonMap("dt", "20260799")), + true); + assertThat(response.getDropped()).containsExactly("dt=20260714"); + assertThat(response.getMissing()).containsExactly("dt=20260799"); + assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) + .containsExactly(singletonMap("dt", "20260715")); + + // Catalog contract: dropPartitions ignores non-existent partitions and unregisters + // metadata only (no data deletion on the server). + catalog.dropPartitions( + identifier, + Arrays.asList(singletonMap("dt", "20260715"), singletonMap("dt", "20260799"))); + assertThat(catalog.listPartitions(identifier)).isEmpty(); + + assertThatThrownBy( + () -> + restCatalog + .api() + .dropPartitions( + identifier, + singletonList(singletonMap("dt", "20260799")), + false)) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("dt=20260799"); + } + @Test void testListPartitions() throws Exception { innerTestListPartitions(true); From 498b04468dea0d2c2365014c47288c4f63d5fe6b Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Fri, 17 Jul 2026 12:03:53 +0800 Subject: [PATCH 2/4] [core][rest] Refine managed partition catalog contract --- .../apache/paimon/catalog/CachingCatalog.java | 19 +++++++++++ .../org/apache/paimon/catalog/Catalog.java | 2 +- .../paimon/catalog/DelegateCatalog.java | 4 +-- .../org/apache/paimon/rest/RESTCatalog.java | 13 ++++--- .../paimon/catalog/CachingCatalogTest.java | 34 +++++++++++++++++++ .../paimon/rest/MockRESTCatalogTest.java | 6 ++-- .../apache/paimon/rest/RESTCatalogTest.java | 12 +++++++ 7 files changed, 80 insertions(+), 10 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index afe4ed8ae6de..01f04cbba3ae 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -335,6 +335,25 @@ public List listPartitions(Identifier identifier) throws TableNotExis return result; } + @Override + public void createPartitions(Identifier identifier, List> partitions) + throws TableNotExistException { + wrapped.createPartitions(identifier, partitions); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + + @Override + public void createPartitions( + Identifier identifier, List> partitions, boolean ignoreIfExists) + throws TableNotExistException { + wrapped.createPartitions(identifier, partitions, ignoreIfExists); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } + } + @Override public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 344282af52f2..b88e4d76edcf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1063,7 +1063,7 @@ void deleteTag(Identifier identifier, String tagName) * #dropPartitions(Identifier, List)} registration DDL for such tables, independent of {@link * #supportsPartitionModification()}. */ - default boolean supportsManagedPartitionListing() { + default boolean supportsManagedFormatTablePartitions() { return false; } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index ead764750b32..f94de3242731 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -317,8 +317,8 @@ public boolean supportsPartitionModification() { } @Override - public boolean supportsManagedPartitionListing() { - return wrapped.supportsManagedPartitionListing(); + public boolean supportsManagedFormatTablePartitions() { + return wrapped.supportsManagedFormatTablePartitions(); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 1996d4dad2f4..1d9715ad19f0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -63,6 +63,7 @@ import org.apache.paimon.table.Instant; import org.apache.paimon.table.Table; import org.apache.paimon.table.TableSnapshot; +import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotNotExistException; @@ -773,7 +774,11 @@ && new CoreOptions(table.options()).partitionedTableInMetastore()) { return; } // Non-managed tables keep the default truncate-based data semantics. - Catalog.super.dropPartitions(identifier, partitions); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.truncatePartitions(partitions); + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override @@ -913,13 +918,13 @@ public List listBranches(Identifier identifier) throws TableNotExistExce @Override public boolean supportsPartitionModification() { - // Upstream parity: REST paimon tables keep commit-based partition maintenance. Managed - // format table partition DDL is capability-gated by supportsManagedPartitionListing(). + // Paimon tables use commit-based partition maintenance. Managed Format Tables use the + // separate catalog-owned partition contract. return false; } @Override - public boolean supportsManagedPartitionListing() { + public boolean supportsManagedFormatTablePartitions() { return true; } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index c28299b79e38..0961050f110d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -56,6 +56,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,6 +66,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.apache.paimon.data.BinaryString.fromString; import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY; @@ -334,6 +336,38 @@ public void testPartitionCache() throws Exception { assertThat(partitionEntryListFromCache).containsAll(partitionEntryList); } + @Test + public void testCreatePartitionsInvalidatesPartitionCache() throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + when(wrapped.listPartitions(identifier)).thenReturn(emptyList(), singletonList(created)); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + catalog.createPartitions(identifier, singletonList(spec)); + + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + + @Test + public void testCreatePartitionsWithIgnoreIfExistsInvalidatesPartitionCache() throws Exception { + Catalog wrapped = Mockito.mock(Catalog.class); + TestableCachingCatalog catalog = + new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker); + Identifier identifier = new Identifier("db", "tbl"); + Map spec = singletonMap("dt", "20260717"); + Partition created = new Partition(spec, 0, 0, 0, 0, -1, false); + when(wrapped.listPartitions(identifier)).thenReturn(emptyList(), singletonList(created)); + + assertThat(catalog.listPartitions(identifier)).isEmpty(); + catalog.createPartitions(identifier, singletonList(spec), false); + + assertThat(catalog.listPartitions(identifier)).containsExactly(created); + } + @Test public void testDeadlock() throws Exception { Catalog underlyCatalog = this.catalog; diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index a980ac4be63a..151cf8341546 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -216,11 +216,11 @@ void testCreateTableDefaultOptions() throws Exception { @Test void testPartitionCapabilities() { - // Paimon tables keep upstream commit-based partition maintenance; managed format table - // partitions ride their own capability so partition expire never routes through the + // Paimon tables use commit-based partition maintenance. Managed Format Tables use a + // separate capability so partition expiration never routes through the // truncate-based Catalog#dropPartitions default (it cannot represent .done markers). assertThat(restCatalog.supportsPartitionModification()).isFalse(); - assertThat(restCatalog.supportsManagedPartitionListing()).isTrue(); + assertThat(restCatalog.supportsManagedFormatTablePartitions()).isTrue(); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index e7783835ebe7..a75e171697a2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -110,6 +110,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; import java.io.File; import java.io.IOException; @@ -1628,6 +1629,17 @@ void testDropPartitionsForManagedFormatTable() throws Exception { .hasMessageContaining("dt=20260799"); } + @Test + void testDropPartitionsLoadsNonManagedTableOnce() throws Exception { + Identifier identifier = Identifier.create("test_db", "drop_partition_table"); + createTable(identifier, emptyMap(), singletonList("col1")); + RESTCatalog catalogSpy = Mockito.spy(restCatalog); + + catalogSpy.dropPartitions(identifier, singletonList(singletonMap("col1", "20260717"))); + + Mockito.verify(catalogSpy, Mockito.times(1)).getTable(identifier); + } + @Test void testListPartitions() throws Exception { innerTestListPartitions(true); From 4083b38aa3f534af507ecf8df38a3e8bb8baad52 Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Fri, 17 Jul 2026 13:58:45 +0800 Subject: [PATCH 3/4] [core][rest] Keep managed partition listing catalog-only --- .../org/apache/paimon/catalog/Catalog.java | 43 ------------- .../paimon/catalog/DelegateCatalog.java | 23 ------- .../org/apache/paimon/rest/RESTCatalog.java | 64 +++++++------------ .../paimon/rest/MockRESTCatalogTest.java | 53 +++++++++++++-- .../apache/paimon/rest/RESTCatalogServer.java | 10 +++ 5 files changed, 80 insertions(+), 113 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index b88e4d76edcf..8e4ad8d51815 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -423,24 +423,6 @@ PagedList listPartitionsPaged( @Nullable String partitionNamePattern) throws TableNotExistException; - /** - * Get a page of partitions without falling back to filesystem discovery. - * - *

This entry point is intended for tables whose partition visibility is owned by the - * catalog. Implementations must fail when the catalog service cannot list partitions. - */ - default PagedList listPartitionsPagedWithoutFallback( - Identifier identifier, - @Nullable Integer maxResults, - @Nullable String pageToken, - @Nullable String partitionNamePattern) - throws TableNotExistException { - throw new UnsupportedOperationException( - String.format( - "Catalog %s does not support managed partition listing.", - getClass().getName())); - } - /** * Get Partition list by partition names of the table. * @@ -454,21 +436,6 @@ List listPartitionsByNames( Identifier identifier, List> partitions) throws TableNotExistException; - /** - * Get partitions by partition names without falling back to filesystem discovery. - * - *

This entry point is intended for tables whose partition visibility is owned by the - * catalog. Implementations must fail when the catalog service cannot list partitions. - */ - default List listPartitionsByNamesWithoutFallback( - Identifier identifier, List> partitions) - throws TableNotExistException { - throw new UnsupportedOperationException( - String.format( - "Catalog %s does not support managed partition listing.", - getClass().getName())); - } - // ======================= view methods =============================== /** @@ -1057,16 +1024,6 @@ void deleteTag(Identifier identifier, String tagName) */ boolean supportsPartitionModification(); - /** - * Whether this catalog supports managed format table partitions: fail-closed catalog-owned - * partition listing plus the {@link #createPartitions(Identifier, List)} and {@link - * #dropPartitions(Identifier, List)} registration DDL for such tables, independent of {@link - * #supportsPartitionModification()}. - */ - default boolean supportsManagedFormatTablePartitions() { - return false; - } - /** * Create partitions of the specify table. Ignore existing partitions. * diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index f94de3242731..290849024fdb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -316,11 +316,6 @@ public boolean supportsPartitionModification() { return wrapped.supportsPartitionModification(); } - @Override - public boolean supportsManagedFormatTablePartitions() { - return wrapped.supportsManagedFormatTablePartitions(); - } - @Override public void createPartitions(Identifier identifier, List> partitions) throws TableNotExistException { @@ -460,17 +455,6 @@ public PagedList listPartitionsPaged( return wrapped.listPartitionsPaged(identifier, maxResults, pageToken, partitionNamePattern); } - @Override - public PagedList listPartitionsPagedWithoutFallback( - Identifier identifier, - Integer maxResults, - String pageToken, - String partitionNamePattern) - throws TableNotExistException { - return wrapped.listPartitionsPagedWithoutFallback( - identifier, maxResults, pageToken, partitionNamePattern); - } - @Override public List listPartitionsByNames( Identifier identifier, List> partitions) @@ -478,13 +462,6 @@ public List listPartitionsByNames( return wrapped.listPartitionsByNames(identifier, partitions); } - @Override - public List listPartitionsByNamesWithoutFallback( - Identifier identifier, List> partitions) - throws TableNotExistException { - return wrapped.listPartitionsByNamesWithoutFallback(identifier, partitions); - } - @Override public TableQueryAuthResult authTableQuery(Identifier identifier, @Nullable List select) throws TableNotExistException { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 1d9715ad19f0..c44378f6d00f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -758,8 +758,7 @@ public void createPartitions( public void dropPartitions(Identifier identifier, List> partitions) throws TableNotExistException { Table table = getTable(identifier); - if (table instanceof FormatTable - && new CoreOptions(table.options()).partitionedTableInMetastore()) { + if (isCatalogManagedFormatTable(table)) { // Managed format table: metadata-only unregistration on the server. Data deletion is // the caller's responsibility (performed with the table FileIO afterwards). try { @@ -790,8 +789,11 @@ public List listPartitions(Identifier identifier) throws TableNotExis } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } catch (NotImplementedException e) { - // not a metastore partitioned table - return listPartitionsFromFileSystem(getTable(identifier)); + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + throw e; + } + return listPartitionsFromFileSystem(table); } } @@ -802,28 +804,18 @@ public PagedList listPartitionsPaged( @Nullable String pageToken, @Nullable String partitionNamePattern) throws TableNotExistException { - try { - return listPartitionsPagedWithoutFallback( - identifier, maxResults, pageToken, partitionNamePattern); - } catch (NotImplementedException e) { - // not a metastore partitioned table - return new PagedList<>(listPartitionsFromFileSystem(getTable(identifier)), null); - } - } - - @Override - public PagedList listPartitionsPagedWithoutFallback( - Identifier identifier, - @Nullable Integer maxResults, - @Nullable String pageToken, - @Nullable String partitionNamePattern) - throws TableNotExistException { try { return api.listPartitionsPaged(identifier, maxResults, pageToken, partitionNamePattern); } catch (NoSuchResourceException e) { throw new TableNotExistException(identifier); } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); + } catch (NotImplementedException e) { + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + throw e; + } + return new PagedList<>(listPartitionsFromFileSystem(table), null); } } @@ -838,20 +830,11 @@ public List listPartitionsByNames( } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); } catch (NotImplementedException e) { - return listPartitionsFromFileSystem(getTable(identifier), partitions); - } - } - - @Override - public List listPartitionsByNamesWithoutFallback( - Identifier identifier, List> partitions) - throws TableNotExistException { - try { - return api.listPartitionsByNames(identifier, partitions); - } catch (NoSuchResourceException e) { - throw new TableNotExistException(identifier); - } catch (ForbiddenException e) { - throw new TableNoPermissionException(identifier, e); + Table table = getTable(identifier); + if (isCatalogManagedFormatTable(table)) { + throw e; + } + return listPartitionsFromFileSystem(table, partitions); } } @@ -918,16 +901,10 @@ public List listBranches(Identifier identifier) throws TableNotExistExce @Override public boolean supportsPartitionModification() { - // Paimon tables use commit-based partition maintenance. Managed Format Tables use the - // separate catalog-owned partition contract. + // Paimon tables use commit-based partition maintenance. return false; } - @Override - public boolean supportsManagedFormatTablePartitions() { - return true; - } - @Override public List listFunctions(String databaseName) throws DatabaseNotExistException { try { @@ -1346,6 +1323,11 @@ RESTApi api() { return api; } + private static boolean isCatalogManagedFormatTable(Table table) { + return table instanceof FormatTable + && new CoreOptions(table.options()).partitionedTableInMetastore(); + } + private FileIO fileIOForData(Path path, Identifier identifier) { return CachingFileIO.wrapWithCachingIfNeeded( dataTokenEnabled diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 151cf8341546..b4af3194b72f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -43,6 +43,7 @@ import org.apache.paimon.rest.auth.DLFTokenLoaderFactory; import org.apache.paimon.rest.auth.RESTAuthParameter; import org.apache.paimon.rest.exceptions.NotAuthorizedException; +import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.types.DataTypes; @@ -215,12 +216,52 @@ void testCreateTableDefaultOptions() throws Exception { } @Test - void testPartitionCapabilities() { - // Paimon tables use commit-based partition maintenance. Managed Format Tables use a - // separate capability so partition expiration never routes through the - // truncate-based Catalog#dropPartitions default (it cannot represent .done markers). - assertThat(restCatalog.supportsPartitionModification()).isFalse(); - assertThat(restCatalog.supportsManagedFormatTablePartitions()).isTrue(); + void testManagedFormatTablePagedPartitionListingDoesNotFallback() throws Exception { + Identifier identifier = createManagedFormatTable(); + restCatalogServer.setPartitionListingSupported(false); + + assertThatThrownBy(() -> restCatalog.listPartitionsPaged(identifier, null, null, null)) + .isInstanceOf(NotImplementedException.class); + } + + @Test + void testManagedFormatTablePartitionListingByNamesDoesNotFallback() throws Exception { + Identifier identifier = createManagedFormatTable(); + restCatalogServer.setPartitionListingSupported(false); + + assertThatThrownBy( + () -> + restCatalog.listPartitionsByNames( + identifier, + Collections.singletonList( + Collections.singletonMap("dt", "20260717")))) + .isInstanceOf(NotImplementedException.class); + } + + @Test + void testManagedFormatTablePartitionListingDoesNotFallback() throws Exception { + Identifier identifier = createManagedFormatTable(); + restCatalogServer.setPartitionListingSupported(false); + + assertThatThrownBy(() -> restCatalog.listPartitions(identifier)) + .isInstanceOf(NotImplementedException.class); + } + + private Identifier createManagedFormatTable() throws Exception { + Identifier identifier = Identifier.create("db1", "managed_partition_table"); + restCatalog.createDatabase(identifier.getDatabaseName(), true); + restCatalog.createTable( + identifier, + Schema.newBuilder() + .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString()) + .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .column("id", DataTypes.INT()) + .column("dt", DataTypes.STRING()) + .partitionKeys("dt") + .build(), + false); + return identifier; } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index c0929184a6c8..f2e2a0b93ded 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -211,6 +211,8 @@ public class RESTCatalogServer { private final List> receivedHeaders = new ArrayList<>(); + private volatile boolean partitionListingSupported = true; + public RESTCatalogServer( String dataPath, AuthProvider authProvider, ConfigResponse config, String warehouse) { this.warehouse = warehouse; @@ -273,6 +275,10 @@ public void removeDataToken(Identifier identifier) { DataTokenStore.removeDataToken(warehouse, identifier.getFullName()); } + public void setPartitionListingSupported(boolean partitionListingSupported) { + this.partitionListingSupported = partitionListingSupported; + } + public void addNoPermissionDatabase(String database) { noPermissionDatabases.add(database); } @@ -503,6 +509,10 @@ && isTableByIdRequest(request.getPath())) { catalog.markDonePartitions( identifier, markDonePartitionsRequest.getPartitionSpecs()); return new MockResponse().setResponseCode(200); + } else if (!partitionListingSupported + && ((isPartitions && "GET".equals(restAuthParameter.method())) + || isListPartitionsByNames)) { + return mockResponse(new ErrorResponse(null, null, "", 501), 501); } else if (isDropPartitions) { return dropPartitionsHandle(restAuthParameter.data(), identifier); } else if (isPartitions) { From 76e453d7d6f4d19f67a789b5817acb0d1b76518d Mon Sep 17 00:00:00 2001 From: Sun Dapeng Date: Fri, 17 Jul 2026 16:41:01 +0800 Subject: [PATCH 4/4] [core][rest] Return structured partition specs --- docs/static/rest-catalog-open-api.yaml | 16 +++++++--- .../responses/CreatePartitionsResponse.java | 13 +++++---- .../responses/DropPartitionsResponse.java | 13 +++++---- .../apache/paimon/rest/RESTApiJsonTest.java | 29 ++++++++++++------- .../apache/paimon/rest/RESTCatalogServer.java | 24 ++++++++------- .../apache/paimon/rest/RESTCatalogTest.java | 6 ++-- 6 files changed, 61 insertions(+), 40 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index ae58d24cd1a3..2754e720d147 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -2457,11 +2457,15 @@ components: dropped: type: array items: - type: string + type: object + additionalProperties: + type: string missing: type: array items: - type: string + type: object + additionalProperties: + type: string CreatePartitionsRequest: type: object required: @@ -2485,11 +2489,15 @@ components: created: type: array items: - type: string + type: object + additionalProperties: + type: string existed: type: array items: - type: string + type: object + additionalProperties: + type: string AlterTableRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java index ef612f889e68..03a923ed21eb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/CreatePartitionsResponse.java @@ -26,6 +26,7 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.Map; /** Response for creating partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) @@ -35,26 +36,26 @@ public class CreatePartitionsResponse implements RESTResponse { private static final String FIELD_EXISTED = "existed"; @JsonProperty(FIELD_CREATED) - private final List created; + private final List> created; @JsonProperty(FIELD_EXISTED) - private final List existed; + private final List> existed; @JsonCreator public CreatePartitionsResponse( - @JsonProperty(FIELD_CREATED) List created, - @JsonProperty(FIELD_EXISTED) List existed) { + @JsonProperty(FIELD_CREATED) List> created, + @JsonProperty(FIELD_EXISTED) List> existed) { this.created = created; this.existed = existed; } @JsonGetter(FIELD_CREATED) - public List getCreated() { + public List> getCreated() { return created; } @JsonGetter(FIELD_EXISTED) - public List getExisted() { + public List> getExisted() { return existed; } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java index 57debbd7ee25..b250b6ddb9a4 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DropPartitionsResponse.java @@ -26,6 +26,7 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.Map; /** Response for dropping (unregistering) partitions. */ @JsonIgnoreProperties(ignoreUnknown = true) @@ -35,26 +36,26 @@ public class DropPartitionsResponse implements RESTResponse { private static final String FIELD_MISSING = "missing"; @JsonProperty(FIELD_DROPPED) - private final List dropped; + private final List> dropped; @JsonProperty(FIELD_MISSING) - private final List missing; + private final List> missing; @JsonCreator public DropPartitionsResponse( - @JsonProperty(FIELD_DROPPED) List dropped, - @JsonProperty(FIELD_MISSING) List missing) { + @JsonProperty(FIELD_DROPPED) List> dropped, + @JsonProperty(FIELD_MISSING) List> missing) { this.dropped = dropped; this.missing = missing; } @JsonGetter(FIELD_DROPPED) - public List getDropped() { + public List> getDropped() { return dropped; } @JsonGetter(FIELD_MISSING) - public List getMissing() { + public List> getMissing() { return missing; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java index 71c394906013..bdf2b5f82c1b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTApiJsonTest.java @@ -54,7 +54,6 @@ import org.junit.Test; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -213,16 +212,20 @@ public void listPartitionsResponseParseTest() throws Exception { @Test public void createPartitionsResponseParseTest() throws Exception { + Map created = new HashMap<>(); + created.put("dt", "20260714"); + created.put("region", "cn"); + Map existed = new HashMap<>(); + existed.put("dt", "20260713"); + existed.put("region", "us"); CreatePartitionsResponse response = new CreatePartitionsResponse( - Arrays.asList("dt=20260714", "dt=20260715"), - Collections.singletonList("dt=20260713")); - + Collections.singletonList(created), Collections.singletonList(existed)); CreatePartitionsResponse parsed = RESTApi.fromJson(RESTApi.toJson(response), CreatePartitionsResponse.class); - assertEquals(response.getCreated(), parsed.getCreated()); - assertEquals(response.getExisted(), parsed.getExisted()); + assertEquals(Collections.singletonList(created), parsed.getCreated()); + assertEquals(Collections.singletonList(existed), parsed.getExisted()); } @Test @@ -250,16 +253,20 @@ public void createPartitionsRequestParseTest() throws Exception { @Test public void dropPartitionsResponseParseTest() throws Exception { + Map dropped = new HashMap<>(); + dropped.put("dt", "20260714"); + dropped.put("region", "cn"); + Map missing = new HashMap<>(); + missing.put("dt", "20260713"); + missing.put("region", "us"); DropPartitionsResponse response = new DropPartitionsResponse( - Arrays.asList("dt=20260714", "dt=20260715"), - Collections.singletonList("dt=20260713")); - + Collections.singletonList(dropped), Collections.singletonList(missing)); DropPartitionsResponse parsed = RESTApi.fromJson(RESTApi.toJson(response), DropPartitionsResponse.class); - assertEquals(response.getDropped(), parsed.getDropped()); - assertEquals(response.getMissing(), parsed.getMissing()); + assertEquals(Collections.singletonList(dropped), parsed.getDropped()); + assertEquals(Collections.singletonList(missing), parsed.getMissing()); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index f2e2a0b93ded..3a0e1539eea1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -1887,14 +1887,14 @@ private MockResponse partitionsApiHandle( return mockResponse(response, 409); } } - List created = new ArrayList<>(); - List existed = new ArrayList<>(); + List> created = new ArrayList<>(); + List> existed = new ArrayList<>(); for (Map spec : request.getPartitionSpecs()) { if (existingSpecs.add(spec)) { storedPartitions.add(new Partition(spec, 0, 0, 0, 0, -1, false)); - created.add(PartitionUtils.buildPartitionName(spec)); + created.add(spec); } else { - existed.add(PartitionUtils.buildPartitionName(spec)); + existed.add(spec); } } return mockResponse(new CreatePartitionsResponse(created, existed), 200); @@ -1911,27 +1911,31 @@ private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifie tableIdentifier.getFullName(), ignored -> new ArrayList<>()); Set> existingSpecs = storedPartitions.stream().map(Partition::spec).collect(Collectors.toSet()); - List missing = new ArrayList<>(); + List> missing = new ArrayList<>(); for (Map spec : request.getPartitionSpecs()) { if (!existingSpecs.contains(spec)) { - missing.add(PartitionUtils.buildPartitionName(spec)); + missing.add(spec); } } if (!request.ignoreIfNotExists() && !missing.isEmpty()) { + List missingNames = + missing.stream() + .map(PartitionUtils::buildPartitionName) + .collect(Collectors.toList()); ErrorResponse response = new ErrorResponse( ErrorResponse.RESOURCE_TYPE_PARTITION, - missing.get(0), - String.format("Partitions %s do not exist.", missing), + missingNames.get(0), + String.format("Partitions %s do not exist.", missingNames), 404); return mockResponse(response, 404); } - List dropped = new ArrayList<>(); + List> dropped = new ArrayList<>(); Set> toDrop = new HashSet<>(request.getPartitionSpecs()); storedPartitions.removeIf( partition -> { if (toDrop.contains(partition.spec())) { - dropped.add(PartitionUtils.buildPartitionName(partition.spec())); + dropped.add(partition.spec()); return true; } return false; diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index a75e171697a2..2de1b1f90be1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -1556,7 +1556,7 @@ void testCreatePartitionsForManagedFormatTable() throws Exception { CreatePartitionsResponse response = restCatalog.api().createPartitions(identifier, partitionSpecs); - assertThat(response.getCreated()).containsExactlyInAnyOrder("dt=20260714", "dt=20260715"); + assertThat(response.getCreated()).containsExactlyInAnyOrderElementsOf(partitionSpecs); assertThat(response.getExisted()).isEmpty(); catalog.createPartitions(identifier, partitionSpecs); @@ -1605,8 +1605,8 @@ void testDropPartitionsForManagedFormatTable() throws Exception { singletonMap("dt", "20260714"), singletonMap("dt", "20260799")), true); - assertThat(response.getDropped()).containsExactly("dt=20260714"); - assertThat(response.getMissing()).containsExactly("dt=20260799"); + assertThat(response.getDropped()).containsExactly(singletonMap("dt", "20260714")); + assertThat(response.getMissing()).containsExactly(singletonMap("dt", "20260799")); assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec)) .containsExactly(singletonMap("dt", "20260715"));