From 6c882e9086827d529f57926374465b54c56ab007 Mon Sep 17 00:00:00 2001 From: Dhruv Arya Date: Fri, 7 Aug 2026 00:48:05 +0000 Subject: [PATCH 1/2] OpenAPI: REST catalog support for table format version 4 Opening this to drive discussion on what the REST catalog spec needs for format version 4. It is a proposal rather than a merge candidate: parts of it depend on table spec changes that are still in review, and one change is deliberately breaking. Details and open questions below. Format version 4 restructures table metadata, and three constraints in the current spec prevent a catalog from serving a v4 table at all: format-version is capped at 3, Snapshot requires manifest-list, and CommitTableResponse requires metadata-location. Changes: - TableMetadata: raise the format-version maximum to 4, and document that location is optional in v4, where it may be supplied by the catalog rather than carried in metadata. - Snapshot: add root-manifest and make manifest-list optional, with a oneOf requiring exactly one of the two. A snapshot with neither is invalid and must not be read as a snapshot with no files. Which of the two applies is determined by format-version, which cannot be expressed here because that field lives in the enclosing TableMetadata, so it is stated normatively. - LoadTableResult and CommitTableResponse: add table-location, which carries the table's base location for resolving relative paths. Format version 4 makes TableMetadata.location optional while allowing relative paths, so a table can have metadata that is unreadable without a catalog-supplied base. CommitTableResponse no longer requires metadata-location, and both responses document that it may be absent when the catalog is the source of truth for table state and no client-visible metadata pointer exists. - UnregisterTableResult: add table-location, so the endpoint that hands back a metadata location cannot return metadata with no base to resolve against. - Move the table endpoints to /v2. Every path whose response can carry table metadata moves: tables, tables/{table}, register, and tables/{table}/unregister. Open questions: - root-manifest is not in the ratified spec. It comes from the adaptive metadata tree proposal (#16025), which is still open, so the field name and shape may change. Note also that SnapshotParser currently writes manifest-list for v4 tables, so the oneOf as written would reject snapshots this repository produces today. Whether to keep the field, defer it, or make it purely additive is the main thing worth discussing. - Moving the table endpoints to /v2 rather than adding /v2 alongside /v1 is a breaking change, and assumes v1 is no longer supported. An additive version that leaves /v1 in place is the obvious alternative if that assumption does not hold. - Whether path versioning is the right mechanism at all, versus a header or a content-type parameter. - Relative path support and the typed content stats are already in the spec for v4; snapshot offloading is not yet, and would interact with the snapshots query parameter on loadTable if it lands later. make lint and make generate both pass. --- open-api/rest-catalog-open-api.py | 207 +++++++++++++++++++++++----- open-api/rest-catalog-open-api.yaml | 190 ++++++++++++++++++++----- 2 files changed, 326 insertions(+), 71 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5e3580ae3825..8b6e64f3c4f3 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -56,14 +56,16 @@ class CatalogConfig(BaseModel): ) endpoints: list[str] | None = Field( None, - description='A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character.', + description='A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. Table endpoints are served under `/v2`, which supports tables at any format version, 1 through 4.', examples=[ [ 'GET /v1/{prefix}/namespaces/{namespace}', 'GET /v1/{prefix}/namespaces', 'POST /v1/{prefix}/namespaces', - 'GET /v1/{prefix}/namespaces/{namespace}/tables/{table}', 'GET /v1/{prefix}/namespaces/{namespace}/views/{view}', + 'GET /v2/{prefix}/namespaces/{namespace}/tables/{table}', + 'POST /v2/{prefix}/namespaces/{namespace}/tables/{table}', + 'POST /v2/{prefix}/namespaces/{namespace}/register', ] ], ) @@ -330,7 +332,24 @@ class Summary(BaseModel): operation: Literal['append', 'replace', 'overwrite', 'delete'] -class Snapshot(BaseModel): +class Snapshot1(BaseModel): + """ + A snapshot of the table's contents at a point in time. + + + Exactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf` + below. A snapshot carrying neither is invalid and must be rejected; it must not be + interpreted as a snapshot with no files. A snapshot carrying both is also invalid. + + + Which of the two is present is determined by the table's `format-version`: format + versions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That + correlation cannot be expressed here because `format-version` lives in the enclosing + `TableMetadata` object, so it is stated normatively and must be enforced by servers + and clients. + + """ + snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') sequence_number: int | None = Field(None, alias='sequence-number') @@ -338,7 +357,58 @@ class Snapshot(BaseModel): manifest_list: str = Field( ..., alias='manifest-list', - description="Location of the snapshot's manifest list file", + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead.", + ) + root_manifest: str | None = Field( + None, + alias='root-manifest', + description="Location of the snapshot's root manifest. The root manifest can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.", + ) + first_row_id: int | None = Field( + None, + alias='first-row-id', + description='The first _row_id assigned to the first row in the first data file in the first manifest', + ) + added_rows: int | None = Field( + None, + alias='added-rows', + description='The upper bound of the number of rows with assigned row IDs', + ) + summary: Summary + schema_id: int | None = Field(None, alias='schema-id') + + +class Snapshot2(BaseModel): + """ + A snapshot of the table's contents at a point in time. + + + Exactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf` + below. A snapshot carrying neither is invalid and must be rejected; it must not be + interpreted as a snapshot with no files. A snapshot carrying both is also invalid. + + + Which of the two is present is determined by the table's `format-version`: format + versions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That + correlation cannot be expressed here because `format-version` lives in the enclosing + `TableMetadata` object, so it is stated normatively and must be enforced by servers + and clients. + + """ + + snapshot_id: int = Field(..., alias='snapshot-id') + parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') + sequence_number: int | None = Field(None, alias='sequence-number') + timestamp_ms: int = Field(..., alias='timestamp-ms') + manifest_list: str | None = Field( + None, + alias='manifest-list', + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead.", + ) + root_manifest: str = Field( + ..., + alias='root-manifest', + description="Location of the snapshot's root manifest. The root manifest can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.", ) first_row_id: int | None = Field( None, @@ -354,6 +424,13 @@ class Snapshot(BaseModel): schema_id: int | None = Field(None, alias='schema-id') +class Snapshot(RootModel[Snapshot1 | Snapshot2]): + root: Snapshot1 | Snapshot2 = Field( + ..., + description="A snapshot of the table's contents at a point in time.\n\n\nExactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf`\nbelow. A snapshot carrying neither is invalid and must be rejected; it must not be\ninterpreted as a snapshot with no files. A snapshot carrying both is also invalid.\n\n\nWhich of the two is present is determined by the table's `format-version`: format\nversions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That\ncorrelation cannot be expressed here because `format-version` lives in the enclosing\n`TableMetadata` object, so it is stated normatively and must be enforced by servers\nand clients.\n", + ) + + class SnapshotReference(BaseModel): type: Literal['tag', 'branch'] snapshot_id: int = Field(..., alias='snapshot-id') @@ -1650,9 +1727,12 @@ class Apply(BaseModel): class TableMetadata(BaseModel): - format_version: int = Field(..., alias='format-version', ge=1, le=3) + format_version: int = Field(..., alias='format-version', ge=1, le=4) table_uuid: str = Field(..., alias='table-uuid') - location: str | None = None + location: str | None = Field( + None, + description="The table's base location. Required through format version 3, where it may be a path without a URI scheme; readers prepend a scheme for consistency with v4 absolute paths. Optional for format version 4, where the location may be managed externally and supplied by the catalog when the table is loaded, and where it must be an absolute path when present. See the `table-location` field of `LoadTableResult`.", + ) last_updated_ms: int | None = Field(None, alias='last-updated-ms') next_row_id: int | None = Field( None, @@ -1705,58 +1785,87 @@ class AddSchemaUpdate(BaseUpdate): class LoadTableResult(BaseModel): """ - Result used when a table is successfully loaded. + Result used when a table is successfully loaded, for tables at any format version. - The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. - Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + The table metadata JSON is returned in the `metadata` field. The location of the table + metadata file is returned in the `metadata-location` field when the table has one, and + the table's base location in the `table-location` field. - The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + ## Metadata location - The following configurations should be respected by clients: + The `metadata-location` field is optional. It is absent when the metadata is staged + but not committed, as in a create transaction, and when the table has no + client-visible metadata location, as for a catalog-managed table where the catalog is + the source of truth for table state and no metadata pointer need exist. + + + Clients must not require this field to be present, and must not use it to bypass the + catalog for reads or commits. To obtain a metadata location for a catalog-managed + table, use the `unregisterTable` endpoint, which returns the table's last metadata + location at the point the table leaves catalog control and further commits are + rejected. + + + ## Table location + + + The `table-location` field carries the table's base location. Format version 4 allows + location fields in metadata to be relative, and such paths must be resolved against + the table location. Format version 4 also makes `metadata.location` optional, so + a table may have metadata that contains relative paths and omits `location`. Servers + must populate `table-location` for any such table, because it cannot be read + otherwise. + + + When both `metadata.location` and `table-location` are present, `table-location` + takes precedence: the catalog is authoritative for table state, and catalog-supplied + locations are what allow a table to be relocated without rewriting metadata. + + + ## Configuration + + + The `config` map returns table-specific configuration for the table's resources, + including its HTTP client and FileIO. For example, config may contain a specific FileIO + implementation class for the table depending on its underlying storage. The following + configurations should be respected by clients: - ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: - `client`: Clients MUST use client-side scan planning - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint - ## AWS Configurations + The following configurations should be respected when working with tables stored in AWS S3: - The following configurations should be respected when working with tables stored in AWS S3 - `client.region`: region to configure client for making requests to AWS - - `s3.access-key-id`: id for credentials that provide access to the data in S3 - - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled - ## Storage Credentials + ## Storage credentials and remote signing - Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. - Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. - ## Remote Signing - - If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config` - field to communicate all signer client settings. When the `remote-signing-config` field is present, clients - SHOULD respect the provided configuration. - - For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration: - - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. - - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. - If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. - If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` + field, and remote signer client settings through the `remote-signing-config` field. + These are the only mechanisms for communicating credentials and signer settings. + Because `config` is an open map, this is stated explicitly: servers must not place + storage credentials or remote signer settings in `config`, and clients must not look + for them there. """ metadata_location: str | None = Field( None, alias='metadata-location', - description='May be null if the table is staged as part of a transaction', + description='Location of the table metadata file. Absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", ) metadata: TableMetadata config: dict[str, str] | None = None @@ -1841,6 +1950,11 @@ class UnregisterTableResult(BaseModel): alias='metadata-location', description='The last metadata location for the table at the time it was unregistered.', ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) metadata: TableMetadata @@ -2026,7 +2140,34 @@ class FunctionStructField(BaseModel): class CommitTableResponse(BaseModel): - metadata_location: str = Field(..., alias='metadata-location') + """ + Result used when a table is successfully updated, for tables at any format version. + + + The table metadata JSON is returned in the `metadata` field. The location of the + committed table metadata file is returned in the `metadata-location` field when the + table has one, and is absent when it does not, as for a catalog-managed table where + the catalog is the source of truth for table state. + + + The `table-location` field carries the table's base location, so that a client + resolving relative paths in the returned metadata does not need an additional + `loadTable` call to obtain the base location. Servers must populate it whenever the + returned metadata contains relative paths and omits `location`, because the metadata + cannot be resolved otherwise; without it the response would not be self-sufficient. + + """ + + metadata_location: str | None = Field( + None, + alias='metadata-location', + description='Location of the committed table metadata file. Absent when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.', + ) + table_location: str | None = Field( + None, + alias='table-location', + description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.", + ) metadata: TableMetadata diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..c00905324b0d 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -522,7 +522,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/tables: + /v2/{prefix}/namespaces/{namespace}/tables: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -578,6 +578,7 @@ paths: the client sends all create and subsequent changes to the table commit route. Changes from the table create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial table state. + operationId: createTable parameters: - $ref: '#/components/parameters/data-access' @@ -968,7 +969,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/register: + /v2/{prefix}/namespaces/{namespace}/register: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1024,7 +1025,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/tables/{table}: + /v2/{prefix}/namespaces/{namespace}/tables/{table}: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1051,6 +1052,7 @@ paths: table. The configuration key "token" is used to pass an access token to be used as a bearer token for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, "urn:ietf:params:oauth:token-type:jwt=". + parameters: - $ref: '#/components/parameters/data-access' - name: If-None-Match @@ -1132,6 +1134,7 @@ paths: committed using this route. Transactions should include all changes to the table, including table initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is used to ensure that the table was not created concurrently. + requestBody: required: true content: @@ -1299,7 +1302,7 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/namespaces/{namespace}/tables/{table}/unregister: + /v2/{prefix}/namespaces/{namespace}/tables/{table}/unregister: parameters: - $ref: '#/components/parameters/prefix' - $ref: '#/components/parameters/namespace' @@ -1323,6 +1326,14 @@ paths: corresponding table metadata. This table metadata must include all commits that happened before the unregister operation. All attempted commits after the unregister operation in this catalog must fail. + + + This is how a client obtains a metadata location for a table that does not expose one + through `loadTable`, such as a catalog-managed table. The location is produced at the + point the table leaves catalog control, after which the catalog rejects further + commits, so the returned location cannot become stale while the catalog continues to + accept writes. Because `metadata-location` is required here, a server must be able to + produce a metadata location for any table it can unregister. responses: 200: $ref: '#/components/responses/UnregisterTableResponse' @@ -2305,12 +2316,16 @@ components: type: string description: A list of endpoints that the server supports. The format of each endpoint must be " ". The HTTP verb and the resource path must be separated by a space character. + Table endpoints are served under `/v2`, which supports tables at any format + version, 1 through 4. example: [ "GET /v1/{prefix}/namespaces/{namespace}", "GET /v1/{prefix}/namespaces", "POST /v1/{prefix}/namespaces", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" + "GET /v1/{prefix}/namespaces/{namespace}/views/{view}", + "GET /v2/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v2/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v2/{prefix}/namespaces/{namespace}/register" ] idempotency-key-lifetime: type: string @@ -2948,12 +2963,30 @@ components: type: string Snapshot: + description: | + A snapshot of the table's contents at a point in time. + + + Exactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf` + below. A snapshot carrying neither is invalid and must be rejected; it must not be + interpreted as a snapshot with no files. A snapshot carrying both is also invalid. + + + Which of the two is present is determined by the table's `format-version`: format + versions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That + correlation cannot be expressed here because `format-version` lives in the enclosing + `TableMetadata` object, so it is stated normatively and must be enforced by servers + and clients. type: object required: - snapshot-id - timestamp-ms - - manifest-list - summary + oneOf: + - required: + - manifest-list + - required: + - root-manifest properties: snapshot-id: type: integer @@ -2969,7 +3002,16 @@ components: format: int64 manifest-list: type: string - description: Location of the snapshot's manifest list file + description: + Location of the snapshot's manifest list file. Used for format versions 1-3 and + must be absent for format version 4, which uses `root-manifest` instead. + root-manifest: + type: string + description: + Location of the snapshot's root manifest. The root manifest can reference data + files, delete files, and other data and delete manifests in a unified structure, + replacing the manifest list. Required for format version 4 and must be absent for + format versions 1-3. first-row-id: type: integer format: int64 @@ -3055,11 +3097,18 @@ components: format-version: type: integer minimum: 1 - maximum: 3 + maximum: 4 table-uuid: type: string location: type: string + description: + The table's base location. Required through format version 3, where it may be a + path without a URI scheme; readers prepend a scheme for consistency with v4 + absolute paths. Optional for format version 4, where the location may be managed + externally and supplied by the catalog when the table is loaded, and where it must + be an absolute path when present. See the `table-location` field of + `LoadTableResult`. last-updated-ms: type: integer format: int64 @@ -3851,60 +3900,93 @@ components: LoadTableResult: description: | - Result used when a table is successfully loaded. + Result used when a table is successfully loaded, for tables at any format version. - The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. - Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + The table metadata JSON is returned in the `metadata` field. The location of the table + metadata file is returned in the `metadata-location` field when the table has one, and + the table's base location in the `table-location` field. - The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + ## Metadata location - The following configurations should be respected by clients: + The `metadata-location` field is optional. It is absent when the metadata is staged + but not committed, as in a create transaction, and when the table has no + client-visible metadata location, as for a catalog-managed table where the catalog is + the source of truth for table state and no metadata pointer need exist. + + + Clients must not require this field to be present, and must not use it to bypass the + catalog for reads or commits. To obtain a metadata location for a catalog-managed + table, use the `unregisterTable` endpoint, which returns the table's last metadata + location at the point the table leaves catalog control and further commits are + rejected. + + + ## Table location + + + The `table-location` field carries the table's base location. Format version 4 allows + location fields in metadata to be relative, and such paths must be resolved against + the table location. Format version 4 also makes `metadata.location` optional, so + a table may have metadata that contains relative paths and omits `location`. Servers + must populate `table-location` for any such table, because it cannot be read + otherwise. + + + When both `metadata.location` and `table-location` are present, `table-location` + takes precedence: the catalog is authoritative for table state, and catalog-supplied + locations are what allow a table to be relocated without rewriting metadata. + + + ## Configuration + + + The `config` map returns table-specific configuration for the table's resources, + including its HTTP client and FileIO. For example, config may contain a specific FileIO + implementation class for the table depending on its underlying storage. The following + configurations should be respected by clients: - ## General Configurations - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: - `client`: Clients MUST use client-side scan planning - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint - ## AWS Configurations + The following configurations should be respected when working with tables stored in AWS S3: - The following configurations should be respected when working with tables stored in AWS S3 - `client.region`: region to configure client for making requests to AWS - - `s3.access-key-id`: id for credentials that provide access to the data in S3 - - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled - ## Storage Credentials - - Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. - Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + ## Storage credentials and remote signing - ## Remote Signing - - If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config` - field to communicate all signer client settings. When the `remote-signing-config` field is present, clients - SHOULD respect the provided configuration. - - For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration: - - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. - - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. - If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact. - If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI. + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` + field, and remote signer client settings through the `remote-signing-config` field. + These are the only mechanisms for communicating credentials and signer settings. + Because `config` is an open map, this is stated explicitly: servers must not place + storage credentials or remote signer settings in `config`, and clients must not look + for them there. type: object required: - metadata properties: metadata-location: type: string - description: May be null if the table is staged as part of a transaction - nullable: true + description: + Location of the table metadata file. Absent when the metadata is staged but not + committed, as in a create transaction, and when the table has no client-visible + metadata location, as for a catalog-managed table where the catalog is the source + of truth for table state. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. Must be an + absolute path with a URI scheme when present, and must be present when the returned + metadata contains relative paths and omits `location`, because the metadata cannot + be resolved otherwise. Takes precedence over `metadata.location`. metadata: $ref: '#/components/schemas/TableMetadata' config: @@ -4163,6 +4245,13 @@ components: type: string description: The last metadata location for the table at the time it was unregistered. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. Must be an + absolute path with a URI scheme when present, and must be present when the returned + metadata contains relative paths and omits `location`, because the metadata cannot + be resolved otherwise. Takes precedence over `metadata.location`. metadata: $ref: '#/components/schemas/TableMetadata' @@ -4929,13 +5018,38 @@ components: nullable: true CommitTableResponse: + description: | + Result used when a table is successfully updated, for tables at any format version. + + + The table metadata JSON is returned in the `metadata` field. The location of the + committed table metadata file is returned in the `metadata-location` field when the + table has one, and is absent when it does not, as for a catalog-managed table where + the catalog is the source of truth for table state. + + + The `table-location` field carries the table's base location, so that a client + resolving relative paths in the returned metadata does not need an additional + `loadTable` call to obtain the base location. Servers must populate it whenever the + returned metadata contains relative paths and omits `location`, because the metadata + cannot be resolved otherwise; without it the response would not be self-sufficient. type: object required: - - metadata-location - metadata properties: metadata-location: type: string + description: + Location of the committed table metadata file. Absent when the table has no + client-visible metadata location, as for a catalog-managed table where the catalog + is the source of truth for table state. + table-location: + type: string + description: + The table's base location, used to resolve relative paths in metadata. Must be an + absolute path with a URI scheme when present, and must be present when the returned + metadata contains relative paths and omits `location`, because the metadata cannot + be resolved otherwise. Takes precedence over `metadata.location`. metadata: $ref: '#/components/schemas/TableMetadata' From 16420fec3dcac07d1395e54cb1a0643d4dac1190 Mon Sep 17 00:00:00 2001 From: Dhruv Arya Date: Fri, 7 Aug 2026 01:05:03 +0000 Subject: [PATCH 2/2] OpenAPI: Rename the v4 snapshot pointer to content-root Renames the field from root-manifest to content-root, and follows the name in the surrounding descriptions so the prose matches ("the snapshot's content root" rather than "the snapshot's root manifest"). make lint and make generate both pass, and the generated model picks up the rename in both oneOf branches. --- open-api/rest-catalog-open-api.py | 26 +++++++++++++------------- open-api/rest-catalog-open-api.yaml | 12 ++++++------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8b6e64f3c4f3..3d68d658fa2c 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -337,13 +337,13 @@ class Snapshot1(BaseModel): A snapshot of the table's contents at a point in time. - Exactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf` + Exactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf` below. A snapshot carrying neither is invalid and must be rejected; it must not be interpreted as a snapshot with no files. A snapshot carrying both is also invalid. Which of the two is present is determined by the table's `format-version`: format - versions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That + versions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That correlation cannot be expressed here because `format-version` lives in the enclosing `TableMetadata` object, so it is stated normatively and must be enforced by servers and clients. @@ -357,12 +357,12 @@ class Snapshot1(BaseModel): manifest_list: str = Field( ..., alias='manifest-list', - description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead.", + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead.", ) - root_manifest: str | None = Field( + content_root: str | None = Field( None, - alias='root-manifest', - description="Location of the snapshot's root manifest. The root manifest can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.", + alias='content-root', + description="Location of the snapshot's content root. The content root can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.", ) first_row_id: int | None = Field( None, @@ -383,13 +383,13 @@ class Snapshot2(BaseModel): A snapshot of the table's contents at a point in time. - Exactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf` + Exactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf` below. A snapshot carrying neither is invalid and must be rejected; it must not be interpreted as a snapshot with no files. A snapshot carrying both is also invalid. Which of the two is present is determined by the table's `format-version`: format - versions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That + versions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That correlation cannot be expressed here because `format-version` lives in the enclosing `TableMetadata` object, so it is stated normatively and must be enforced by servers and clients. @@ -403,12 +403,12 @@ class Snapshot2(BaseModel): manifest_list: str | None = Field( None, alias='manifest-list', - description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `root-manifest` instead.", + description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead.", ) - root_manifest: str = Field( + content_root: str = Field( ..., - alias='root-manifest', - description="Location of the snapshot's root manifest. The root manifest can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.", + alias='content-root', + description="Location of the snapshot's content root. The content root can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.", ) first_row_id: int | None = Field( None, @@ -427,7 +427,7 @@ class Snapshot2(BaseModel): class Snapshot(RootModel[Snapshot1 | Snapshot2]): root: Snapshot1 | Snapshot2 = Field( ..., - description="A snapshot of the table's contents at a point in time.\n\n\nExactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf`\nbelow. A snapshot carrying neither is invalid and must be rejected; it must not be\ninterpreted as a snapshot with no files. A snapshot carrying both is also invalid.\n\n\nWhich of the two is present is determined by the table's `format-version`: format\nversions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That\ncorrelation cannot be expressed here because `format-version` lives in the enclosing\n`TableMetadata` object, so it is stated normatively and must be enforced by servers\nand clients.\n", + description="A snapshot of the table's contents at a point in time.\n\n\nExactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf`\nbelow. A snapshot carrying neither is invalid and must be rejected; it must not be\ninterpreted as a snapshot with no files. A snapshot carrying both is also invalid.\n\n\nWhich of the two is present is determined by the table's `format-version`: format\nversions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That\ncorrelation cannot be expressed here because `format-version` lives in the enclosing\n`TableMetadata` object, so it is stated normatively and must be enforced by servers\nand clients.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index c00905324b0d..3f1ad503db9b 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -2967,13 +2967,13 @@ components: A snapshot of the table's contents at a point in time. - Exactly one of `manifest-list` and `root-manifest` is present, enforced by the `oneOf` + Exactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf` below. A snapshot carrying neither is invalid and must be rejected; it must not be interpreted as a snapshot with no files. A snapshot carrying both is also invalid. Which of the two is present is determined by the table's `format-version`: format - versions 1-3 use `manifest-list`, and format version 4 uses `root-manifest`. That + versions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That correlation cannot be expressed here because `format-version` lives in the enclosing `TableMetadata` object, so it is stated normatively and must be enforced by servers and clients. @@ -2986,7 +2986,7 @@ components: - required: - manifest-list - required: - - root-manifest + - content-root properties: snapshot-id: type: integer @@ -3004,11 +3004,11 @@ components: type: string description: Location of the snapshot's manifest list file. Used for format versions 1-3 and - must be absent for format version 4, which uses `root-manifest` instead. - root-manifest: + must be absent for format version 4, which uses `content-root` instead. + content-root: type: string description: - Location of the snapshot's root manifest. The root manifest can reference data + Location of the snapshot's content root. The content root can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.