OpenAI: IRC spec changes for v4 [WIP][Discuss]

open-api/rest-catalog-open-api.py

@@ -56,14 +56,16 @@ class CatalogConfig(BaseModel):
     )
     endpoints: list[str] | None = Field(
         None,
-        description='A list of endpoints that the server supports. The format of each endpoint must be "<HTTP verb> <resource path from OpenAPI REST spec>". The HTTP verb and the resource path must be separated by a space character.',
+        description='A list of endpoints that the server supports. The format of each endpoint must be "<HTTP verb> <resource path from OpenAPI REST spec>". The HTTP verb and the resource path must be separated by a space character. Table endpoints are served under `/v2`, which supports tables at any format version, 1 through 4.',
         examples=[
             [
                 'GET /v1/{prefix}/namespaces/{namespace}',
                 'GET /v1/{prefix}/namespaces',
                 'POST /v1/{prefix}/namespaces',
-                'GET /v1/{prefix}/namespaces/{namespace}/tables/{table}',
                 'GET /v1/{prefix}/namespaces/{namespace}/views/{view}',
+                'GET /v2/{prefix}/namespaces/{namespace}/tables/{table}',
+                'POST /v2/{prefix}/namespaces/{namespace}/tables/{table}',
+                'POST /v2/{prefix}/namespaces/{namespace}/register',
             ]
         ],
     )
@@ -330,15 +332,83 @@ class Summary(BaseModel):
     operation: Literal['append', 'replace', 'overwrite', 'delete']
 
 
-class Snapshot(BaseModel):
+class Snapshot1(BaseModel):
+    """
+    A snapshot of the table's contents at a point in time.
+
+
+    Exactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf`
+    below. A snapshot carrying neither is invalid and must be rejected; it must not be
+    interpreted as a snapshot with no files. A snapshot carrying both is also invalid.
+
+
+    Which of the two is present is determined by the table's `format-version`: format
+    versions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That
+    correlation cannot be expressed here because `format-version` lives in the enclosing
+    `TableMetadata` object, so it is stated normatively and must be enforced by servers
+    and clients.
+
+    """
+
     snapshot_id: int = Field(..., alias='snapshot-id')
     parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id')
     sequence_number: int | None = Field(None, alias='sequence-number')
     timestamp_ms: int = Field(..., alias='timestamp-ms')
     manifest_list: str = Field(
         ...,
         alias='manifest-list',
-        description="Location of the snapshot's manifest list file",
+        description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead.",
+    )
+    content_root: str | None = Field(
+        None,
+        alias='content-root',
+        description="Location of the snapshot's content root. The content root can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.",
+    )
+    first_row_id: int | None = Field(
+        None,
+        alias='first-row-id',
+        description='The first _row_id assigned to the first row in the first data file in the first manifest',
+    )
+    added_rows: int | None = Field(
+        None,
+        alias='added-rows',
+        description='The upper bound of the number of rows with assigned row IDs',
+    )
+    summary: Summary
+    schema_id: int | None = Field(None, alias='schema-id')
+
+
+class Snapshot2(BaseModel):
+    """
+    A snapshot of the table's contents at a point in time.
+
+
+    Exactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf`
+    below. A snapshot carrying neither is invalid and must be rejected; it must not be
+    interpreted as a snapshot with no files. A snapshot carrying both is also invalid.
+
+
+    Which of the two is present is determined by the table's `format-version`: format
+    versions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That
+    correlation cannot be expressed here because `format-version` lives in the enclosing
+    `TableMetadata` object, so it is stated normatively and must be enforced by servers
+    and clients.
+
+    """
+
+    snapshot_id: int = Field(..., alias='snapshot-id')
+    parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id')
+    sequence_number: int | None = Field(None, alias='sequence-number')
+    timestamp_ms: int = Field(..., alias='timestamp-ms')
+    manifest_list: str | None = Field(
+        None,
+        alias='manifest-list',
+        description="Location of the snapshot's manifest list file. Used for format versions 1-3 and must be absent for format version 4, which uses `content-root` instead.",
+    )
+    content_root: str = Field(
+        ...,
+        alias='content-root',
+        description="Location of the snapshot's content root. The content root can reference data files, delete files, and other data and delete manifests in a unified structure, replacing the manifest list. Required for format version 4 and must be absent for format versions 1-3.",
     )
     first_row_id: int | None = Field(
         None,
@@ -354,6 +424,13 @@ class Snapshot(BaseModel):
     schema_id: int | None = Field(None, alias='schema-id')
 
 
+class Snapshot(RootModel[Snapshot1 | Snapshot2]):
+    root: Snapshot1 | Snapshot2 = Field(
+        ...,
+        description="A snapshot of the table's contents at a point in time.\n\n\nExactly one of `manifest-list` and `content-root` is present, enforced by the `oneOf`\nbelow. A snapshot carrying neither is invalid and must be rejected; it must not be\ninterpreted as a snapshot with no files. A snapshot carrying both is also invalid.\n\n\nWhich of the two is present is determined by the table's `format-version`: format\nversions 1-3 use `manifest-list`, and format version 4 uses `content-root`. That\ncorrelation cannot be expressed here because `format-version` lives in the enclosing\n`TableMetadata` object, so it is stated normatively and must be enforced by servers\nand clients.\n",
+    )
+
+
 class SnapshotReference(BaseModel):
     type: Literal['tag', 'branch']
     snapshot_id: int = Field(..., alias='snapshot-id')
@@ -1650,9 +1727,12 @@ class Apply(BaseModel):
 
 
 class TableMetadata(BaseModel):
-    format_version: int = Field(..., alias='format-version', ge=1, le=3)
+    format_version: int = Field(..., alias='format-version', ge=1, le=4)
     table_uuid: str = Field(..., alias='table-uuid')
-    location: str | None = None
+    location: str | None = Field(
+        None,
+        description="The table's base location. Required through format version 3, where it may be a path without a URI scheme; readers prepend a scheme for consistency with v4 absolute paths. Optional for format version 4, where the location may be managed externally and supplied by the catalog when the table is loaded, and where it must be an absolute path when present. See the `table-location` field of `LoadTableResult`.",
+    )
     last_updated_ms: int | None = Field(None, alias='last-updated-ms')
     next_row_id: int | None = Field(
         None,
@@ -1705,58 +1785,87 @@ class AddSchemaUpdate(BaseUpdate):
 
 class LoadTableResult(BaseModel):
     """
-    Result used when a table is successfully loaded.
+    Result used when a table is successfully loaded, for tables at any format version.
 
 
-    The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed.
-    Clients can check whether metadata has changed by comparing metadata locations after the table has been created.
+    The table metadata JSON is returned in the `metadata` field. The location of the table
+    metadata file is returned in the `metadata-location` field when the table has one, and
+    the table's base location in the `table-location` field.
 
 
-    The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage.
+    ## Metadata location
 
 
-    The following configurations should be respected by clients:
+    The `metadata-location` field is optional. It is absent when the metadata is staged
+    but not committed, as in a create transaction, and when the table has no
+    client-visible metadata location, as for a catalog-managed table where the catalog is
+    the source of truth for table state and no metadata pointer need exist.
+
+
+    Clients must not require this field to be present, and must not use it to bypass the
+    catalog for reads or commits. To obtain a metadata location for a catalog-managed
+    table, use the `unregisterTable` endpoint, which returns the table's last metadata
+    location at the point the table leaves catalog control and further commits are
+    rejected.
+
+
+    ## Table location
+
+
+    The `table-location` field carries the table's base location. Format version 4 allows
+    location fields in metadata to be relative, and such paths must be resolved against
+    the table location. Format version 4 also makes `metadata.location` optional, so
+    a table may have metadata that contains relative paths and omits `location`. Servers
+    must populate `table-location` for any such table, because it cannot be read
+    otherwise.
+
+
+    When both `metadata.location` and `table-location` are present, `table-location`
+    takes precedence: the catalog is authoritative for table state, and catalog-supplied
+    locations are what allow a table to be relocated without rewriting metadata.
+
+
+    ## Configuration
+
+
+    The `config` map returns table-specific configuration for the table's resources,
+    including its HTTP client and FileIO. For example, config may contain a specific FileIO
+    implementation class for the table depending on its underlying storage. The following
+    configurations should be respected by clients:
 
-    ## General Configurations
 
     - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled
     - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values:
       - `client`: Clients MUST use client-side scan planning
       - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint
 
-    ## AWS Configurations
+    The following configurations should be respected when working with tables stored in AWS S3:
 
-    The following configurations should be respected when working with tables stored in AWS S3
      - `client.region`: region to configure client for making requests to AWS
-     - `s3.access-key-id`: id for credentials that provide access to the data in S3
-     - `s3.secret-access-key`: secret for credentials that provide access to data in S3
-     - `s3.session-token`: if present, this value should be used for as the session token
      - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document.
      - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled
 
-    ## Storage Credentials
+    ## Storage credentials and remote signing
 
-    Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field.
-    Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials.
 
-    ## Remote Signing
-
-    If remote signing for a specific storage provider is enabled, the server SHOULD use the `remote-signing-config`
-    field to communicate all signer client settings. When the `remote-signing-config` field is present, clients
-    SHOULD respect the provided configuration.
-
-    For backward compatibility, the following `config` properties are still supported but **DEPRECATED** and SHOULD NOT be used by clients able to consume the remote signing configuration:
-     - `signer.endpoint` **DEPRECATED**.: the remote signer endpoint. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI.
-     - `signer.uri` **DEPRECATED**.: the base URI to resolve `signer.endpoint` against. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set.
-    If any of these properties is present, clients SHOULD use them to compute the actual remote signing endpoint URI to contact.
-    If none of these properties is present, clients SHOULD contact the default remote signing endpoint using the catalog's base URI.
+    Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials`
+    field, and remote signer client settings through the `remote-signing-config` field.
+    These are the only mechanisms for communicating credentials and signer settings.
+    Because `config` is an open map, this is stated explicitly: servers must not place
+    storage credentials or remote signer settings in `config`, and clients must not look
+    for them there.
 
     """
 
     metadata_location: str | None = Field(
         None,
         alias='metadata-location',
-        description='May be null if the table is staged as part of a transaction',
+        description='Location of the table metadata file. Absent when the metadata is staged but not committed, as in a create transaction, and when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.',
+    )
+    table_location: str | None = Field(
+        None,
+        alias='table-location',
+        description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.",
     )
     metadata: TableMetadata
     config: dict[str, str] | None = None
@@ -1841,6 +1950,11 @@ class UnregisterTableResult(BaseModel):
         alias='metadata-location',
         description='The last metadata location for the table at the time it was unregistered.',
     )
+    table_location: str | None = Field(
+        None,
+        alias='table-location',
+        description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.",
+    )
     metadata: TableMetadata
 
 
@@ -2026,7 +2140,34 @@ class FunctionStructField(BaseModel):
 
 
 class CommitTableResponse(BaseModel):
-    metadata_location: str = Field(..., alias='metadata-location')
+    """
+    Result used when a table is successfully updated, for tables at any format version.
+
+
+    The table metadata JSON is returned in the `metadata` field. The location of the
+    committed table metadata file is returned in the `metadata-location` field when the
+    table has one, and is absent when it does not, as for a catalog-managed table where
+    the catalog is the source of truth for table state.
+
+
+    The `table-location` field carries the table's base location, so that a client
+    resolving relative paths in the returned metadata does not need an additional
+    `loadTable` call to obtain the base location. Servers must populate it whenever the
+    returned metadata contains relative paths and omits `location`, because the metadata
+    cannot be resolved otherwise; without it the response would not be self-sufficient.
+
+    """
+
+    metadata_location: str | None = Field(
+        None,
+        alias='metadata-location',
+        description='Location of the committed table metadata file. Absent when the table has no client-visible metadata location, as for a catalog-managed table where the catalog is the source of truth for table state.',
+    )
+    table_location: str | None = Field(
+        None,
+        alias='table-location',
+        description="The table's base location, used to resolve relative paths in metadata. Must be an absolute path with a URI scheme when present, and must be present when the returned metadata contains relative paths and omits `location`, because the metadata cannot be resolved otherwise. Takes precedence over `metadata.location`.",
+    )
     metadata: TableMetadata