diff --git a/docker-compose.yml b/docker-compose.yml index 517d9928..4d944cc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,12 @@ services: # S3 Protocol S3_PROTOCOL_ACCESS_KEY_ID: 625729a08b95bf1b7ff351a663f3a23c S3_PROTOCOL_ACCESS_KEY_SECRET: 850181e4652dd023b7a98c58ae0d2d34bd487ee0cc3254aed6eda37307425907 + # Iceberg Protocol + ICEBERG_BUCKET_DETECTION_MODE: "FULL_PATH" + ICEBERG_CATALOG_URL: http://rest-catalog:8181/v1 + ICEBERG_CATALOG_AUTH_TYPE: token + ICEBERG_CATALOG_AUTH_TOKEN: token + ICEBERG_S3_DELETE_ENABLED: true tenant_db: extends: @@ -74,6 +80,13 @@ services: service: imgproxy file: ./.docker/docker-compose-infra.yml + rest-catalog: + depends_on: + - minio_setup + extends: + service: rest-catalog + file: ./.docker/docker-compose-infra.yml + # Optional for rate-limiting # redis: # extends: diff --git a/src/config.ts b/src/config.ts index 57226135..5f52a879 100644 --- a/src/config.ts +++ b/src/config.ts @@ -183,6 +183,7 @@ type StorageConfigType = { icebergMaxCatalogsCount: number icebergBucketDetectionSuffix: string icebergBucketDetectionMode: 'BUCKET' | 'FULL_PATH' + icebergS3DeleteEnabled: boolean } function getOptionalConfigFromEnv(key: string, fallback?: string): string | undefined { @@ -518,6 +519,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { 10 ), icebergMaxTableCount: parseInt(getOptionalConfigFromEnv('ICEBERG_MAX_TABLES') || '10', 10), + icebergS3DeleteEnabled: getOptionalConfigFromEnv('ICEBERG_S3_DELETE_ENABLED') === 'true', } as StorageConfigType const serviceKey = getOptionalConfigFromEnv('SERVICE_KEY') || '' diff --git a/src/http/routes/s3/commands/delete-object.ts b/src/http/routes/s3/commands/delete-object.ts index 4ee6b0d2..7ef70830 100644 --- a/src/http/routes/s3/commands/delete-object.ts +++ b/src/http/routes/s3/commands/delete-object.ts @@ -1,6 +1,7 @@ import { S3ProtocolHandler } from '@storage/protocols/s3/s3-handler' import { S3Router } from '../router' import { ROUTE_OPERATIONS } from '../../operations' +import { getConfig } from '../../../../config' const DeleteObjectInput = { summary: 'Delete Object', @@ -55,6 +56,8 @@ const DeleteObjectsInput = { }, } as const +const { icebergS3DeleteEnabled } = getConfig() + export default function DeleteObject(s3Router: S3Router) { // Delete multiple objects s3Router.post( @@ -85,4 +88,23 @@ export default function DeleteObject(s3Router: S3Router) { }) } ) + + // Delete single object + if (icebergS3DeleteEnabled) { + s3Router.delete( + '/:Bucket/*', + { type: 'iceberg', schema: DeleteObjectInput, operation: ROUTE_OPERATIONS.S3_DELETE_OBJECT }, + async (req, ctx) => { + const internalBucketName = ctx.req.internalIcebergBucketName + + if (!internalBucketName) { + throw new Error('Iceberg bucket name is required') + } + + await ctx.req.storage.backend.deleteObject(internalBucketName, req.Params['*'], undefined) + + return {} + } + ) + } } diff --git a/src/storage/protocols/iceberg/knex.ts b/src/storage/protocols/iceberg/knex.ts index c0e77792..52771e94 100644 --- a/src/storage/protocols/iceberg/knex.ts +++ b/src/storage/protocols/iceberg/knex.ts @@ -192,10 +192,7 @@ export class KnexMetastore implements Metastore { async findCatalogById(param: { tenantId: string; id: string }): Promise { const table = this.ops.multiTenant ? 'iceberg_catalogs' : 'buckets_analytics' - const query = this.db - .withSchema(this.ops.schema) - .table(table) - .select('id', 'type') + const query = this.db.withSchema(this.ops.schema).table(table).select('id') if (this.ops.multiTenant) { query.select('tenant_id')