Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ services:
# S3 Protocol
S3_PROTOCOL_ACCESS_KEY_ID: 625729a08b95bf1b7ff351a663f3a23c
S3_PROTOCOL_ACCESS_KEY_SECRET: 850181e4652dd023b7a98c58ae0d2d34bd487ee0cc3254aed6eda37307425907
# Iceberg Protocol
ICEBERG_BUCKET_DETECTION_MODE: "FULL_PATH"
ICEBERG_CATALOG_URL: http://rest-catalog:8181/v1
ICEBERG_CATALOG_AUTH_TYPE: token
ICEBERG_CATALOG_AUTH_TOKEN: token
ICEBERG_S3_DELETE_ENABLED: true

tenant_db:
extends:
Expand Down Expand Up @@ -74,6 +80,13 @@ services:
service: imgproxy
file: ./.docker/docker-compose-infra.yml

rest-catalog:
depends_on:
- minio_setup
extends:
service: rest-catalog
file: ./.docker/docker-compose-infra.yml

# Optional for rate-limiting
# redis:
# extends:
Expand Down
2 changes: 2 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ type StorageConfigType = {
icebergMaxCatalogsCount: number
icebergBucketDetectionSuffix: string
icebergBucketDetectionMode: 'BUCKET' | 'FULL_PATH'
icebergS3DeleteEnabled: boolean
}

function getOptionalConfigFromEnv(key: string, fallback?: string): string | undefined {
Expand Down Expand Up @@ -518,6 +519,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType {
10
),
icebergMaxTableCount: parseInt(getOptionalConfigFromEnv('ICEBERG_MAX_TABLES') || '10', 10),
icebergS3DeleteEnabled: getOptionalConfigFromEnv('ICEBERG_S3_DELETE_ENABLED') === 'true',
} as StorageConfigType

const serviceKey = getOptionalConfigFromEnv('SERVICE_KEY') || ''
Expand Down
22 changes: 22 additions & 0 deletions src/http/routes/s3/commands/delete-object.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { S3ProtocolHandler } from '@storage/protocols/s3/s3-handler'
import { S3Router } from '../router'
import { ROUTE_OPERATIONS } from '../../operations'
import { getConfig } from '../../../../config'

const DeleteObjectInput = {
summary: 'Delete Object',
Expand Down Expand Up @@ -55,6 +56,8 @@ const DeleteObjectsInput = {
},
} as const

const { icebergS3DeleteEnabled } = getConfig()

export default function DeleteObject(s3Router: S3Router) {
// Delete multiple objects
s3Router.post(
Expand Down Expand Up @@ -85,4 +88,23 @@ export default function DeleteObject(s3Router: S3Router) {
})
}
)

// Delete single object
if (icebergS3DeleteEnabled) {
s3Router.delete(
'/:Bucket/*',
{ type: 'iceberg', schema: DeleteObjectInput, operation: ROUTE_OPERATIONS.S3_DELETE_OBJECT },
async (req, ctx) => {
const internalBucketName = ctx.req.internalIcebergBucketName

if (!internalBucketName) {
throw new Error('Iceberg bucket name is required')
}

await ctx.req.storage.backend.deleteObject(internalBucketName, req.Params['*'], undefined)

return {}
}
)
}
}
5 changes: 1 addition & 4 deletions src/storage/protocols/iceberg/knex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,10 +192,7 @@ export class KnexMetastore implements Metastore {
async findCatalogById(param: { tenantId: string; id: string }): Promise<IcebergBucket> {
const table = this.ops.multiTenant ? 'iceberg_catalogs' : 'buckets_analytics'

const query = this.db
.withSchema(this.ops.schema)
.table(table)
.select<IcebergBucket[]>('id', 'type')
const query = this.db.withSchema(this.ops.schema).table(table).select<IcebergBucket[]>('id')

if (this.ops.multiTenant) {
query.select('tenant_id')
Expand Down
Loading