diff --git a/services/datalake/pod-datalake/Dockerfile b/services/datalake/pod-datalake/Dockerfile index ebb6e0c8e8b..e74c1c8c5ae 100644 --- a/services/datalake/pod-datalake/Dockerfile +++ b/services/datalake/pod-datalake/Dockerfile @@ -1,4 +1,4 @@ -FROM hardcoreeng/front-base:v20250916 +FROM hardcoreeng/base-slim:v20250916 WORKDIR /app COPY bundle/bundle.js ./ diff --git a/services/datalake/pod-datalake/package.json b/services/datalake/pod-datalake/package.json index 8cb98a4aebf..36e5c10b75c 100644 --- a/services/datalake/pod-datalake/package.json +++ b/services/datalake/pod-datalake/package.json @@ -17,7 +17,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "node ../../../common/scripts/esbuild.js --keep-names=true --sourcemap=external --external=sharp", + "bundle": "node ../../../common/scripts/esbuild.js --keep-names=true --sourcemap=external", "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/datalake", "docker:tbuild": "docker build -t hardcoreeng/datalake . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/datalake", "docker:abuild": "docker build -t hardcoreeng/datalake . --platform=linux/arm64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/datalake", @@ -72,7 +72,6 @@ "express": "^4.21.2", "express-fileupload": "^1.5.1", "postgres": "^3.4.7", - "sharp": "~0.34.3", "@aws-sdk/client-s3": "^3.738.0", "@aws-sdk/s3-request-presigner": "^3.738.0", "@aws-sdk/lib-storage": "^3.738.0", diff --git a/services/datalake/pod-datalake/src/handlers/image.ts b/services/datalake/pod-datalake/src/handlers/image.ts deleted file mode 100644 index 49334b2bcc6..00000000000 --- a/services/datalake/pod-datalake/src/handlers/image.ts +++ /dev/null @@ -1,328 +0,0 @@ -// -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Analytics } from '@hcengineering/analytics' -import { MeasureContext, type WorkspaceUuid } from '@hcengineering/core' -import { type Request, type Response } from 'express' -import { createReadStream, createWriteStream } from 'fs' -import sharp from 'sharp' -import { Readable } from 'stream' -import { pipeline } from 'stream/promises' - -import config from '../config' -import { CacheEntry, type Datalake, createCache, streamToBuffer } from '../datalake' -import { TemporaryDir } from '../tempdir' - -const cacheControl = 'public, max-age=31536000, immutable' -const prefferedImageFormats = ['webp', 'avif', 'jpeg', 'png'] - -const cache = createCache(config.Cache) - -const QualityConfig = { - jpeg: { - quality: 85, // default + 5 - progressive: true, - chromaSubsampling: '4:4:4' - } satisfies sharp.JpegOptions, - avif: { - quality: 60, // default + 10 - effort: 5, // default + 1 - chromaSubsampling: '4:4:4' // default - } satisfies sharp.AvifOptions, - webp: { - quality: 80, // default - alphaQuality: 100, // default - smartSubsample: true, // Better sharpness - effort: 5 // default + 1 - } satisfies sharp.WebpOptions, - heif: { - quality: 80, // default + 30 - effort: 5 // default + 1 - } satisfies sharp.HeifOptions, - png: { - quality: 100, // default - effort: 7 // default - } satisfies sharp.PngOptions -} - -interface ImageTransform { - format: string - dpr?: number - width?: number - height?: number - fit?: 'cover' | 'contain' -} - -function parseImageTransform (accept: string, transform: string): ImageTransform { - const image: ImageTransform = { - format: 'jpeg', - dpr: 1, - fit: 'cover' - } - - // select format based on Accept header - const formats = accept.split(',') - for (const format of formats) { - const [type] = format.split(';') - const [clazz, kind] = type.split('/') - if (clazz === 'image' && prefferedImageFormats.includes(kind)) { - image.format = kind - break - } - } - - // parse transforms - transform.split(',').forEach((param) => { - const [key, value] = param.split('=') - switch (key) { - case 'dpr': - image.dpr = parseFloat(value) - break - case 'width': - image.width = parseInt(value) - break - case 'height': - image.height = parseInt(value) - break - } - }) - - return image -} - -export async function handleImageGet ( - ctx: MeasureContext, - req: Request, - res: Response, - datalake: Datalake, - tempDir: TemporaryDir -): Promise { - const { name, transform } = req.params - const workspace = req.params.workspace as WorkspaceUuid - - const accept = req.headers.accept ?? 'image/*' - const { format, width, height, fit } = getImageTransformParams(accept, transform) - - const tmpFile = tempDir.tmpFile() - const outFile = tempDir.tmpFile() - - const cleanup = (): void => { - tempDir.rm(tmpFile, outFile) - } - - req.on('error', cleanup) - req.on('close', cleanup) - res.on('error', cleanup) - res.on('finish', cleanup) - - const cached = cacheGet(workspace, name, { width, height, format, fit }) - if (cached !== undefined) { - await writeCacheEntryToResponse(ctx, cached, res) - return - } - - const blob = await datalake.get(ctx, workspace, name, {}) - if (blob == null) { - res.status(404).send() - return - } - - try { - await writeTempFile(tmpFile, blob.body) - } finally { - blob.body.destroy() - } - - try { - const { contentType, size } = await ctx.with( - 'sharp', - { format }, - () => { - return runPipeline(tmpFile, outFile, { format, width, height, fit }) - }, - { fit, width, height, size: blob.size } - ) - - if (cacheEnabled(size)) { - const buffer = await streamToBuffer(createReadStream(outFile)) - - const entry = { - name: blob.name, - etag: blob.etag, - size: blob.size, - bodyEtag: blob.etag, - bodyLength: buffer.length, - lastModified: blob.lastModified, - body: buffer, - contentType, - cacheControl - } - - cachePut(workspace, name, { width, height, format, fit }, entry) - - await writeCacheEntryToResponse(ctx, entry, res) - } else { - await writeFileToResponse(ctx, outFile, res, { contentType, cacheControl }) - } - } catch (err: any) { - Analytics.handleError(err) - ctx.error('image processing error', { workspace, name, error: err }) - - const headers = { - contentType: blob.contentType, - cacheControl: blob.cacheControl ?? cacheControl - } - - await writeFileToResponse(ctx, tmpFile, res, headers) - } -} - -interface ImageTransformParams { - format: string - width: number | undefined - height: number | undefined - fit: 'cover' | 'contain' -} - -async function runPipeline ( - inFile: string, - outFile: string, - params: ImageTransformParams -): Promise<{ contentType: string, size: number }> { - const { format, width, height, fit } = params - - let pipeline: sharp.Sharp | undefined - - try { - pipeline = sharp(inFile, { sequentialRead: true }) - - // auto orient image based on exif to prevent resize use wrong orientation - pipeline = pipeline.rotate() - - pipeline.resize({ - width, - height, - fit, - withoutEnlargement: true - }) - - let contentType = 'image/jpeg' - switch (format) { - case 'jpeg': - pipeline = pipeline.jpeg(QualityConfig.jpeg) - contentType = 'image/jpeg' - break - case 'avif': - pipeline = pipeline.avif(QualityConfig.avif) - contentType = 'image/avif' - break - case 'heif': - pipeline = pipeline.heif(QualityConfig.heif) - contentType = 'image/heif' - break - case 'webp': - pipeline = pipeline.webp(QualityConfig.webp) - contentType = 'image/webp' - break - case 'png': - pipeline = pipeline.png(QualityConfig.png) - contentType = 'image/png' - break - } - - const { size } = await pipeline.toFile(outFile) - - return { contentType, size } - } finally { - pipeline?.destroy() - } -} - -function getImageTransformParams (accept: string, transform: string): ImageTransformParams { - const image = parseImageTransform(accept, transform) - const format = image.format - - const dpr = image.dpr === undefined || Number.isNaN(image.dpr) ? 1 : image.dpr - const width = - image.width === undefined || Number.isNaN(image.width) ? undefined : Math.min(Math.round(image.width * dpr), 2048) - const height = - image.height === undefined || Number.isNaN(image.height) - ? undefined - : Math.min(Math.round(image.height * dpr), 2048) - const fit = image.fit ?? 'cover' - - return { format, width, height, fit } -} - -async function writeTempFile (path: string, stream: Readable): Promise { - const outp = createWriteStream(path) - await pipeline(stream, outp) -} - -async function writeCacheEntryToResponse (ctx: MeasureContext, cached: CacheEntry, res: Response): Promise { - const readable = Readable.from(cached.body) - await writeToResponse(ctx, readable, res, { contentType: cached.contentType, cacheControl }) -} - -async function writeFileToResponse ( - ctx: MeasureContext, - path: string, - res: Response, - headers: { contentType: string, cacheControl: string } -): Promise { - const stream = createReadStream(path) - await writeToResponse(ctx, stream, res, headers) -} - -async function writeToResponse ( - ctx: MeasureContext, - stream: Readable, - res: Response, - headers: { contentType: string, cacheControl: string } -): Promise { - res.setHeader('Content-Type', headers.contentType) - res.setHeader('Cache-Control', headers.cacheControl) - - try { - await pipeline(stream, res) - } catch (err: any) { - // ignore abort errors to avoid flooding the logs - if (err.name === 'AbortError' || err.code === 'ERR_STREAM_PREMATURE_CLOSE') { - return - } - Analytics.handleError(err) - const error = err instanceof Error ? err.message : String(err) - ctx.error('error writing response', { error }) - if (!res.headersSent) { - res.status(500).send('Internal Server Error') - } - } -} - -function cacheKey (workspace: WorkspaceUuid, name: string, params: ImageTransformParams): string { - return `${workspace}/${name}/${params.width ?? 0}/${params.height ?? 0}/${params.format}/${params.fit}` -} - -function cacheEnabled (size: number): boolean { - return cache.enabled(size) -} - -function cacheGet (workspace: WorkspaceUuid, name: string, params: ImageTransformParams): CacheEntry | undefined { - return cache.get(cacheKey(workspace, name, params)) -} - -function cachePut (workspace: WorkspaceUuid, name: string, params: ImageTransformParams, entry: CacheEntry): void { - cache.set(cacheKey(workspace, name, params), entry) -} diff --git a/services/datalake/pod-datalake/src/handlers/index.ts b/services/datalake/pod-datalake/src/handlers/index.ts index 24816503db5..eba0e508768 100644 --- a/services/datalake/pod-datalake/src/handlers/index.ts +++ b/services/datalake/pod-datalake/src/handlers/index.ts @@ -14,7 +14,6 @@ // export * from './blob' -export * from './image' export * from './meta' export * from './multipart' export * from './s3' diff --git a/services/datalake/pod-datalake/src/server.ts b/services/datalake/pod-datalake/src/server.ts index 63c724734ea..7cac5cc1903 100644 --- a/services/datalake/pod-datalake/src/server.ts +++ b/services/datalake/pod-datalake/src/server.ts @@ -44,7 +44,6 @@ import { handleBlobGet, handleBlobHead, handleBlobList, - handleImageGet, handleMetaGet, handleMetaPut, handleMetaPatch, @@ -262,10 +261,6 @@ export async function createServer ( wrapRequest(ctx, 'multipartUploadAvort', handleMultipartUploadAbort) ) - // Image - - app.get('/image/:transform/:workspace/:name', withBlob, wrapRequest(ctx, 'transformImage', handleImageGet)) // no auth - const sendErrorToAnalytics = (err: any): boolean => { const ignoreMessages = [ 'Unexpected end of form', // happens when the client closes the connection before the upload is complete