-
Notifications
You must be signed in to change notification settings - Fork 3.8k
feat(files): let the agent read HEIC photos #6346
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| /** | ||
| * @vitest-environment node | ||
| */ | ||
| import { describe, expect, it } from 'vitest' | ||
| import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' | ||
|
|
||
| /** | ||
| * An ISO-BMFF `ftyp` box: 4-byte size, the `ftyp` marker, the major brand, a | ||
| * 4-byte minor version, then any compatible brands. | ||
| */ | ||
| function ftypHeader(brand: string, compatible: string[] = []): Buffer { | ||
| const size = 16 + compatible.length * 4 | ||
| const header = Buffer.alloc(size) | ||
| header.writeUInt32BE(size, 0) | ||
| header.write('ftyp', 4, 'ascii') | ||
| header.write(brand, 8, 'ascii') | ||
| compatible.forEach((entry, index) => header.write(entry, 16 + index * 4, 'ascii')) | ||
| return header | ||
| } | ||
|
|
||
| describe('isHeifContainer', () => { | ||
| it.each(['heic', 'heix', 'heim', 'heis', 'hevc', 'hevx', 'mif1', 'msf1'])( | ||
| 'detects the %s brand', | ||
| (brand) => { | ||
| expect(isHeifContainer(ftypHeader(brand))).toBe(true) | ||
| } | ||
| ) | ||
|
|
||
| it.each(['avif', 'avis'])( | ||
| 'also claims the %s brand — the question is "is this HEIF", not "which codec"', | ||
| (brand) => { | ||
| expect(isHeifContainer(ftypHeader(brand))).toBe(true) | ||
| } | ||
| ) | ||
|
|
||
| it('rejects other image formats', () => { | ||
| expect(isHeifContainer(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe( | ||
| false | ||
| ) | ||
| expect(isHeifContainer(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe( | ||
| false | ||
| ) | ||
| }) | ||
|
|
||
| it('rejects a HEIF brand that is not behind an ftyp box', () => { | ||
| const riff = Buffer.alloc(16) | ||
| riff.write('RIFF', 0, 'ascii') | ||
| riff.write('heic', 8, 'ascii') | ||
| expect(isHeifContainer(riff)).toBe(false) | ||
| }) | ||
|
|
||
| it('rejects an unknown brand in a well-formed ftyp box', () => { | ||
| expect(isHeifContainer(ftypHeader('qt '))).toBe(false) | ||
| }) | ||
|
|
||
| it('detects a HEIF brand declared only among the compatible brands', () => { | ||
| // Standards-valid: a generic major brand with the HEIF brand listed after it. | ||
| expect(isHeifContainer(ftypHeader('isom', ['iso2', 'heic', 'mif1']))).toBe(true) | ||
| expect(isHeifContainer(ftypHeader('mp42', ['heix']))).toBe(true) | ||
| }) | ||
|
|
||
| it('rejects a box whose compatible brands are all non-HEIF', () => { | ||
| expect(isHeifContainer(ftypHeader('isom', ['iso2', 'mp41', 'mp42']))).toBe(false) | ||
| }) | ||
|
|
||
| it('does not read compatible brands past the declared box size', () => { | ||
| const truncated = ftypHeader('isom', ['heic']) | ||
| truncated.writeUInt32BE(16, 0) | ||
| expect(isHeifContainer(truncated)).toBe(false) | ||
| }) | ||
|
|
||
| it('rejects buffers too short to carry a brand', () => { | ||
| expect(isHeifContainer(Buffer.alloc(0))).toBe(false) | ||
| expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false) | ||
| }) | ||
| }) | ||
|
|
||
| describe('transcodeHeicToJpeg', () => { | ||
| it('returns null for bytes libheif cannot decode', async () => { | ||
| // Also proves the dynamic `heic-convert` import resolves at runtime, which no | ||
| // amount of type-checking establishes for a lazily loaded WebAssembly module. | ||
| expect(await transcodeHeicToJpeg(ftypHeader('heic'))).toBeNull() | ||
| }) | ||
| }) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| import { createLogger } from '@sim/logger' | ||
| import { getErrorMessage } from '@sim/utils/errors' | ||
|
|
||
| const logger = createLogger('HeicTranscode') | ||
|
|
||
| /** | ||
| * ISO-BMFF major brands in the HEIF family. The brand occupies bytes 8-11, | ||
| * immediately after the `ftyp` box marker at 4-7. | ||
| * | ||
| * The list is deliberately broad, `avif` included. It answers "are these bytes | ||
| * worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot | ||
| * answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1. | ||
| */ | ||
| const HEIF_BRANDS = new Set([ | ||
| 'heic', | ||
| 'heix', | ||
| 'heim', | ||
| 'heis', | ||
| 'hevc', | ||
| 'hevx', | ||
| 'mif1', | ||
| 'msf1', | ||
| 'avif', | ||
| 'avis', | ||
| ]) | ||
|
|
||
| /** | ||
| * Whether these bytes are an ISO-BMFF container in the HEIF family. | ||
| * | ||
| * Sniffed rather than read off the declared type because the common case is a | ||
| * `.heic` stored as `application/octet-stream`, where the declared type says | ||
| * nothing at all. | ||
| */ | ||
| export function isHeifContainer(buffer: Buffer): boolean { | ||
| if (buffer.length < 12) return false | ||
| if (buffer.toString('ascii', 4, 8) !== 'ftyp') return false | ||
| if (HEIF_BRANDS.has(buffer.toString('ascii', 8, 12))) return true | ||
|
|
||
| // A standards-valid HEIF may carry a generic major brand such as `isom` and name | ||
| // the HEIF brand only among the compatible brands, which follow the 4-byte | ||
| // minor_version at offset 12 and run to the end of the box. A declared size of 0 | ||
| // or 1 (the ISO-BMFF size escapes, which `ftyp` does not use) leaves `end` below | ||
| // the loop's start, so those simply do not scan. | ||
| const end = Math.min(buffer.readUInt32BE(0), buffer.length) | ||
| for (let offset = 16; offset + 4 <= end; offset += 4) { | ||
| if (HEIF_BRANDS.has(buffer.toString('ascii', offset, offset + 4))) return true | ||
| } | ||
| return false | ||
| } | ||
|
waleedlatif1 marked this conversation as resolved.
|
||
|
|
||
| /** | ||
| * Transcode a HEVC-coded HEIF still to JPEG. | ||
| * | ||
| * Two reasons, neither with a workaround: no vision model accepts HEIC (the Claude | ||
| * Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's prebuilt libvips | ||
| * ships libheif with AV1 but not HEVC — it decodes AVIF and rejects an iPhone photo. | ||
| * | ||
| * Returns `null` when the bytes cannot be decoded; never a partial image. | ||
| */ | ||
| export async function transcodeHeicToJpeg(buffer: Buffer): Promise<Buffer | null> { | ||
| try { | ||
| const convert = (await import('heic-convert')).default | ||
| const jpeg = await convert({ buffer, format: 'JPEG' }) | ||
| logger.info('Transcoded HEIC image', { | ||
| inputBytes: buffer.length, | ||
| outputBytes: jpeg.length, | ||
| }) | ||
| return Buffer.from(jpeg) | ||
| } catch (error) { | ||
| logger.warn('Failed to transcode HEIC image', { | ||
| bytes: buffer.length, | ||
| brand: buffer.toString('ascii', 8, 12), | ||
| error: getErrorMessage(error), | ||
| }) | ||
| return null | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.