From f464be6ce5afe7a87943e091f30c13976deb1285 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 14:54:49 +0800 Subject: [PATCH 01/16] Add WebP support to image fetch --- pkg/images/image.go | 1 + pkg/images/image_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/pkg/images/image.go b/pkg/images/image.go index 608c4891..a76c698f 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -16,6 +16,7 @@ import ( "time" "golang.org/x/image/draw" + _ "golang.org/x/image/webp" ) const DefaultJPEGQuality = 85 diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 9602cc95..6db6ad8d 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -1,6 +1,7 @@ package images import ( + "encoding/base64" "image" "image/color" "image/png" @@ -92,6 +93,43 @@ func TestFetchRemote(t *testing.T) { } } +func TestFetchRemoteWebP(t *testing.T) { + t.Parallel() + + const webpBase64 = "UklGRjwAAABXRUJQVlA4IDAAAADQAQCdASoBAAEAAUAmJaACdLoB+AADsAD+8ut//NgVzXPv9//S4P0uD9Lg/9KQAAA=" + + data, err := base64.StdEncoding.DecodeString(webpBase64) + if err != nil { + t.Fatalf("decode webp fixture: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.webp" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + + w.Header().Set("Content-Type", "image/webp") + if _, err := w.Write(data); err != nil { + t.Fatalf("write webp payload: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/cover.webp") + if err != nil { + t.Fatalf("fetch remote webp: %v", err) + } + + if format != "webp" { + t.Fatalf("expected webp format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 1 || bounds.Dy() != 1 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestResize(t *testing.T) { t.Parallel() From c73aa210eb2fad88d43d847e39347a0adfad541d Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:08:44 +0800 Subject: [PATCH 02/16] Handle brotli-encoded images --- go.mod | 3 ++- go.sum | 2 ++ pkg/images/image.go | 24 +++++++++++++++++++- pkg/images/image_test.go | 48 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 30c5e2c8..e572a4b4 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/testcontainers/testcontainers-go v0.38.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0 golang.org/x/crypto v0.42.0 + golang.org/x/image v0.31.0 golang.org/x/term v0.35.0 golang.org/x/text v0.29.0 gopkg.in/yaml.v3 v3.0.1 @@ -24,6 +25,7 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -79,7 +81,6 @@ require ( go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/metric v1.37.0 // indirect go.opentelemetry.io/otel/trace v1.37.0 // indirect - golang.org/x/image v0.31.0 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.36.0 // indirect diff --git a/go.sum b/go.sum index bd40345e..75e04a36 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= diff --git a/pkg/images/image.go b/pkg/images/image.go index a76c698f..9e308441 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -17,6 +17,8 @@ import ( "golang.org/x/image/draw" _ "golang.org/x/image/webp" + + "github.com/andybalholm/brotli" ) const DefaultJPEGQuality = 85 @@ -184,7 +186,7 @@ func openSource(parsed *url.URL) (io.ReadCloser, error) { return nil, fmt.Errorf("download image: unexpected status %s", resp.Status) } - return resp.Body, nil + return wrapHTTPBody(resp), nil case "file": return openLocal(parsed) case "": @@ -194,6 +196,26 @@ func openSource(parsed *url.URL) (io.ReadCloser, error) { } } +type composedReadCloser struct { + io.Reader + io.Closer +} + +func wrapHTTPBody(resp *http.Response) io.ReadCloser { + encoding := strings.TrimSpace(strings.ToLower(resp.Header.Get("Content-Encoding"))) + if idx := strings.IndexRune(encoding, ','); idx >= 0 { + encoding = encoding[:idx] + } + switch encoding { + case "", "identity": + return resp.Body + case "br": + return composedReadCloser{Reader: brotli.NewReader(resp.Body), Closer: resp.Body} + default: + return resp.Body + } +} + func openLocal(parsed *url.URL) (io.ReadCloser, error) { pathValue := parsed.Path if pathValue == "" { diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 6db6ad8d..49bdb179 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -1,6 +1,7 @@ package images import ( + "bytes" "encoding/base64" "image" "image/color" @@ -13,6 +14,8 @@ import ( "testing" _ "image/jpeg" + + "github.com/andybalholm/brotli" ) func createTestImage(width, height int) image.Image { @@ -130,6 +133,51 @@ func TestFetchRemoteWebP(t *testing.T) { } } +func TestFetchRemoteBrotliEncoded(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.png" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + + var pngBuf bytes.Buffer + if err := png.Encode(&pngBuf, createTestImage(32, 24)); err != nil { + t.Fatalf("encode png: %v", err) + } + + var brBuf bytes.Buffer + writer := brotli.NewWriterLevel(&brBuf, brotli.BestCompression) + if _, err := writer.Write(pngBuf.Bytes()); err != nil { + t.Fatalf("write brotli: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close brotli writer: %v", err) + } + + w.Header().Set("Content-Encoding", "br") + w.Header().Set("Content-Type", "image/png") + if _, err := w.Write(brBuf.Bytes()); err != nil { + t.Fatalf("write brotli payload: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/cover.png") + if err != nil { + t.Fatalf("fetch brotli image: %v", err) + } + + if format != "png" { + t.Fatalf("expected png format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 32 || bounds.Dy() != 24 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestResize(t *testing.T) { t.Parallel() From f8810fda897a2c9dafc4f9b781939622ea54f996 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:18:04 +0800 Subject: [PATCH 03/16] Support WebP post cover processing --- go.mod | 1 + go.sum | 2 ++ pkg/images/image.go | 27 ++++++++++++++++++--------- pkg/images/image_test.go | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index e572a4b4..e041980b 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/chai2010/webp v1.4.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect diff --git a/go.sum b/go.sum index 75e04a36..b6b9f2a8 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/chai2010/webp v1.4.0 h1:6DA2pkkRUPnbOHvvsmGI3He1hBKf/bkRlniAiSGuEko= +github.com/chai2010/webp v1.4.0/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= diff --git a/pkg/images/image.go b/pkg/images/image.go index 9e308441..ab6efbcc 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -19,6 +19,7 @@ import ( _ "golang.org/x/image/webp" "github.com/andybalholm/brotli" + "github.com/chai2010/webp" ) const DefaultJPEGQuality = 85 @@ -51,23 +52,26 @@ func Resize(src stdimage.Image, width, height int) stdimage.Image { } func DetermineExtension(source, format string) string { - ext := strings.ToLower(filepath.Ext(source)) - if ext == "" { - ext = "." + strings.ToLower(format) - } + ext := strings.ToLower(strings.TrimSpace(filepath.Ext(source))) + format = strings.ToLower(strings.TrimSpace(format)) switch ext { case ".jpeg": return ".jpg" - case ".jpg", ".png": + case ".jpg", ".png", ".webp": return ext - default: - if format == "png" { - return ".png" - } + } + switch format { + case "jpeg", "jpg": return ".jpg" + case "png": + return ".png" + case "webp": + return ".webp" } + + return ".jpg" } func BuildFileName(slug, ext, fallback string) string { @@ -93,6 +97,9 @@ func Save(path string, img stdimage.Image, ext string, quality int) error { case ".png": encoder := &png.Encoder{CompressionLevel: png.DefaultCompression} return encoder.Encode(fh, img) + case ".webp": + options := &webp.Options{Lossless: false, Quality: float32(quality)} + return webp.Encode(fh, img, options) default: options := &jpeg.Options{Quality: quality} return jpeg.Encode(fh, img, options) @@ -139,6 +146,8 @@ func MIMEFromExtension(ext string) string { return "image/png" case ".jpg": return "image/jpeg" + case ".webp": + return "image/webp" default: return "image/png" } diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 49bdb179..84cd8128 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -202,8 +202,10 @@ func TestDetermineExtension(t *testing.T) { {"explicit png", "example.png", "jpeg", ".png"}, {"explicit jpg", "example.jpg", "png", ".jpg"}, {"explicit jpeg", "example.jpeg", "png", ".jpg"}, + {"explicit webp", "example.webp", "jpeg", ".webp"}, {"missing ext png format", "example", "png", ".png"}, {"missing ext jpeg format", "example", "jpeg", ".jpg"}, + {"missing ext webp format", "example", "webp", ".webp"}, {"unknown ext falls back to jpg", "example.gif", "jpeg", ".jpg"}, } @@ -256,6 +258,7 @@ func TestMIMEFromExtension(t *testing.T) { ".jpg": "image/jpeg", ".jpeg": "image/png", ".gif": "image/png", + ".webp": "image/webp", } for ext, want := range tests { @@ -345,6 +348,37 @@ func TestSaveJPEG(t *testing.T) { } } +func TestSaveWebP(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "photo.webp") + + if err := Save(path, createTestImage(40, 20), ".webp", 80); err != nil { + t.Fatalf("save webp: %v", err) + } + + fh, err := os.Open(path) + if err != nil { + t.Fatalf("open webp: %v", err) + } + defer fh.Close() + + img, format, err := image.Decode(fh) + if err != nil { + t.Fatalf("decode webp: %v", err) + } + + if format != "webp" { + t.Fatalf("expected webp format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 40 || bounds.Dy() != 20 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestNormalizeRelativeURL(t *testing.T) { t.Parallel() From aa58caaffa2d1151341385f43cf8a6231dcb2181 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:25:06 +0800 Subject: [PATCH 04/16] Improve test handler error signaling --- pkg/images/image_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 84cd8128..53314e74 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -108,7 +108,9 @@ func TestFetchRemoteWebP(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/cover.webp" { - t.Fatalf("unexpected path: %s", r.URL.Path) + t.Errorf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) + return } w.Header().Set("Content-Type", "image/webp") From f247b2522f9f94348356136da530de9f7cbbbe6c Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:28:36 +0800 Subject: [PATCH 05/16] Adjust WebP test handler error reporting --- pkg/images/image_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 53314e74..102603c4 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -115,7 +115,7 @@ func TestFetchRemoteWebP(t *testing.T) { w.Header().Set("Content-Type", "image/webp") if _, err := w.Write(data); err != nil { - t.Fatalf("write webp payload: %v", err) + t.Errorf("write webp payload: %v", err) } })) defer server.Close() From 2c6663e4e3c03e0f57e38ce28a9048052c473247 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:34:41 +0800 Subject: [PATCH 06/16] Extract image globals --- pkg/images/globals.go | 10 ++++++++++ pkg/images/image.go | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 pkg/images/globals.go diff --git a/pkg/images/globals.go b/pkg/images/globals.go new file mode 100644 index 00000000..515a107c --- /dev/null +++ b/pkg/images/globals.go @@ -0,0 +1,10 @@ +package images + +import "io" + +const DefaultJPEGQuality = 85 + +type composedReadCloser struct { + io.Reader + io.Closer +} diff --git a/pkg/images/image.go b/pkg/images/image.go index ab6efbcc..7f22aab1 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -22,8 +22,6 @@ import ( "github.com/chai2010/webp" ) -const DefaultJPEGQuality = 85 - func Fetch(source string) (stdimage.Image, string, error) { parsed, err := url.Parse(source) if err != nil { @@ -205,11 +203,6 @@ func openSource(parsed *url.URL) (io.ReadCloser, error) { } } -type composedReadCloser struct { - io.Reader - io.Closer -} - func wrapHTTPBody(resp *http.Response) io.ReadCloser { encoding := strings.TrimSpace(strings.ToLower(resp.Header.Get("Content-Encoding"))) if idx := strings.IndexRune(encoding, ','); idx >= 0 { From 5e5a24ac8532dad1d7b371722b3a0368ac983e86 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:42:25 +0800 Subject: [PATCH 07/16] Ensure remote image fetch prefers supported formats --- pkg/images/globals.go | 5 ++++- pkg/images/image.go | 10 +++++++++- pkg/images/image_test.go | 43 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/pkg/images/globals.go b/pkg/images/globals.go index 515a107c..11480f81 100644 --- a/pkg/images/globals.go +++ b/pkg/images/globals.go @@ -2,7 +2,10 @@ package images import "io" -const DefaultJPEGQuality = 85 +const ( + DefaultJPEGQuality = 85 + supportedImageAcceptHeader = "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1" +) type composedReadCloser struct { io.Reader diff --git a/pkg/images/image.go b/pkg/images/image.go index 7f22aab1..c211e141 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -183,7 +183,15 @@ func openSource(parsed *url.URL) (io.ReadCloser, error) { switch parsed.Scheme { case "http", "https": client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Get(parsed.String()) + + req, err := http.NewRequest(http.MethodGet, parsed.String(), nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + req.Header.Set("Accept", supportedImageAcceptHeader) + + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("download image: %w", err) } diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 102603c4..c9d3bfc7 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "testing" _ "image/jpeg" @@ -135,6 +136,48 @@ func TestFetchRemoteWebP(t *testing.T) { } } +func TestFetchRemoteSetsAcceptHeader(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.png" { + t.Errorf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) + return + } + + accept := r.Header.Get("Accept") + if strings.Contains(accept, "image/avif") { + t.Errorf("unexpected avif accept header: %s", accept) + w.WriteHeader(http.StatusNotAcceptable) + return + } + + if accept != supportedImageAcceptHeader { + t.Errorf("unexpected accept header: %s", accept) + } + + if err := png.Encode(w, createTestImage(10, 10)); err != nil { + t.Errorf("encode png: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/cover.png") + if err != nil { + t.Fatalf("fetch remote image: %v", err) + } + + if format != "png" { + t.Fatalf("expected png format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 10 || bounds.Dy() != 10 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestFetchRemoteBrotliEncoded(t *testing.T) { t.Parallel() From f4266d795c9ef20c7e8912232a48e3d682bdb40f Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 15:54:48 +0800 Subject: [PATCH 08/16] Improve remote image fetch robustness --- pkg/images/globals.go | 1 + pkg/images/image.go | 25 ++++++++++++++++--------- pkg/images/image_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/pkg/images/globals.go b/pkg/images/globals.go index 11480f81..fe6bc6c9 100644 --- a/pkg/images/globals.go +++ b/pkg/images/globals.go @@ -5,6 +5,7 @@ import "io" const ( DefaultJPEGQuality = 85 supportedImageAcceptHeader = "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1" + defaultRemoteImageUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15" ) type composedReadCloser struct { diff --git a/pkg/images/image.go b/pkg/images/image.go index c211e141..2e241e25 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -28,7 +28,7 @@ func Fetch(source string) (stdimage.Image, string, error) { return nil, "", fmt.Errorf("parse url: %w", err) } - reader, err := openSource(parsed) + reader, contentType, err := openSource(parsed) if err != nil { return nil, "", err } @@ -36,6 +36,10 @@ func Fetch(source string) (stdimage.Image, string, error) { img, format, err := stdimage.Decode(reader) if err != nil { + ct := strings.TrimSpace(contentType) + if ct != "" { + return nil, "", fmt.Errorf("decode image (content-type %q): %w", ct, err) + } return nil, "", fmt.Errorf("decode image: %w", err) } @@ -179,35 +183,38 @@ func NormalizeRelativeURL(rel string) string { return b.String() } -func openSource(parsed *url.URL) (io.ReadCloser, error) { +func openSource(parsed *url.URL) (io.ReadCloser, string, error) { switch parsed.Scheme { case "http", "https": client := &http.Client{Timeout: 10 * time.Second} req, err := http.NewRequest(http.MethodGet, parsed.String(), nil) if err != nil { - return nil, fmt.Errorf("create request: %w", err) + return nil, "", fmt.Errorf("create request: %w", err) } req.Header.Set("Accept", supportedImageAcceptHeader) + req.Header.Set("User-Agent", defaultRemoteImageUA) resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("download image: %w", err) + return nil, "", fmt.Errorf("download image: %w", err) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() - return nil, fmt.Errorf("download image: unexpected status %s", resp.Status) + return nil, "", fmt.Errorf("download image: unexpected status %s", resp.Status) } - return wrapHTTPBody(resp), nil + return wrapHTTPBody(resp), resp.Header.Get("Content-Type"), nil case "file": - return openLocal(parsed) + reader, err := openLocal(parsed) + return reader, "", err case "": - return os.Open(parsed.Path) + reader, err := os.Open(parsed.Path) + return reader, "", err default: - return nil, fmt.Errorf("unsupported image scheme: %s", parsed.Scheme) + return nil, "", fmt.Errorf("unsupported image scheme: %s", parsed.Scheme) } } diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index c9d3bfc7..36b2426d 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -157,6 +157,10 @@ func TestFetchRemoteSetsAcceptHeader(t *testing.T) { t.Errorf("unexpected accept header: %s", accept) } + if ua := r.Header.Get("User-Agent"); ua != defaultRemoteImageUA { + t.Errorf("unexpected user agent: %s", ua) + } + if err := png.Encode(w, createTestImage(10, 10)); err != nil { t.Errorf("encode png: %v", err) } @@ -223,6 +227,26 @@ func TestFetchRemoteBrotliEncoded(t *testing.T) { } } +func TestFetchRemoteDecodeErrorIncludesContentType(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("")) + })) + defer server.Close() + + _, _, err := Fetch(server.URL + "/cover.png") + if err == nil { + t.Fatal("expected error decoding html response") + } + + if !strings.Contains(err.Error(), "content-type \"text/html\"") { + t.Fatalf("expected content-type in error, got %v", err) + } +} + func TestResize(t *testing.T) { t.Parallel() From 875c79d930760ff7fce384cbc68a42b663a5163c Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 16:10:10 +0800 Subject: [PATCH 09/16] Handle compressed remote SEO images --- pkg/images/globals.go | 30 ++++++++++++++ pkg/images/image.go | 65 +++++++++++++++++++++-------- pkg/images/image_test.go | 88 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 16 deletions(-) diff --git a/pkg/images/globals.go b/pkg/images/globals.go index fe6bc6c9..50208663 100644 --- a/pkg/images/globals.go +++ b/pkg/images/globals.go @@ -12,3 +12,33 @@ type composedReadCloser struct { io.Reader io.Closer } + +type multiCloser []io.Closer + +func (m multiCloser) Close() error { + var firstErr error + + for _, closer := range m { + if closer == nil { + continue + } + + if err := closer.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + + return firstErr +} + +type noErrorCloseFunc func() + +func (f noErrorCloseFunc) Close() error { + if f == nil { + return nil + } + + f() + + return nil +} diff --git a/pkg/images/image.go b/pkg/images/image.go index 2e241e25..1cc97a4c 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -1,6 +1,7 @@ package images import ( + "compress/gzip" "fmt" stdimage "image" _ "image/gif" @@ -20,6 +21,7 @@ import ( "github.com/andybalholm/brotli" "github.com/chai2010/webp" + "github.com/klauspost/compress/zstd" ) func Fetch(source string) (stdimage.Image, string, error) { @@ -28,7 +30,7 @@ func Fetch(source string) (stdimage.Image, string, error) { return nil, "", fmt.Errorf("parse url: %w", err) } - reader, contentType, err := openSource(parsed) + reader, contentType, encoding, err := openSource(parsed) if err != nil { return nil, "", err } @@ -36,10 +38,20 @@ func Fetch(source string) (stdimage.Image, string, error) { img, format, err := stdimage.Decode(reader) if err != nil { - ct := strings.TrimSpace(contentType) - if ct != "" { - return nil, "", fmt.Errorf("decode image (content-type %q): %w", ct, err) + var details []string + + if ct := strings.TrimSpace(contentType); ct != "" { + details = append(details, fmt.Sprintf("content-type %q", ct)) + } + + if enc := strings.TrimSpace(encoding); enc != "" { + details = append(details, fmt.Sprintf("content-encoding %q", enc)) + } + + if len(details) > 0 { + return nil, "", fmt.Errorf("decode image (%s): %w", strings.Join(details, ", "), err) } + return nil, "", fmt.Errorf("decode image: %w", err) } @@ -183,14 +195,14 @@ func NormalizeRelativeURL(rel string) string { return b.String() } -func openSource(parsed *url.URL) (io.ReadCloser, string, error) { +func openSource(parsed *url.URL) (io.ReadCloser, string, string, error) { switch parsed.Scheme { case "http", "https": client := &http.Client{Timeout: 10 * time.Second} req, err := http.NewRequest(http.MethodGet, parsed.String(), nil) if err != nil { - return nil, "", fmt.Errorf("create request: %w", err) + return nil, "", "", fmt.Errorf("create request: %w", err) } req.Header.Set("Accept", supportedImageAcceptHeader) @@ -198,38 +210,59 @@ func openSource(parsed *url.URL) (io.ReadCloser, string, error) { resp, err := client.Do(req) if err != nil { - return nil, "", fmt.Errorf("download image: %w", err) + return nil, "", "", fmt.Errorf("download image: %w", err) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() - return nil, "", fmt.Errorf("download image: unexpected status %s", resp.Status) + return nil, "", "", fmt.Errorf("download image: unexpected status %s", resp.Status) } - return wrapHTTPBody(resp), resp.Header.Get("Content-Type"), nil + reader, encoding, err := wrapHTTPBody(resp) + if err != nil { + return nil, "", "", err + } + + return reader, resp.Header.Get("Content-Type"), encoding, nil case "file": reader, err := openLocal(parsed) - return reader, "", err + return reader, "", "", err case "": reader, err := os.Open(parsed.Path) - return reader, "", err + return reader, "", "", err default: - return nil, "", fmt.Errorf("unsupported image scheme: %s", parsed.Scheme) + return nil, "", "", fmt.Errorf("unsupported image scheme: %s", parsed.Scheme) } } -func wrapHTTPBody(resp *http.Response) io.ReadCloser { +func wrapHTTPBody(resp *http.Response) (io.ReadCloser, string, error) { encoding := strings.TrimSpace(strings.ToLower(resp.Header.Get("Content-Encoding"))) if idx := strings.IndexRune(encoding, ','); idx >= 0 { encoding = encoding[:idx] } switch encoding { case "", "identity": - return resp.Body + return resp.Body, encoding, nil case "br": - return composedReadCloser{Reader: brotli.NewReader(resp.Body), Closer: resp.Body} + return composedReadCloser{Reader: brotli.NewReader(resp.Body), Closer: resp.Body}, encoding, nil + case "gzip": + reader, err := gzip.NewReader(resp.Body) + if err != nil { + _ = resp.Body.Close() + return nil, encoding, fmt.Errorf("prepare gzip decoder: %w", err) + } + + return composedReadCloser{Reader: reader, Closer: multiCloser{reader, resp.Body}}, encoding, nil + case "zstd", "zstandard": + decoder, err := zstd.NewReader(resp.Body) + if err != nil { + _ = resp.Body.Close() + return nil, encoding, fmt.Errorf("prepare zstd decoder: %w", err) + } + + return composedReadCloser{Reader: decoder, Closer: multiCloser{noErrorCloseFunc(decoder.Close), resp.Body}}, encoding, nil default: - return resp.Body + return resp.Body, encoding, nil } } diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 36b2426d..80ec0620 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -17,6 +17,7 @@ import ( _ "image/jpeg" "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" ) func createTestImage(width, height int) image.Image { @@ -227,6 +228,56 @@ func TestFetchRemoteBrotliEncoded(t *testing.T) { } } +func TestFetchRemoteZstdEncoded(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.png" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + + var pngBuf bytes.Buffer + if err := png.Encode(&pngBuf, createTestImage(18, 12)); err != nil { + t.Fatalf("encode png: %v", err) + } + + var zstdBuf bytes.Buffer + writer, err := zstd.NewWriter(&zstdBuf) + if err != nil { + t.Fatalf("create zstd writer: %v", err) + } + + if _, err := writer.Write(pngBuf.Bytes()); err != nil { + t.Fatalf("write zstd payload: %v", err) + } + + if err := writer.Close(); err != nil { + t.Fatalf("close zstd writer: %v", err) + } + + w.Header().Set("Content-Encoding", "zstd") + w.Header().Set("Content-Type", "image/png") + if _, err := w.Write(zstdBuf.Bytes()); err != nil { + t.Fatalf("write zstd payload: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/cover.png") + if err != nil { + t.Fatalf("fetch zstd image: %v", err) + } + + if format != "png" { + t.Fatalf("expected png format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 18 || bounds.Dy() != 12 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestFetchRemoteDecodeErrorIncludesContentType(t *testing.T) { t.Parallel() @@ -247,6 +298,43 @@ func TestFetchRemoteDecodeErrorIncludesContentType(t *testing.T) { } } +func TestFetchRemoteDecodeErrorIncludesContentEncoding(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "br") + w.Header().Set("Content-Type", "text/html") + + var brBuf bytes.Buffer + writer := brotli.NewWriterLevel(&brBuf, brotli.BestCompression) + if _, err := writer.Write([]byte("")); err != nil { + t.Fatalf("write brotli payload: %v", err) + } + + if err := writer.Close(); err != nil { + t.Fatalf("close brotli writer: %v", err) + } + + if _, err := w.Write(brBuf.Bytes()); err != nil { + t.Fatalf("write brotli payload: %v", err) + } + })) + defer server.Close() + + _, _, err := Fetch(server.URL + "/cover.png") + if err == nil { + t.Fatal("expected error decoding brotli encoded html") + } + + if !strings.Contains(err.Error(), "content-encoding \"br\"") { + t.Fatalf("expected content-encoding in error, got %v", err) + } + + if !strings.Contains(err.Error(), "content-type \"text/html\"") { + t.Fatalf("expected content-type in error, got %v", err) + } +} + func TestResize(t *testing.T) { t.Parallel() From c615e15bb507f63dfccbf49f04e848645ee65c1e Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 16:29:56 +0800 Subject: [PATCH 10/16] Harden remote image decoding --- pkg/images/image.go | 106 ++++++++++++++++++++++++++++++++++++++- pkg/images/image_test.go | 80 ++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 4 deletions(-) diff --git a/pkg/images/image.go b/pkg/images/image.go index 1cc97a4c..fe0e1a74 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -1,7 +1,9 @@ package images import ( + "bytes" "compress/gzip" + "errors" "fmt" stdimage "image" _ "image/gif" @@ -15,6 +17,7 @@ import ( "path/filepath" "strings" "time" + "unicode" "golang.org/x/image/draw" _ "golang.org/x/image/webp" @@ -34,9 +37,12 @@ func Fetch(source string) (stdimage.Image, string, error) { if err != nil { return nil, "", err } - defer reader.Close() + payload, err := readImagePayload(reader) + if err != nil { + return nil, "", err + } - img, format, err := stdimage.Decode(reader) + img, format, err := decodeImagePayload(payload) if err != nil { var details []string @@ -58,6 +64,102 @@ func Fetch(source string) (stdimage.Image, string, error) { return img, format, nil } +const maxRemoteImageBytes = 32 << 20 // 32MiB should cover large blog assets. + +func readImagePayload(reader io.ReadCloser) ([]byte, error) { + defer reader.Close() + + limited := io.LimitReader(reader, maxRemoteImageBytes+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, fmt.Errorf("read image payload: %w", err) + } + + if len(data) == 0 { + return nil, errors.New("empty image payload") + } + + if len(data) > maxRemoteImageBytes { + return nil, fmt.Errorf("image payload exceeds %d bytes", maxRemoteImageBytes) + } + + return data, nil +} + +func decodeImagePayload(data []byte) (stdimage.Image, string, error) { + attempts := [][]byte{data} + + trimmed := trimLeadingNoise(data) + if len(trimmed) > 0 && !bytes.Equal(trimmed, data) { + attempts = append(attempts, trimmed) + } + + if start, ok := findEmbeddedImageStart(trimmed); ok && start > 0 && start < len(trimmed) { + attempts = append(attempts, trimmed[start:]) + } + + var lastErr error + for _, candidate := range attempts { + img, format, err := stdimage.Decode(bytes.NewReader(candidate)) + if err == nil { + return img, format, nil + } + + lastErr = err + } + + if lastErr == nil { + lastErr = errors.New("image: unknown format") + } + + return nil, "", lastErr +} + +func trimLeadingNoise(data []byte) []byte { + trimmed := bytes.TrimLeftFunc(data, func(r rune) bool { + return r == unicode.ReplacementChar || unicode.IsSpace(r) + }) + + if len(trimmed) >= 3 && bytes.Equal(trimmed[:3], []byte{0xEF, 0xBB, 0xBF}) { + trimmed = trimmed[3:] + } + + return trimmed +} + +func findEmbeddedImageStart(data []byte) (int, bool) { + if idx := bytes.Index(data, []byte{0xFF, 0xD8, 0xFF}); idx >= 0 { + return idx, true + } + + if idx := bytes.Index(data, []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1A, '\n'}); idx >= 0 { + return idx, true + } + + if idx := bytes.Index(data, []byte("GIF87a")); idx >= 0 { + return idx, true + } + + if idx := bytes.Index(data, []byte("GIF89a")); idx >= 0 { + return idx, true + } + + for idx := bytes.Index(data, []byte("RIFF")); idx >= 0; { + if len(data)-idx >= 12 && bytes.Equal(data[idx+8:idx+12], []byte("WEBP")) { + return idx, true + } + + next := bytes.Index(data[idx+4:], []byte("RIFF")) + if next < 0 { + break + } + + idx += 4 + next + } + + return 0, false +} + func Resize(src stdimage.Image, width, height int) stdimage.Image { dst := stdimage.NewRGBA(stdimage.Rect(0, 0, width, height)) draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil) diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 80ec0620..108df024 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "image" "image/color" + "image/jpeg" "image/png" "net/http" "net/http/httptest" @@ -14,8 +15,6 @@ import ( "strings" "testing" - _ "image/jpeg" - "github.com/andybalholm/brotli" "github.com/klauspost/compress/zstd" ) @@ -137,6 +136,83 @@ func TestFetchRemoteWebP(t *testing.T) { } } +func TestFetchRemoteJPEGWithLeadingNoise(t *testing.T) { + t.Parallel() + + img := createTestImage(24, 16) + var jpegBuf bytes.Buffer + if err := jpeg.Encode(&jpegBuf, img, &jpeg.Options{Quality: 85}); err != nil { + t.Fatalf("encode jpeg: %v", err) + } + + junkPrefix := []byte{0xEF, 0xBB, 0xBF, '\n', '\r', ' '} + payload := append(junkPrefix, jpegBuf.Bytes()...) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.jpg" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(payload); err != nil { + t.Fatalf("write jpeg payload: %v", err) + } + })) + defer server.Close() + + imgResult, format, err := Fetch(server.URL + "/cover.jpg") + if err != nil { + t.Fatalf("fetch jpeg with leading noise: %v", err) + } + + if format != "jpeg" { + t.Fatalf("expected jpeg format, got %q", format) + } + + bounds := imgResult.Bounds() + if bounds.Dx() != 24 || bounds.Dy() != 16 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + +func TestFetchRemoteJPEGEmbeddedAfterHTML(t *testing.T) { + t.Parallel() + + img := createTestImage(30, 22) + var jpegBuf bytes.Buffer + if err := jpeg.Encode(&jpegBuf, img, &jpeg.Options{Quality: 80}); err != nil { + t.Fatalf("encode jpeg: %v", err) + } + + payload := append([]byte("preview"), jpegBuf.Bytes()...) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/asset" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(payload); err != nil { + t.Fatalf("write jpeg payload: %v", err) + } + })) + defer server.Close() + + imgResult, format, err := Fetch(server.URL + "/asset") + if err != nil { + t.Fatalf("fetch embedded jpeg: %v", err) + } + + if format != "jpeg" { + t.Fatalf("expected jpeg format, got %q", format) + } + + bounds := imgResult.Bounds() + if bounds.Dx() != 30 || bounds.Dy() != 22 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestFetchRemoteSetsAcceptHeader(t *testing.T) { t.Parallel() From 64c7a5f1d61b23a727888fc5fc11664f51cdd1a8 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 16:49:22 +0800 Subject: [PATCH 11/16] Handle compressed remote image payloads --- pkg/images/image.go | 141 ++++++++++++++++++++++++++++++++++----- pkg/images/image_test.go | 48 +++++++++++++ 2 files changed, 173 insertions(+), 16 deletions(-) diff --git a/pkg/images/image.go b/pkg/images/image.go index fe0e1a74..0bdc792d 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -3,6 +3,8 @@ package images import ( "bytes" "compress/gzip" + "compress/zlib" + "crypto/sha256" "errors" "fmt" stdimage "image" @@ -66,6 +68,8 @@ func Fetch(source string) (stdimage.Image, string, error) { const maxRemoteImageBytes = 32 << 20 // 32MiB should cover large blog assets. +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + func readImagePayload(reader io.ReadCloser) ([]byte, error) { defer reader.Close() @@ -87,25 +91,38 @@ func readImagePayload(reader io.ReadCloser) ([]byte, error) { } func decodeImagePayload(data []byte) (stdimage.Image, string, error) { - attempts := [][]byte{data} + queue := [][]byte{data} + seen := make(map[[32]byte]struct{}) - trimmed := trimLeadingNoise(data) - if len(trimmed) > 0 && !bytes.Equal(trimmed, data) { - attempts = append(attempts, trimmed) - } + var lastErr error - if start, ok := findEmbeddedImageStart(trimmed); ok && start > 0 && start < len(trimmed) { - attempts = append(attempts, trimmed[start:]) - } + for len(queue) > 0 { + candidate := queue[0] + queue = queue[1:] + + hash := sha256.Sum256(candidate) + if _, exists := seen[hash]; exists { + continue + } + seen[hash] = struct{}{} - var lastErr error - for _, candidate := range attempts { img, format, err := stdimage.Decode(bytes.NewReader(candidate)) if err == nil { return img, format, nil } lastErr = err + + trimmed := trimLeadingNoise(candidate) + if len(trimmed) > 0 && len(trimmed) != len(candidate) { + queue = append(queue, trimmed) + } + + if start, ok := findEmbeddedImageStart(candidate); ok && start > 0 && start < len(candidate) { + queue = append(queue, candidate[start:]) + } + + queue = append(queue, expandCompressedCandidate(candidate)...) } if lastErr == nil { @@ -116,15 +133,107 @@ func decodeImagePayload(data []byte) (stdimage.Image, string, error) { } func trimLeadingNoise(data []byte) []byte { - trimmed := bytes.TrimLeftFunc(data, func(r rune) bool { - return r == unicode.ReplacementChar || unicode.IsSpace(r) - }) + trimmed := dropUTF8BOM(data) + trimmed = bytes.TrimLeftFunc(trimmed, unicode.IsSpace) + + return dropUTF8BOM(trimmed) +} + +func dropUTF8BOM(data []byte) []byte { + for len(data) >= len(utf8BOM) && bytes.Equal(data[:len(utf8BOM)], utf8BOM) { + data = data[len(utf8BOM):] + } + + return data +} + +func expandCompressedCandidate(data []byte) [][]byte { + var expansions [][]byte + + if decoded, err := tryGzipDecode(data); err == nil { + expansions = append(expansions, decoded) + } + + if decoded, err := tryZlibDecode(data); err == nil { + expansions = append(expansions, decoded) + } + + if decoded, err := tryZstdDecode(data); err == nil { + expansions = append(expansions, decoded) + } + + return expansions +} + +func tryGzipDecode(data []byte) ([]byte, error) { + if len(data) < 2 || data[0] != 0x1F || data[1] != 0x8B { + return nil, errors.New("not gzip") + } + + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer reader.Close() + + return readLimited(reader, maxRemoteImageBytes) +} + +func tryZlibDecode(data []byte) ([]byte, error) { + if len(data) < 2 { + return nil, errors.New("not zlib") + } + + cmf := data[0] + flg := data[1] + + if cmf&0x0F != 8 { // compression method deflate + return nil, errors.New("not zlib deflate") + } + + if (uint16(cmf)<<8|uint16(flg))%31 != 0 { + return nil, errors.New("invalid zlib header") + } + + reader, err := zlib.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer reader.Close() + + return readLimited(reader, maxRemoteImageBytes) +} + +func tryZstdDecode(data []byte) ([]byte, error) { + if len(data) < 4 || data[0] != 0x28 || data[1] != 0xB5 || data[2] != 0x2F || data[3] != 0xFD { + return nil, errors.New("not zstd") + } + + decoder, err := zstd.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer decoder.Close() + + return readLimited(decoder, maxRemoteImageBytes) +} - if len(trimmed) >= 3 && bytes.Equal(trimmed[:3], []byte{0xEF, 0xBB, 0xBF}) { - trimmed = trimmed[3:] +func readLimited(reader io.Reader, limit int) ([]byte, error) { + limited := io.LimitReader(reader, int64(limit)+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, err + } + + if len(data) > limit { + return nil, fmt.Errorf("decompressed payload exceeds %d bytes", limit) } - return trimmed + if len(data) == 0 { + return nil, errors.New("decompressed payload empty") + } + + return data, nil } func findEmbeddedImageStart(data []byte) (int, bool) { diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 108df024..e1e3a2bd 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -2,6 +2,7 @@ package images import ( "bytes" + "compress/gzip" "encoding/base64" "image" "image/color" @@ -213,6 +214,53 @@ func TestFetchRemoteJPEGEmbeddedAfterHTML(t *testing.T) { } } +func TestFetchRemoteJPEGCompressedWithoutEncodingHeader(t *testing.T) { + t.Parallel() + + img := createTestImage(40, 28) + var jpegBuf bytes.Buffer + if err := jpeg.Encode(&jpegBuf, img, &jpeg.Options{Quality: 85}); err != nil { + t.Fatalf("encode jpeg: %v", err) + } + + var compressed bytes.Buffer + gzipWriter := gzip.NewWriter(&compressed) + if _, err := gzipWriter.Write(jpegBuf.Bytes()); err != nil { + t.Fatalf("compress jpeg: %v", err) + } + if err := gzipWriter.Close(); err != nil { + t.Fatalf("close gzip writer: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.jpg" { + t.Errorf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) + return + } + + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(compressed.Bytes()); err != nil { + t.Fatalf("write compressed payload: %v", err) + } + })) + defer server.Close() + + imgResult, format, err := Fetch(server.URL + "/cover.jpg") + if err != nil { + t.Fatalf("fetch compressed jpeg: %v", err) + } + + if format != "jpeg" { + t.Fatalf("expected jpeg format, got %q", format) + } + + bounds := imgResult.Bounds() + if bounds.Dx() != 40 || bounds.Dy() != 28 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestFetchRemoteSetsAcceptHeader(t *testing.T) { t.Parallel() From 63e5f991f12a0840536e63838821aa2cd5824a27 Mon Sep 17 00:00:00 2001 From: Gus Date: Fri, 10 Oct 2025 17:03:35 +0800 Subject: [PATCH 12/16] Improve remote image diagnostics and brotli fallback --- metal/cli/seo/pictures.go | 9 +++ pkg/images/errors.go | 132 ++++++++++++++++++++++++++++++++++++++ pkg/images/image.go | 35 +++++----- pkg/images/image_test.go | 98 ++++++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 15 deletions(-) create mode 100644 pkg/images/errors.go diff --git a/metal/cli/seo/pictures.go b/metal/cli/seo/pictures.go index af67cd5e..0ce6c37f 100644 --- a/metal/cli/seo/pictures.go +++ b/metal/cli/seo/pictures.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/oullin/handler/payload" + "github.com/oullin/pkg/cli" pkgimages "github.com/oullin/pkg/images" "github.com/oullin/pkg/portal" ) @@ -36,6 +37,14 @@ func (g *Generator) preparePostImage(post payload.PostResponse) (preparedImage, img, format, err := pkgimages.Fetch(source) if err != nil { + var decodeErr *pkgimages.DecodeError + if errors.As(err, &decodeErr) { + cli.Errorln("Failed to decode remote image. Diagnostics:") + for _, line := range decodeErr.Diagnostics() { + cli.Grayln(" " + line) + } + } + return preparedImage{}, err } diff --git a/pkg/images/errors.go b/pkg/images/errors.go new file mode 100644 index 00000000..06f0cabb --- /dev/null +++ b/pkg/images/errors.go @@ -0,0 +1,132 @@ +package images + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "strings" +) + +type DecodeError struct { + Err error + ContentType string + ContentEncoding string + SniffedType string + Size int + Hash string + PrefixHex string +} + +func newDecodeError(err error, payload []byte, contentType, encoding string) *DecodeError { + sample := payload + if len(sample) > 512 { + sample = sample[:512] + } + + sniffed := http.DetectContentType(sample) + + details := &DecodeError{ + Err: err, + ContentType: strings.TrimSpace(contentType), + ContentEncoding: strings.TrimSpace(encoding), + SniffedType: sniffed, + Size: len(payload), + } + + if len(payload) > 0 { + sum := sha256.Sum256(payload) + details.Hash = hex.EncodeToString(sum[:]) + + prefixLen := len(payload) + if prefixLen > 16 { + prefixLen = 16 + } + + details.PrefixHex = hex.EncodeToString(payload[:prefixLen]) + } + + return details +} + +func (e *DecodeError) Error() string { + if e == nil { + return "decode image: " + } + + var parts []string + + if ct := strings.TrimSpace(e.ContentType); ct != "" { + parts = append(parts, fmt.Sprintf("content-type %q", ct)) + } + + if enc := strings.TrimSpace(e.ContentEncoding); enc != "" { + parts = append(parts, fmt.Sprintf("content-encoding %q", enc)) + } + + if sniff := strings.TrimSpace(e.SniffedType); sniff != "" { + parts = append(parts, fmt.Sprintf("sniffed %q", sniff)) + } + + if e.Size > 0 { + parts = append(parts, fmt.Sprintf("size %d bytes", e.Size)) + } + + if e.Hash != "" { + parts = append(parts, fmt.Sprintf("sha256 %s", e.Hash)) + } + + if e.PrefixHex != "" { + parts = append(parts, fmt.Sprintf("prefix %s", e.PrefixHex)) + } + + if len(parts) == 0 { + return fmt.Sprintf("decode image: %v", e.Err) + } + + return fmt.Sprintf("decode image (%s): %v", strings.Join(parts, ", "), e.Err) +} + +func (e *DecodeError) Unwrap() error { + if e == nil { + return nil + } + + return e.Err +} + +func (e *DecodeError) Diagnostics() []string { + if e == nil { + return nil + } + + var lines []string + + if ct := strings.TrimSpace(e.ContentType); ct != "" { + lines = append(lines, fmt.Sprintf("content-type: %s", ct)) + } else { + lines = append(lines, "content-type: (missing)") + } + + if enc := strings.TrimSpace(e.ContentEncoding); enc != "" { + lines = append(lines, fmt.Sprintf("content-encoding: %s", enc)) + } else { + lines = append(lines, "content-encoding: (missing)") + } + + if sniff := strings.TrimSpace(e.SniffedType); sniff != "" { + lines = append(lines, fmt.Sprintf("sniffed-type: %s", sniff)) + } + + lines = append(lines, fmt.Sprintf("payload-bytes: %d", e.Size)) + + if e.Hash != "" { + lines = append(lines, fmt.Sprintf("payload-sha256: %s", e.Hash)) + } + + if e.PrefixHex != "" { + lines = append(lines, fmt.Sprintf("payload-prefix-hex: %s", e.PrefixHex)) + } + + return lines +} diff --git a/pkg/images/image.go b/pkg/images/image.go index 0bdc792d..97726cae 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -46,21 +46,7 @@ func Fetch(source string) (stdimage.Image, string, error) { img, format, err := decodeImagePayload(payload) if err != nil { - var details []string - - if ct := strings.TrimSpace(contentType); ct != "" { - details = append(details, fmt.Sprintf("content-type %q", ct)) - } - - if enc := strings.TrimSpace(encoding); enc != "" { - details = append(details, fmt.Sprintf("content-encoding %q", enc)) - } - - if len(details) > 0 { - return nil, "", fmt.Errorf("decode image (%s): %w", strings.Join(details, ", "), err) - } - - return nil, "", fmt.Errorf("decode image: %w", err) + return nil, "", newDecodeError(err, payload, contentType, encoding) } return img, format, nil @@ -150,6 +136,10 @@ func dropUTF8BOM(data []byte) []byte { func expandCompressedCandidate(data []byte) [][]byte { var expansions [][]byte + if decoded, err := tryBrotliDecode(data); err == nil { + expansions = append(expansions, decoded) + } + if decoded, err := tryGzipDecode(data); err == nil { expansions = append(expansions, decoded) } @@ -165,6 +155,21 @@ func expandCompressedCandidate(data []byte) [][]byte { return expansions } +func tryBrotliDecode(data []byte) ([]byte, error) { + reader := brotli.NewReader(bytes.NewReader(data)) + + decoded, err := readLimited(reader, maxRemoteImageBytes) + if err != nil { + return nil, err + } + + if len(decoded) == 0 { + return nil, errors.New("brotli decoded empty") + } + + return decoded, nil +} + func tryGzipDecode(data []byte) ([]byte, error) { if len(data) < 2 || data[0] != 0x1F || data[1] != 0x8B { return nil, errors.New("not gzip") diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index e1e3a2bd..7cc7f472 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "encoding/base64" + "errors" "image" "image/color" "image/jpeg" @@ -352,6 +353,51 @@ func TestFetchRemoteBrotliEncoded(t *testing.T) { } } +func TestFetchRemoteBrotliWithoutEncodingHeader(t *testing.T) { + t.Parallel() + + var pngBuf bytes.Buffer + if err := png.Encode(&pngBuf, createTestImage(28, 16)); err != nil { + t.Fatalf("encode png: %v", err) + } + + var brBuf bytes.Buffer + writer := brotli.NewWriterLevel(&brBuf, brotli.BestCompression) + if _, err := writer.Write(pngBuf.Bytes()); err != nil { + t.Fatalf("write brotli: %v", err) + } + + if err := writer.Close(); err != nil { + t.Fatalf("close brotli writer: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.png" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + + w.Header().Set("Content-Type", "image/png") + if _, err := w.Write(brBuf.Bytes()); err != nil { + t.Fatalf("write brotli payload: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/cover.png") + if err != nil { + t.Fatalf("fetch brotli payload without header: %v", err) + } + + if format != "png" { + t.Fatalf("expected png format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 28 || bounds.Dy() != 16 { + t.Fatalf("unexpected dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } +} + func TestFetchRemoteZstdEncoded(t *testing.T) { t.Parallel() @@ -459,6 +505,58 @@ func TestFetchRemoteDecodeErrorIncludesContentEncoding(t *testing.T) { } } +func TestFetchRemoteDecodeErrorProvidesDiagnostics(t *testing.T) { + t.Parallel() + + payload := []byte("forbidden") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(payload); err != nil { + t.Fatalf("write payload: %v", err) + } + })) + defer server.Close() + + _, _, err := Fetch(server.URL + "/cover.jpg") + if err == nil { + t.Fatal("expected decode error") + } + + var decodeErr *DecodeError + if !errors.As(err, &decodeErr) { + t.Fatalf("expected DecodeError, got %T", err) + } + + if decodeErr.SniffedType == "" || !strings.Contains(decodeErr.SniffedType, "text/html") { + t.Fatalf("expected sniffed type to mention text/html, got %q", decodeErr.SniffedType) + } + + if decodeErr.Size != len(payload) { + t.Fatalf("unexpected payload size: got %d want %d", decodeErr.Size, len(payload)) + } + + if decodeErr.Hash == "" { + t.Fatal("expected hash to be populated") + } + + if !strings.HasPrefix(decodeErr.PrefixHex, "3c68") { // " Date: Mon, 13 Oct 2025 10:14:18 +0800 Subject: [PATCH 13/16] Decode AVIF SEO cover images --- pkg/images/avif.go | 133 +++++++++++++++++++++++++++++++++++++++ pkg/images/avif_stub.go | 18 ++++++ pkg/images/image.go | 40 ++++++++++++ pkg/images/image_test.go | 45 +++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 pkg/images/avif.go create mode 100644 pkg/images/avif_stub.go diff --git a/pkg/images/avif.go b/pkg/images/avif.go new file mode 100644 index 00000000..908efe94 --- /dev/null +++ b/pkg/images/avif.go @@ -0,0 +1,133 @@ +//go:build cgo + +package images + +/* +#cgo pkg-config: libheif +#include +#include +*/ +import "C" + +import ( + "errors" + "fmt" + stdimage "image" + "image/color" + "unsafe" +) + +const avifSupported = true + +type avifImage struct { + img stdimage.Image + cfg stdimage.Config +} + +func decodeAVIF(data []byte) (stdimage.Image, error) { + decoded, err := decodeAVIFInternal(data, false) + if err != nil { + return nil, err + } + + return decoded.img, nil +} + +func decodeAVIFConfig(data []byte) (stdimage.Config, error) { + decoded, err := decodeAVIFInternal(data, true) + if err != nil { + return stdimage.Config{}, err + } + + return decoded.cfg, nil +} + +func decodeAVIFInternal(data []byte, configOnly bool) (avifImage, error) { + if len(data) == 0 { + return avifImage{}, errors.New("avif payload empty") + } + + ctx := C.heif_context_alloc() + if ctx == nil { + return avifImage{}, errors.New("heif context alloc failed") + } + defer C.heif_context_free(ctx) + + ptr := C.CBytes(data) + defer C.free(ptr) + + size := C.size_t(len(data)) + + err := C.heif_context_read_from_memory(ctx, ptr, size, nil) + if herr := translateHeifError("read avif payload", err); herr != nil { + return avifImage{}, herr + } + + var handle *C.struct_heif_image_handle + err = C.heif_context_get_primary_image_handle(ctx, &handle) + if herr := translateHeifError("get avif handle", err); herr != nil { + return avifImage{}, herr + } + defer C.heif_image_handle_release(handle) + + width := int(C.heif_image_handle_get_width(handle)) + height := int(C.heif_image_handle_get_height(handle)) + if width <= 0 || height <= 0 { + return avifImage{}, fmt.Errorf("invalid avif dimensions %dx%d", width, height) + } + + cfg := stdimage.Config{ColorModel: color.NRGBAModel, Width: width, Height: height} + + if configOnly { + return avifImage{cfg: cfg}, nil + } + + var img *C.struct_heif_image + err = C.heif_decode_image(handle, &img, C.heif_colorspace_RGB, C.heif_chroma_interleaved_RGBA, nil) + if herr := translateHeifError("decode avif", err); herr != nil { + return avifImage{}, herr + } + defer C.heif_image_release(img) + + var stride C.int + plane := C.heif_image_get_plane_readonly(img, C.heif_channel_interleaved, &stride) + if plane == nil { + return avifImage{}, errors.New("avif plane unavailable") + } + + rowStride := int(stride) + if rowStride < width*4 { + return avifImage{}, fmt.Errorf("avif stride smaller than width: %d < %d", rowStride, width*4) + } + + total := rowStride * height + if total <= 0 { + return avifImage{}, fmt.Errorf("invalid avif buffer size: %d", total) + } + + buf := C.GoBytes(unsafe.Pointer(plane), C.int(total)) + + out := stdimage.NewNRGBA(stdimage.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + srcStart := y * rowStride + srcEnd := srcStart + width*4 + dstStart := y * out.Stride + dstEnd := dstStart + width*4 + copy(out.Pix[dstStart:dstEnd], buf[srcStart:srcEnd]) + } + + return avifImage{img: out, cfg: cfg}, nil +} + +func translateHeifError(context string, err C.struct_heif_error) error { + if err.code == C.heif_error_Ok { + return nil + } + + message := "heif" + if err.message != nil { + message = C.GoString(err.message) + } + + return fmt.Errorf("%s: %s (code=%d subcode=%d)", context, message, int(err.code), int(err.subcode)) +} diff --git a/pkg/images/avif_stub.go b/pkg/images/avif_stub.go new file mode 100644 index 00000000..529d4cb7 --- /dev/null +++ b/pkg/images/avif_stub.go @@ -0,0 +1,18 @@ +//go:build !cgo + +package images + +import ( + "errors" + stdimage "image" +) + +const avifSupported = false + +func decodeAVIF(data []byte) (stdimage.Image, error) { + return nil, errors.New("avif decoding requires cgo support") +} + +func decodeAVIFConfig(data []byte) (stdimage.Config, error) { + return stdimage.Config{}, errors.New("avif decoding requires cgo support") +} diff --git a/pkg/images/image.go b/pkg/images/image.go index 97726cae..8b92365e 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -5,6 +5,7 @@ import ( "compress/gzip" "compress/zlib" "crypto/sha256" + "encoding/binary" "errors" "fmt" stdimage "image" @@ -99,6 +100,16 @@ func decodeImagePayload(data []byte) (stdimage.Image, string, error) { lastErr = err + if isAVIF(candidate) { + if !avifSupported { + lastErr = errors.New("avif decoding requires cgo support") + } else if avifImg, avifErr := decodeAVIF(candidate); avifErr == nil { + return avifImg, "avif", nil + } else { + lastErr = avifErr + } + } + trimmed := trimLeadingNoise(candidate) if len(trimmed) > 0 && len(trimmed) != len(candidate) { queue = append(queue, trimmed) @@ -155,6 +166,35 @@ func expandCompressedCandidate(data []byte) [][]byte { return expansions } +func isAVIF(data []byte) bool { + if len(data) < 16 { + return false + } + + boxSize := binary.BigEndian.Uint32(data[:4]) + if boxSize == 0 || int(boxSize) > len(data) { + boxSize = uint32(len(data)) + } + + if string(data[4:8]) != "ftyp" { + return false + } + + brands := [][]byte{data[8:12]} + for offset := 16; offset+4 <= int(boxSize); offset += 4 { + brands = append(brands, data[offset:offset+4]) + } + + for _, brand := range brands { + switch string(brand) { + case "avif", "avis", "avio": + return true + } + } + + return false +} + func tryBrotliDecode(data []byte) ([]byte, error) { reader := brotli.NewReader(bytes.NewReader(data)) diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 7cc7f472..58a9b9d1 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -21,6 +21,8 @@ import ( "github.com/klauspost/compress/zstd" ) +const avifFixtureBase64 = "AAAAHGZ0eXBhdmlmAAAAAGF2aWZtaWYxbWlhZgAAAOptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABwaWN0AAAAAAAAAAAAAAAAAAAAAA5waXRtAAAAAAABAAAAImlsb2MAAAAAREAAAQABAAAAAAEOAAEAAAAAAAAAHwAAACNpaW5mAAAAAAABAAAAFWluZmUCAAAAAAEAAGF2MDEAAAAAamlwcnAAAABLaXBjbwAAABNjb2xybmNseAABAA0ABoAAAAAMYXYxQ4EADAAAAAAUaXNwZQAAAAAAAAACAAAAAgAAABBwaXhpAAAAAAMICAgAAAAXaXBtYQAAAAAAAAABAAEEAYIDBAAAACdtZGF0EgAKCBgANggIaDQgMhEYAAooooQAALATVl9ApOsM/A==" + func createTestImage(width, height int) image.Image { img := image.NewRGBA(image.Rect(0, 0, width, height)) for y := 0; y < height; y++ { @@ -448,6 +450,49 @@ func TestFetchRemoteZstdEncoded(t *testing.T) { } } +func TestFetchRemoteAVIF(t *testing.T) { + t.Parallel() + + data, err := base64.StdEncoding.DecodeString(avifFixtureBase64) + if err != nil { + t.Fatalf("decode avif fixture: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/cover.jpg" { + t.Errorf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) + return + } + + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(data); err != nil { + t.Errorf("write avif payload: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/cover.jpg") + if err != nil { + t.Fatalf("fetch remote avif: %v", err) + } + + if format != "avif" { + t.Fatalf("expected avif format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() != 2 || bounds.Dy() != 2 { + t.Fatalf("unexpected avif dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } + + if rgba, ok := img.(*image.NRGBA); ok { + if rgba.Pix[0] < 200 || rgba.Pix[1] > 10 || rgba.Pix[2] > 10 || rgba.Pix[3] < 200 { + t.Fatalf("unexpected avif pixel: %v", rgba.Pix[:4]) + } + } +} + func TestFetchRemoteDecodeErrorIncludesContentType(t *testing.T) { t.Parallel() From 2cdcc5a951877c0e2d1357e0f58e4913a2be3636 Mon Sep 17 00:00:00 2001 From: Gus Date: Mon, 13 Oct 2025 11:44:15 +0800 Subject: [PATCH 14/16] Handle GitHub AVIF attachments without CGO --- pkg/images/avif.go | 133 -------------------------------- pkg/images/avif_stub.go | 18 ----- pkg/images/image.go | 162 +++++++++++++++++++++++++++++++++++---- pkg/images/image_test.go | 55 ++++++++----- 4 files changed, 181 insertions(+), 187 deletions(-) delete mode 100644 pkg/images/avif.go delete mode 100644 pkg/images/avif_stub.go diff --git a/pkg/images/avif.go b/pkg/images/avif.go deleted file mode 100644 index 908efe94..00000000 --- a/pkg/images/avif.go +++ /dev/null @@ -1,133 +0,0 @@ -//go:build cgo - -package images - -/* -#cgo pkg-config: libheif -#include -#include -*/ -import "C" - -import ( - "errors" - "fmt" - stdimage "image" - "image/color" - "unsafe" -) - -const avifSupported = true - -type avifImage struct { - img stdimage.Image - cfg stdimage.Config -} - -func decodeAVIF(data []byte) (stdimage.Image, error) { - decoded, err := decodeAVIFInternal(data, false) - if err != nil { - return nil, err - } - - return decoded.img, nil -} - -func decodeAVIFConfig(data []byte) (stdimage.Config, error) { - decoded, err := decodeAVIFInternal(data, true) - if err != nil { - return stdimage.Config{}, err - } - - return decoded.cfg, nil -} - -func decodeAVIFInternal(data []byte, configOnly bool) (avifImage, error) { - if len(data) == 0 { - return avifImage{}, errors.New("avif payload empty") - } - - ctx := C.heif_context_alloc() - if ctx == nil { - return avifImage{}, errors.New("heif context alloc failed") - } - defer C.heif_context_free(ctx) - - ptr := C.CBytes(data) - defer C.free(ptr) - - size := C.size_t(len(data)) - - err := C.heif_context_read_from_memory(ctx, ptr, size, nil) - if herr := translateHeifError("read avif payload", err); herr != nil { - return avifImage{}, herr - } - - var handle *C.struct_heif_image_handle - err = C.heif_context_get_primary_image_handle(ctx, &handle) - if herr := translateHeifError("get avif handle", err); herr != nil { - return avifImage{}, herr - } - defer C.heif_image_handle_release(handle) - - width := int(C.heif_image_handle_get_width(handle)) - height := int(C.heif_image_handle_get_height(handle)) - if width <= 0 || height <= 0 { - return avifImage{}, fmt.Errorf("invalid avif dimensions %dx%d", width, height) - } - - cfg := stdimage.Config{ColorModel: color.NRGBAModel, Width: width, Height: height} - - if configOnly { - return avifImage{cfg: cfg}, nil - } - - var img *C.struct_heif_image - err = C.heif_decode_image(handle, &img, C.heif_colorspace_RGB, C.heif_chroma_interleaved_RGBA, nil) - if herr := translateHeifError("decode avif", err); herr != nil { - return avifImage{}, herr - } - defer C.heif_image_release(img) - - var stride C.int - plane := C.heif_image_get_plane_readonly(img, C.heif_channel_interleaved, &stride) - if plane == nil { - return avifImage{}, errors.New("avif plane unavailable") - } - - rowStride := int(stride) - if rowStride < width*4 { - return avifImage{}, fmt.Errorf("avif stride smaller than width: %d < %d", rowStride, width*4) - } - - total := rowStride * height - if total <= 0 { - return avifImage{}, fmt.Errorf("invalid avif buffer size: %d", total) - } - - buf := C.GoBytes(unsafe.Pointer(plane), C.int(total)) - - out := stdimage.NewNRGBA(stdimage.Rect(0, 0, width, height)) - for y := 0; y < height; y++ { - srcStart := y * rowStride - srcEnd := srcStart + width*4 - dstStart := y * out.Stride - dstEnd := dstStart + width*4 - copy(out.Pix[dstStart:dstEnd], buf[srcStart:srcEnd]) - } - - return avifImage{img: out, cfg: cfg}, nil -} - -func translateHeifError(context string, err C.struct_heif_error) error { - if err.code == C.heif_error_Ok { - return nil - } - - message := "heif" - if err.message != nil { - message = C.GoString(err.message) - } - - return fmt.Errorf("%s: %s (code=%d subcode=%d)", context, message, int(err.code), int(err.subcode)) -} diff --git a/pkg/images/avif_stub.go b/pkg/images/avif_stub.go deleted file mode 100644 index 529d4cb7..00000000 --- a/pkg/images/avif_stub.go +++ /dev/null @@ -1,18 +0,0 @@ -//go:build !cgo - -package images - -import ( - "errors" - stdimage "image" -) - -const avifSupported = false - -func decodeAVIF(data []byte) (stdimage.Image, error) { - return nil, errors.New("avif decoding requires cgo support") -} - -func decodeAVIFConfig(data []byte) (stdimage.Config, error) { - return stdimage.Config{}, errors.New("avif decoding requires cgo support") -} diff --git a/pkg/images/image.go b/pkg/images/image.go index 8b92365e..06067cfb 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -36,21 +36,65 @@ func Fetch(source string) (stdimage.Image, string, error) { return nil, "", fmt.Errorf("parse url: %w", err) } - reader, contentType, encoding, err := openSource(parsed) - if err != nil { - return nil, "", err + queue := []*url.URL{parsed} + seen := make(map[string]struct{}) + + var ( + lastErr error + lastDecodeErr error + lastPayload []byte + lastContentType string + lastEncoding string + ) + + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + + if current == nil { + continue + } + + key := current.String() + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + + reader, contentType, encoding, err := openSource(current) + if err != nil { + lastErr = err + continue + } + + payload, err := readImagePayload(reader) + if err != nil { + lastErr = err + continue + } + + img, format, err := decodeImagePayload(payload) + if err == nil { + return img, format, nil + } + + lastDecodeErr = err + lastPayload = payload + lastContentType = contentType + lastEncoding = encoding + + queue = append(queue, githubAttachmentFallbacks(current, payload)...) } - payload, err := readImagePayload(reader) - if err != nil { - return nil, "", err + + if lastDecodeErr != nil { + return nil, "", newDecodeError(lastDecodeErr, lastPayload, lastContentType, lastEncoding) } - img, format, err := decodeImagePayload(payload) - if err != nil { - return nil, "", newDecodeError(err, payload, contentType, encoding) + if lastErr != nil { + return nil, "", lastErr } - return img, format, nil + return nil, "", errors.New("failed to fetch image") } const maxRemoteImageBytes = 32 << 20 // 32MiB should cover large blog assets. @@ -101,13 +145,7 @@ func decodeImagePayload(data []byte) (stdimage.Image, string, error) { lastErr = err if isAVIF(candidate) { - if !avifSupported { - lastErr = errors.New("avif decoding requires cgo support") - } else if avifImg, avifErr := decodeAVIF(candidate); avifErr == nil { - return avifImg, "avif", nil - } else { - lastErr = avifErr - } + lastErr = fmt.Errorf("decode avif payload: %w", err) } trimmed := trimLeadingNoise(candidate) @@ -195,6 +233,96 @@ func isAVIF(data []byte) bool { return false } +func githubAttachmentFallbacks(u *url.URL, payload []byte) []*url.URL { + if u == nil || len(payload) == 0 { + return nil + } + + if !isAVIF(payload) { + return nil + } + + id, ok := githubAttachmentID(u) + if !ok { + return nil + } + + basePaths := []*url.URL{ + {Scheme: u.Scheme, Host: u.Host, Path: path.Join("/user-attachments/assets", id)}, + {Scheme: "https", Host: "github.com", Path: path.Join("/user-attachments/assets", id)}, + } + + variants := make([]*url.URL, 0, len(basePaths)*5) + + for _, base := range basePaths { + if base.Host == "" { + continue + } + + for _, option := range []struct { + format string + name string + }{ + {format: "png", name: "large"}, + {format: "png", name: "medium"}, + {format: "jpg", name: "large"}, + {format: "jpg", name: "medium"}, + } { + clone := *base + query := clone.Query() + query.Set("format", option.format) + query.Set("name", option.name) + clone.RawQuery = query.Encode() + variants = append(variants, &clone) + } + + clone := *base + query := clone.Query() + query.Set("raw", "1") + clone.RawQuery = query.Encode() + variants = append(variants, &clone) + } + + return variants +} + +func githubAttachmentID(u *url.URL) (string, bool) { + if u == nil { + return "", false + } + + trimmedPath := strings.Trim(u.Path, "/") + parts := strings.Split(trimmedPath, "/") + if len(parts) >= 3 && parts[0] == "user-attachments" && parts[1] == "assets" { + id := parts[2] + if id != "" { + return id, true + } + } + + if strings.Contains(strings.ToLower(u.Host), "github-production-user-asset") { + base := path.Base(u.Path) + if base == "" { + return "", false + } + + if dot := strings.LastIndex(base, "."); dot >= 0 { + base = base[:dot] + } + + if dash := strings.Index(base, "-"); dash >= 0 && dash+1 < len(base) { + base = base[dash+1:] + } + + base = strings.TrimSpace(base) + if base != "" { + return base, true + } + } + + return "", false +} + func tryBrotliDecode(data []byte) ([]byte, error) { reader := brotli.NewReader(bytes.NewReader(data)) diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 58a9b9d1..085ff6eb 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -450,46 +450,63 @@ func TestFetchRemoteZstdEncoded(t *testing.T) { } } -func TestFetchRemoteAVIF(t *testing.T) { +func TestFetchRemoteAVIFGitHubFallback(t *testing.T) { t.Parallel() - data, err := base64.StdEncoding.DecodeString(avifFixtureBase64) + avifData, err := base64.StdEncoding.DecodeString(avifFixtureBase64) if err != nil { t.Fatalf("decode avif fixture: %v", err) } + pngImage := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + pngImage.Set(0, 0, color.NRGBA{R: 200, G: 10, B: 10, A: 255}) + pngImage.Set(1, 1, color.NRGBA{R: 20, G: 200, B: 20, A: 255}) + + var pngBuf bytes.Buffer + if err := png.Encode(&pngBuf, pngImage); err != nil { + t.Fatalf("encode fallback png: %v", err) + } + + const attachmentID = "e5abb532-59bf-49bb-a9d2-0c31872718d7" + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/cover.jpg" { - t.Errorf("unexpected path: %s", r.URL.Path) - http.NotFound(w, r) - return - } + requests = append(requests, r.URL.String()) - w.Header().Set("Content-Type", "image/jpeg") - if _, err := w.Write(data); err != nil { - t.Errorf("write avif payload: %v", err) + switch { + case r.URL.Path == "/user-attachments/assets/"+attachmentID && r.URL.RawQuery == "": + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(avifData); err != nil { + t.Errorf("write avif payload: %v", err) + } + case r.URL.Path == "/user-attachments/assets/"+attachmentID && r.URL.Query().Get("format") == "png": + w.Header().Set("Content-Type", "image/png") + if _, err := w.Write(pngBuf.Bytes()); err != nil { + t.Errorf("write png payload: %v", err) + } + default: + t.Errorf("unexpected request: %s", r.URL) + http.NotFound(w, r) } })) defer server.Close() - img, format, err := Fetch(server.URL + "/cover.jpg") + img, format, err := Fetch(server.URL + "/user-attachments/assets/" + attachmentID) if err != nil { - t.Fatalf("fetch remote avif: %v", err) + t.Fatalf("fetch remote avif fallback: %v", err) } - if format != "avif" { - t.Fatalf("expected avif format, got %q", format) + if format != "png" { + t.Fatalf("expected png format, got %q", format) } bounds := img.Bounds() if bounds.Dx() != 2 || bounds.Dy() != 2 { - t.Fatalf("unexpected avif dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + t.Fatalf("unexpected fallback dimensions: %dx%d", bounds.Dx(), bounds.Dy()) } - if rgba, ok := img.(*image.NRGBA); ok { - if rgba.Pix[0] < 200 || rgba.Pix[1] > 10 || rgba.Pix[2] > 10 || rgba.Pix[3] < 200 { - t.Fatalf("unexpected avif pixel: %v", rgba.Pix[:4]) - } + if len(requests) < 2 { + t.Fatalf("expected fallback requests, got %v", requests) } } From dd5ecfd1ea28345ac5324a10ae5cd81b501697ef Mon Sep 17 00:00:00 2001 From: Gus Date: Mon, 13 Oct 2025 12:06:08 +0800 Subject: [PATCH 15/16] Decode GitHub AVIF attachments --- go.mod | 2 ++ go.sum | 4 ++++ pkg/images/image.go | 27 +++++++++++++++++++-- pkg/images/image_test.go | 52 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e041980b..e4b54bf9 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/ebitengine/purego v0.8.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gen2brain/avif v0.4.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -74,6 +75,7 @@ require ( github.com/shirou/gopsutil/v4 v4.25.7 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/stretchr/testify v1.10.0 // indirect + github.com/tetratelabs/wazero v1.9.0 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/go.sum b/go.sum index b6b9f2a8..42c1177e 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gen2brain/avif v0.4.4 h1:Ga/ss7qcWWQm2bxFpnjYjhJsNfZrWs5RsyklgFjKRSE= +github.com/gen2brain/avif v0.4.4/go.mod h1:/XCaJcjZraQwKVhpu9aEd9aLOssYOawLvhMBtmHVGqk= github.com/getsentry/sentry-go v0.35.1 h1:iopow6UVLE2aXu46xKVIs8Z9D/YZkJrHkgozrxa+tOQ= github.com/getsentry/sentry-go v0.35.1/go.mod h1:C55omcY9ChRQIUcVcGcs+Zdy4ZpQGvNJ7JYHIoSWOtE= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -152,6 +154,8 @@ github.com/testcontainers/testcontainers-go v0.38.0 h1:d7uEapLcv2P8AvH8ahLqDMMxd github.com/testcontainers/testcontainers-go v0.38.0/go.mod h1:C52c9MoHpWO+C4aqmgSU+hxlR5jlEayWtgYrb8Pzz1w= github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0 h1:KFdx9A0yF94K70T6ibSuvgkQQeX1xKlZVF3hEagXEtY= github.com/testcontainers/testcontainers-go/modules/postgres v0.38.0/go.mod h1:T/QRECND6N6tAKMxF1Za+G2tpwnGEHcODzHRsgIpw9M= +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= diff --git a/pkg/images/image.go b/pkg/images/image.go index 06067cfb..5a5fff1d 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -19,6 +19,7 @@ import ( "path" "path/filepath" "strings" + "sync" "time" "unicode" @@ -27,6 +28,7 @@ import ( "github.com/andybalholm/brotli" "github.com/chai2010/webp" + "github.com/gen2brain/avif" "github.com/klauspost/compress/zstd" ) @@ -99,7 +101,10 @@ func Fetch(source string) (stdimage.Image, string, error) { const maxRemoteImageBytes = 32 << 20 // 32MiB should cover large blog assets. -var utf8BOM = []byte{0xEF, 0xBB, 0xBF} +var ( + utf8BOM = []byte{0xEF, 0xBB, 0xBF} + avifInitOnce sync.Once +) func readImagePayload(reader io.ReadCloser) ([]byte, error) { defer reader.Close() @@ -145,7 +150,12 @@ func decodeImagePayload(data []byte) (stdimage.Image, string, error) { lastErr = err if isAVIF(candidate) { - lastErr = fmt.Errorf("decode avif payload: %w", err) + avifImg, avifErr := decodeAVIF(candidate) + if avifErr == nil { + return avifImg, "avif", nil + } + + lastErr = fmt.Errorf("decode avif payload: %w", avifErr) } trimmed := trimLeadingNoise(candidate) @@ -233,6 +243,19 @@ func isAVIF(data []byte) bool { return false } +func decodeAVIF(data []byte) (stdimage.Image, error) { + avifInitOnce.Do(func() { + avif.InitDecoder() + }) + + img, err := avif.Decode(bytes.NewReader(data)) + if err != nil { + return nil, err + } + + return img, nil +} + func githubAttachmentFallbacks(u *url.URL, payload []byte) []*url.URL { if u == nil || len(payload) == 0 { return nil diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 085ff6eb..15e63cee 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -450,6 +450,51 @@ func TestFetchRemoteZstdEncoded(t *testing.T) { } } +func TestFetchRemoteAVIFDecode(t *testing.T) { + t.Parallel() + + avifData, err := base64.StdEncoding.DecodeString(avifFixtureBase64) + if err != nil { + t.Fatalf("decode avif fixture: %v", err) + } + + var requests []string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.URL.String()) + + if r.URL.Path != "/user-attachments/assets/e5abb532-59bf-49bb-a9d2-0c31872718d7" { + t.Errorf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) + return + } + + w.Header().Set("Content-Type", "image/jpeg") + if _, err := w.Write(avifData); err != nil { + t.Errorf("write avif payload: %v", err) + } + })) + defer server.Close() + + img, format, err := Fetch(server.URL + "/user-attachments/assets/e5abb532-59bf-49bb-a9d2-0c31872718d7") + if err != nil { + t.Fatalf("fetch avif image: %v", err) + } + + if format != "avif" { + t.Fatalf("expected avif format, got %q", format) + } + + bounds := img.Bounds() + if bounds.Dx() <= 0 || bounds.Dy() <= 0 { + t.Fatalf("unexpected avif dimensions: %dx%d", bounds.Dx(), bounds.Dy()) + } + + if len(requests) != 1 { + t.Fatalf("expected single request, got %v", requests) + } +} + func TestFetchRemoteAVIFGitHubFallback(t *testing.T) { t.Parallel() @@ -458,6 +503,11 @@ func TestFetchRemoteAVIFGitHubFallback(t *testing.T) { t.Fatalf("decode avif fixture: %v", err) } + brokenAvif := append([]byte(nil), avifData...) + if len(brokenAvif) > 40 { + brokenAvif = brokenAvif[:40] + } + pngImage := image.NewNRGBA(image.Rect(0, 0, 2, 2)) pngImage.Set(0, 0, color.NRGBA{R: 200, G: 10, B: 10, A: 255}) pngImage.Set(1, 1, color.NRGBA{R: 20, G: 200, B: 20, A: 255}) @@ -476,7 +526,7 @@ func TestFetchRemoteAVIFGitHubFallback(t *testing.T) { switch { case r.URL.Path == "/user-attachments/assets/"+attachmentID && r.URL.RawQuery == "": w.Header().Set("Content-Type", "image/jpeg") - if _, err := w.Write(avifData); err != nil { + if _, err := w.Write(brokenAvif); err != nil { t.Errorf("write avif payload: %v", err) } case r.URL.Path == "/user-attachments/assets/"+attachmentID && r.URL.Query().Get("format") == "png": From 7048ae65e2aec363103b5cad71e9db722ab355c7 Mon Sep 17 00:00:00 2001 From: Gus Date: Mon, 13 Oct 2025 12:39:22 +0800 Subject: [PATCH 16/16] Prefer PNG/JPEG Accept headers for GitHub fallbacks --- pkg/images/globals.go | 2 ++ pkg/images/image.go | 52 ++++++++++++++++++++++++++++++---------- pkg/images/image_test.go | 29 ++++++++++++++++++++-- 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/pkg/images/globals.go b/pkg/images/globals.go index 50208663..25739df0 100644 --- a/pkg/images/globals.go +++ b/pkg/images/globals.go @@ -6,6 +6,8 @@ const ( DefaultJPEGQuality = 85 supportedImageAcceptHeader = "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1" defaultRemoteImageUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15" + fallbackPNGAcceptHeader = "image/png,image/*;q=0.8,*/*;q=0.1" + fallbackJPEGAcceptHeader = "image/jpeg,image/*;q=0.8,*/*;q=0.1" ) type composedReadCloser struct { diff --git a/pkg/images/image.go b/pkg/images/image.go index 5a5fff1d..d45dc7bf 100644 --- a/pkg/images/image.go +++ b/pkg/images/image.go @@ -32,13 +32,26 @@ import ( "github.com/klauspost/compress/zstd" ) +type fetchRequest struct { + URL *url.URL + Accept string +} + +func (r fetchRequest) key() string { + if r.URL == nil { + return r.Accept + } + + return r.Accept + " " + r.URL.String() +} + func Fetch(source string) (stdimage.Image, string, error) { parsed, err := url.Parse(source) if err != nil { return nil, "", fmt.Errorf("parse url: %w", err) } - queue := []*url.URL{parsed} + queue := []fetchRequest{{URL: parsed, Accept: supportedImageAcceptHeader}} seen := make(map[string]struct{}) var ( @@ -53,11 +66,11 @@ func Fetch(source string) (stdimage.Image, string, error) { current := queue[0] queue = queue[1:] - if current == nil { + if current.URL == nil { continue } - key := current.String() + key := current.key() if _, exists := seen[key]; exists { continue } @@ -256,7 +269,8 @@ func decodeAVIF(data []byte) (stdimage.Image, error) { return img, nil } -func githubAttachmentFallbacks(u *url.URL, payload []byte) []*url.URL { +func githubAttachmentFallbacks(current fetchRequest, payload []byte) []fetchRequest { + u := current.URL if u == nil || len(payload) == 0 { return nil } @@ -275,7 +289,7 @@ func githubAttachmentFallbacks(u *url.URL, payload []byte) []*url.URL { {Scheme: "https", Host: "github.com", Path: path.Join("/user-attachments/assets", id)}, } - variants := make([]*url.URL, 0, len(basePaths)*5) + variants := make([]fetchRequest, 0, len(basePaths)*5) for _, base := range basePaths { if base.Host == "" { @@ -296,14 +310,22 @@ func githubAttachmentFallbacks(u *url.URL, payload []byte) []*url.URL { query.Set("format", option.format) query.Set("name", option.name) clone.RawQuery = query.Encode() - variants = append(variants, &clone) + accept := current.Accept + switch option.format { + case "png": + accept = fallbackPNGAcceptHeader + case "jpg": + accept = fallbackJPEGAcceptHeader + } + + variants = append(variants, fetchRequest{URL: &clone, Accept: accept}) } clone := *base query := clone.Query() query.Set("raw", "1") clone.RawQuery = query.Encode() - variants = append(variants, &clone) + variants = append(variants, fetchRequest{URL: &clone, Accept: current.Accept}) } return variants @@ -602,20 +624,26 @@ func NormalizeRelativeURL(rel string) string { return b.String() } -func openSource(parsed *url.URL) (io.ReadCloser, string, string, error) { +func openSource(req fetchRequest) (io.ReadCloser, string, string, error) { + parsed := req.URL switch parsed.Scheme { case "http", "https": client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequest(http.MethodGet, parsed.String(), nil) + httpReq, err := http.NewRequest(http.MethodGet, parsed.String(), nil) if err != nil { return nil, "", "", fmt.Errorf("create request: %w", err) } - req.Header.Set("Accept", supportedImageAcceptHeader) - req.Header.Set("User-Agent", defaultRemoteImageUA) + accept := strings.TrimSpace(req.Accept) + if accept == "" { + accept = supportedImageAcceptHeader + } + + httpReq.Header.Set("Accept", accept) + httpReq.Header.Set("User-Agent", defaultRemoteImageUA) - resp, err := client.Do(req) + resp, err := client.Do(httpReq) if err != nil { return nil, "", "", fmt.Errorf("download image: %w", err) } diff --git a/pkg/images/image_test.go b/pkg/images/image_test.go index 15e63cee..417f6ce0 100644 --- a/pkg/images/image_test.go +++ b/pkg/images/image_test.go @@ -518,10 +518,16 @@ func TestFetchRemoteAVIFGitHubFallback(t *testing.T) { } const attachmentID = "e5abb532-59bf-49bb-a9d2-0c31872718d7" - var requests []string + + type requestInfo struct { + URL string + Accept string + } + + var requests []requestInfo server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests = append(requests, r.URL.String()) + requests = append(requests, requestInfo{URL: r.URL.String(), Accept: r.Header.Get("Accept")}) switch { case r.URL.Path == "/user-attachments/assets/"+attachmentID && r.URL.RawQuery == "": @@ -558,6 +564,25 @@ func TestFetchRemoteAVIFGitHubFallback(t *testing.T) { if len(requests) < 2 { t.Fatalf("expected fallback requests, got %v", requests) } + + if got := requests[0].Accept; got != supportedImageAcceptHeader { + t.Fatalf("unexpected accept header for initial request: %s", got) + } + + var sawPNG bool + for _, req := range requests[1:] { + if strings.Contains(req.URL, "format=png") { + if req.Accept != fallbackPNGAcceptHeader { + t.Fatalf("expected png fallback accept header %q, got %q", fallbackPNGAcceptHeader, req.Accept) + } + sawPNG = true + break + } + } + + if !sawPNG { + t.Fatalf("expected png fallback request, got %v", requests) + } } func TestFetchRemoteDecodeErrorIncludesContentType(t *testing.T) {