LibGfx+animation: Only store changed pixels in animation frames

Meta/gn/secondary/Userland/Libraries/LibGfx/BUILD.gn

@@ -63,6 +63,7 @@ shared_library("LibGfx") {
     "ICC/TagTypes.cpp",
     "ICC/Tags.cpp",
     "ICC/WellKnownProfiles.cpp",
+    "ImageFormats/AnimationWriter.cpp",
     "ImageFormats/BMPLoader.cpp",
     "ImageFormats/BMPWriter.cpp",
     "ImageFormats/BooleanDecoder.cpp",

Tests/LibGfx/TestImageWriter.cpp

@@ -9,6 +9,7 @@
 #include <LibGfx/ICC/BinaryWriter.h>
 #include <LibGfx/ICC/Profile.h>
 #include <LibGfx/ICC/WellKnownProfiles.h>
+#include <LibGfx/ImageFormats/AnimationWriter.h>
 #include <LibGfx/ImageFormats/BMPLoader.h>
 #include <LibGfx/ImageFormats/BMPWriter.h>
 #include <LibGfx/ImageFormats/JPEGLoader.h>
@@ -176,3 +177,38 @@ TEST_CASE(test_webp_animation)
     EXPECT_EQ(frame1.duration, 200);
     expect_bitmaps_equal(*frame1.image, *rgba_bitmap);
 }
+
+TEST_CASE(test_webp_incremental_animation)
+{
+    auto rgb_bitmap_1 = TRY_OR_FAIL(create_test_rgb_bitmap());
+
+    auto rgb_bitmap_2 = TRY_OR_FAIL(create_test_rgb_bitmap());
+
+    // WebP frames can't be at odd coordinates. Make a pixel at an odd coordinate different to make sure we handle this.
+    rgb_bitmap_2->scanline(3)[3] = Color::Red;
+
+    // 20 kiB is enough for two 47x33 frames.
+    auto stream_buffer = TRY_OR_FAIL(ByteBuffer::create_uninitialized(20 * 1024));
+    FixedMemoryStream stream { Bytes { stream_buffer } };
+
+    auto animation_writer = TRY_OR_FAIL(Gfx::WebPWriter::start_encoding_animation(stream, rgb_bitmap_1->size()));
+
+    TRY_OR_FAIL(animation_writer->add_frame(*rgb_bitmap_1, 100));
+    TRY_OR_FAIL(animation_writer->add_frame_relative_to_last_frame(*rgb_bitmap_2, 200, *rgb_bitmap_1));
+
+    auto encoded_animation = ReadonlyBytes { stream_buffer.data(), stream.offset() };
+
+    auto decoded_animation_plugin = TRY_OR_FAIL(Gfx::WebPImageDecoderPlugin::create(encoded_animation));
+    EXPECT(decoded_animation_plugin->is_animated());
+    EXPECT_EQ(decoded_animation_plugin->frame_count(), 2u);
+    EXPECT_EQ(decoded_animation_plugin->loop_count(), 0u);
+    EXPECT_EQ(decoded_animation_plugin->size(), rgb_bitmap_1->size());
+
+    auto frame0 = TRY_OR_FAIL(decoded_animation_plugin->frame(0));
+    EXPECT_EQ(frame0.duration, 100);
+    expect_bitmaps_equal(*frame0.image, *rgb_bitmap_1);
+
+    auto frame1 = TRY_OR_FAIL(decoded_animation_plugin->frame(1));
+    EXPECT_EQ(frame1.duration, 200);
+    expect_bitmaps_equal(*frame1.image, *rgb_bitmap_2);
+}

Userland/Libraries/LibGfx/CMakeLists.txt

@@ -36,6 +36,7 @@ set(SOURCES
     ICC/Tags.cpp
     ICC/TagTypes.cpp
     ICC/WellKnownProfiles.cpp
+    ImageFormats/AnimationWriter.cpp
     ImageFormats/BMPLoader.cpp
     ImageFormats/BMPWriter.cpp
     ImageFormats/BooleanDecoder.cpp

Userland/Libraries/LibGfx/ImageFormats/AnimationWriter.cpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2024, Nico Weber <thakis@chromium.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/ImageFormats/AnimationWriter.h>
+#include <LibGfx/Rect.h>
+
+namespace Gfx {
+
+AnimationWriter::~AnimationWriter() = default;
+
+static bool are_scanlines_equal(Bitmap const& a, Bitmap const& b, int y)
+{
+    for (int x = 0; x < a.width(); ++x) {
+        if (a.get_pixel(x, y) != b.get_pixel(x, y))
+            return false;
+    }
+    return true;
+}
+
+static bool are_columns_equal(Bitmap const& a, Bitmap const& b, int x, int y1, int y2)
+{
+    for (int y = y1; y < y2; ++y) {
+        if (a.get_pixel(x, y) != b.get_pixel(x, y))
+            return false;
+    }
+    return true;
+}
+
+static Gfx::IntRect rect_where_pixels_are_different(Bitmap const& a, Bitmap const& b)
+{
+    VERIFY(a.size() == b.size());
+
+    // FIXME: This works on physical pixels.
+    VERIFY(a.scale() == 1);
+    VERIFY(b.scale() == 1);
+
+    int number_of_equal_pixels_at_top = 0;
+    while (number_of_equal_pixels_at_top < a.height() && are_scanlines_equal(a, b, number_of_equal_pixels_at_top))
+        ++number_of_equal_pixels_at_top;
+
+    int number_of_equal_pixels_at_bottom = 0;
+    while (number_of_equal_pixels_at_bottom < a.height() - number_of_equal_pixels_at_top && are_scanlines_equal(a, b, a.height() - number_of_equal_pixels_at_bottom - 1))
+        ++number_of_equal_pixels_at_bottom;
+
+    int const y1 = number_of_equal_pixels_at_top;
+    int const y2 = a.height() - number_of_equal_pixels_at_bottom;
+
+    int number_of_equal_pixels_at_left = 0;
+    while (number_of_equal_pixels_at_left < a.width() && are_columns_equal(a, b, number_of_equal_pixels_at_left, y1, y2))
+        ++number_of_equal_pixels_at_left;
+
+    int number_of_equal_pixels_at_right = 0;
+    while (number_of_equal_pixels_at_right < a.width() - number_of_equal_pixels_at_left && are_columns_equal(a, b, a.width() - number_of_equal_pixels_at_right - 1, y1, y2))
+        ++number_of_equal_pixels_at_right;
+
+    // WebP can only encode even-sized animation frame positions.
+    // FIXME: Change API shape in some way so that the AnimationWriter base class doesn't have to know about this detail of a subclass.
+    if (number_of_equal_pixels_at_left % 2 != 0)
+        --number_of_equal_pixels_at_left;
+    if (number_of_equal_pixels_at_top % 2 != 0)
+        --number_of_equal_pixels_at_top;
+
+    Gfx::IntRect rect;
+    rect.set_x(number_of_equal_pixels_at_left);
+    rect.set_y(number_of_equal_pixels_at_top);
+    rect.set_width(a.width() - number_of_equal_pixels_at_left - number_of_equal_pixels_at_right);
+    rect.set_height(a.height() - number_of_equal_pixels_at_top - number_of_equal_pixels_at_bottom);
+
+    return rect;
+}
+
+ErrorOr<void> AnimationWriter::add_frame_relative_to_last_frame(Bitmap& frame, int duration_ms, RefPtr<Bitmap> last_frame)
+{
+    if (!last_frame)
+        return add_frame(frame, duration_ms);
+
+    auto rect = rect_where_pixels_are_different(*last_frame, frame);
+
+    // FIXME: It would be nice to have a way to crop a bitmap without copying the data.
+    auto differences = TRY(frame.cropped(rect));
+
+    // FIXME: Another idea: If all frames of the animation have no alpha,
+    //        this could set color values of pixels that are in the changed rect that are
+    //        equal to the last frame to transparent black and set the frame to be blended.
+    //        That might take less space after compression.
+
+    // This assumes a replacement disposal method.
+    return add_frame(differences, duration_ms, rect.location());
+}
+
+}

Userland/Libraries/LibGfx/ImageFormats/AnimationWriter.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2024, Nico Weber <thakis@chromium.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Error.h>
+#include <LibGfx/Forward.h>
+#include <LibGfx/Point.h>
+
+namespace Gfx {
+
+class AnimationWriter {
+public:
+    virtual ~AnimationWriter();
+
+    // Flushes the frame to disk.
+    // IntRect { at, at + bitmap.size() } must fit in the dimensions
+    // passed to `start_writing_animation()`.
+    // FIXME: Consider passing in disposal method and blend mode.
+    virtual ErrorOr<void> add_frame(Bitmap&, int duration_ms, IntPoint at = {}) = 0;
+
+    ErrorOr<void> add_frame_relative_to_last_frame(Bitmap&, int duration_ms, RefPtr<Bitmap> last_frame);
+};
+
+}

Userland/Libraries/LibGfx/ImageFormats/GIFLoader.cpp

@@ -337,6 +337,8 @@ static ErrorOr<void> load_gif_frame_descriptors(GIFLoadingContext& context)
             image->use_global_color_map = !(packed_fields & 0x80);
             image->interlaced = (packed_fields & 0x40) != 0;
 
+            dbgln_if(GIF_DEBUG, "Image descriptor: x={}, y={}, width={}, height={}, use_global_color_map={}, local_map_size_exponent={}, interlaced={}", image->x, image->y, image->width, image->height, image->use_global_color_map, (packed_fields & 7) + 1, image->interlaced);
+
             if (!image->use_global_color_map) {
                 size_t local_color_table_size = AK::exp2<size_t>((packed_fields & 7) + 1);
 

Userland/Libraries/LibGfx/ImageFormats/WebPLoader.cpp

@@ -13,6 +13,7 @@
 #include <LibGfx/ImageFormats/WebPLoader.h>
 #include <LibGfx/ImageFormats/WebPLoaderLossless.h>
 #include <LibGfx/ImageFormats/WebPLoaderLossy.h>
+#include <LibGfx/ImageFormats/WebPShared.h>
 #include <LibGfx/Painter.h>
 #include <LibRIFF/ChunkID.h>
 #include <LibRIFF/RIFF.h>
@@ -24,43 +25,6 @@ namespace Gfx {
 
 namespace {
 
-struct VP8XHeader {
-    bool has_icc;
-    bool has_alpha;
-    bool has_exif;
-    bool has_xmp;
-    bool has_animation;
-    u32 width;
-    u32 height;
-};
-
-struct ANIMChunk {
-    u32 background_color;
-    u16 loop_count;
-};
-
-struct ANMFChunk {
-    u32 frame_x;
-    u32 frame_y;
-    u32 frame_width;
-    u32 frame_height;
-    u32 frame_duration_in_milliseconds;
-
-    enum class BlendingMethod {
-        UseAlphaBlending = 0,
-        DoNotBlend = 1,
-    };
-    BlendingMethod blending_method;
-
-    enum class DisposalMethod {
-        DoNotDispose = 0,
-        DisposeToBackgroundColor = 1,
-    };
-    DisposalMethod disposal_method;
-
-    ReadonlyBytes frame_data;
-};
-
 // "For a still image, the image data consists of a single frame, which is made up of:
 //     An optional alpha subchunk.
 //     A bitstream subchunk."

Userland/Libraries/LibGfx/ImageFormats/WebPShared.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2024, Nico Weber <thakis@chromium.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Span.h>
+#include <AK/Types.h>
+
+namespace Gfx {
+
+struct VP8XHeader {
+    bool has_icc { false };
+    bool has_alpha { false };
+    bool has_exif { false };
+    bool has_xmp { false };
+    bool has_animation { false };
+    u32 width { 0 };
+    u32 height { 0 };
+};
+
+struct ANIMChunk {
+    u32 background_color { 0 };
+    u16 loop_count { 0 };
+};
+
+struct ANMFChunk {
+    u32 frame_x { 0 };
+    u32 frame_y { 0 };
+    u32 frame_width { 0 };
+    u32 frame_height { 0 };
+    u32 frame_duration_in_milliseconds { 0 };
+
+    enum class BlendingMethod {
+        UseAlphaBlending = 0,
+        DoNotBlend = 1,
+    };
+    BlendingMethod blending_method { BlendingMethod::UseAlphaBlending };
+
+    enum class DisposalMethod {
+        DoNotDispose = 0,
+        DisposeToBackgroundColor = 1,
+    };
+    DisposalMethod disposal_method { DisposalMethod::DoNotDispose };
+
+    ReadonlyBytes frame_data;
+};
+
+}

Userland/Libraries/LibGfx/ImageFormats/WebPWriter.cpp

@@ -11,6 +11,8 @@
 #include <AK/Debug.h>
 #include <LibCompress/DeflateTables.h>
 #include <LibGfx/Bitmap.h>
+#include <LibGfx/ImageFormats/AnimationWriter.h>
+#include <LibGfx/ImageFormats/WebPShared.h>
 #include <LibGfx/ImageFormats/WebPWriter.h>
 #include <LibRIFF/RIFF.h>
 
@@ -189,16 +191,6 @@ static ErrorOr<void> write_VP8L_image_data(Stream& stream, Bitmap const& bitmap)
     return {};
 }
 
-struct VP8XHeader {
-    bool has_icc { false };
-    bool has_alpha { false };
-    bool has_exif { false };
-    bool has_xmp { false };
-    bool has_animation { false };
-    u32 width { 0 };
-    u32 height { 0 };
-};
-
 static u8 vp8x_flags_from_header(VP8XHeader const& header)
 {
     u8 flags = 0;
@@ -362,30 +354,20 @@ static ErrorOr<void> align_to_two(SeekableStream& stream)
     return {};
 }
 
-struct ANMFChunk {
-    u32 frame_x { 0 };
-    u32 frame_y { 0 };
-    u32 frame_width { 0 };
-    u32 frame_height { 0 };
-    u32 frame_duration_in_milliseconds { 0 };
-
-    enum class BlendingMethod {
-        UseAlphaBlending = 0,
-        DoNotBlend = 1,
-    };
-    BlendingMethod blending_method { BlendingMethod::UseAlphaBlending };
-
-    enum class DisposalMethod {
-        DoNotDispose = 0,
-        DisposeToBackgroundColor = 1,
-    };
-    DisposalMethod disposal_method { DisposalMethod::DoNotDispose };
-
-    ReadonlyBytes frame_data;
-};
-
 static ErrorOr<void> write_ANMF_chunk(Stream& stream, ANMFChunk const& chunk)
 {
+    if (chunk.frame_width > (1 << 24) || chunk.frame_height > (1 << 24))
+        return Error::from_string_literal("WebP dimensions too large for ANMF chunk");
+
+    if (chunk.frame_width == 0 || chunk.frame_height == 0)
+        return Error::from_string_literal("WebP lossless animation frames must be at least one pixel wide and tall");
+
+    if (chunk.frame_x % 2 != 0 || chunk.frame_y % 2 != 0)
+        return Error::from_string_literal("WebP lossless animation frames must be at at even coordinates");
+
+    dbgln_if(WEBP_DEBUG, "writing ANMF frame_x {} frame_y {} frame_width {} frame_height {} frame_duration {} blending_method {} disposal_method {}",
+        chunk.frame_x, chunk.frame_y, chunk.frame_width, chunk.frame_height, chunk.frame_duration_in_milliseconds, (int)chunk.blending_method, (int)chunk.disposal_method);
+
     TRY(write_chunk_header(stream, "ANMF"sv, 16 + chunk.frame_data.size()));
 
     LittleEndianOutputBitStream bit_stream { MaybeOwned<Stream>(stream) };
@@ -498,11 +480,6 @@ ErrorOr<void> WebPAnimationWriter::set_alpha_bit_in_header()
     return {};
 }
 
-struct ANIMChunk {
-    u32 background_color { 0 };
-    u16 loop_count { 0 };
-};
-
 static ErrorOr<void> write_ANIM_chunk(Stream& stream, ANIMChunk const& chunk)
 {
     TRY(write_chunk_header(stream, "ANIM"sv, 6)); // Size of the ANIM chunk.