Add support for alpha channel in RgbMap and Palette::findBestfit()

This include several changes:
- Color::getIndex() can return palette values with alpha != 255
- Fix Transparent and Blurs ink for indexed images to make better use
  of palette entries with alpha values
- Fix bilinear resize algorithm for indexed images with alpha
- New RgbMap with four parameters: R, G, B, A
- Add one extra color scale function used in the alpha channel of the
  new RgbMap
- Fix color curve, convolution matrix, invert color, and median filters
  to take care of this new alpha channel on indexed images
- Fix ordered dithering and quantization

Related to #286

src/app/color.cpp

@@ -616,8 +616,13 @@ int Color::getIndex() const
 
     case Color::RgbType:
     case Color::HsvType:
-    case Color::GrayType:
-      return get_current_palette()->findBestfit(getRed(), getGreen(), getBlue(), getAlpha(), 0);
+    case Color::GrayType: {
+      int i = get_current_palette()->findExactMatch(getRed(), getGreen(), getBlue(), getAlpha());
+      if (i >= 0)
+        return i;
+      else
+        return get_current_palette()->findBestfit(getRed(), getGreen(), getBlue(), getAlpha(), 0);
+    }
 
     case Color::IndexType:
       return m_value.index;

src/app/commands/cmd_sprite_size.cpp

@@ -28,6 +28,7 @@
 #include "doc/cel.h"
 #include "doc/cels_range.h"
 #include "doc/image.h"
+#include "doc/layer.h"
 #include "doc/mask.h"
 #include "doc/primitives.h"
 #include "doc/sprite.h"
@@ -95,10 +96,12 @@ class SpriteSizeJob : public Job {
         ImageRef new_image(Image::create(image->pixelFormat(), MAX(1, w), MAX(1, h)));
 
         doc::algorithm::fixup_image_transparent_colors(image);
-        doc::algorithm::resize_image(image, new_image.get(),
+        doc::algorithm::resize_image(
+          image, new_image.get(),
           m_resize_method,
           m_sprite->palette(cel->frame()),
-          m_sprite->rgbMap(cel->frame()));
+          m_sprite->rgbMap(cel->frame()),
+          (cel->layer()->isBackground() ? -1: m_sprite->transparentColor()));
 
         api.replaceImage(m_sprite, cel->imageRef(), new_image);
       }
@@ -125,10 +128,12 @@ class SpriteSizeJob : public Job {
         gfx::Rect(
           scale_x(m_document->mask()->bounds().x-1),
           scale_y(m_document->mask()->bounds().y-1), MAX(1, w), MAX(1, h)));
-      algorithm::resize_image(old_bitmap.get(), new_mask->bitmap(),
-          m_resize_method,
-          m_sprite->palette(0), // Ignored
-          m_sprite->rgbMap(0)); // Ignored
+      algorithm::resize_image(
+        old_bitmap.get(), new_mask->bitmap(),
+        m_resize_method,
+        m_sprite->palette(0), // Ignored
+        m_sprite->rgbMap(0),  // Ignored
+        -1);                  // Ignored
 
       // Reshrink
       new_mask->intersect(new_mask->bounds());

src/app/commands/filters/filter_target_buttons.cpp

@@ -61,17 +61,15 @@ FilterTargetButtons::FilterTargetButtons(int imgtype, bool withChannels)
       case IMAGE_INDEXED:
         r = check_button_new("R", 2, 0, 0, 0);
         g = check_button_new("G", 0, 0, 0, 0);
-        b = check_button_new("B", 0, (imgtype == IMAGE_RGB) ? 0: 2, 0, 0);
+        b = check_button_new("B", 0, 0, 0, 0);
+        a = check_button_new("A", 0, 2, 0, 0);
 
         r->setId("r");
         g->setId("g");
         b->setId("b");
+        a->setId("a");
 
-        if (imgtype == IMAGE_RGB) {
-          a = check_button_new("A", 0, 2, 0, 0);
-          a->setId("a");
-        }
-        else {
+        if (imgtype == IMAGE_INDEXED) {
           index = check_button_new("Index", 0, 0, 0, 0);
           index->setId("i");
         }

src/app/tools/ink_processing.h

@@ -10,6 +10,7 @@
 #include "app/tools/shading_options.h"
 #include "doc/blend_funcs.h"
 #include "doc/image_impl.h"
+#include "doc/layer.h"
 #include "doc/palette.h"
 #include "doc/rgbmap.h"
 #include "doc/sprite.h"
@@ -241,21 +242,30 @@ class TransparentInkProcessing<IndexedTraits> : public DoubleInkProcessing<Trans
     m_palette(get_current_palette()),
     m_rgbmap(loop->getRgbMap()),
     m_opacity(loop->getOpacity()),
-    m_color(m_palette->getEntry(loop->getPrimaryColor())) {
+    m_color(m_palette->getEntry(loop->getPrimaryColor())),
+    m_maskColor(loop->getLayer()->isBackground() ? -1: loop->sprite()->transparentColor()) {
   }
 
   void processPixel(int x, int y) {
-    color_t c = rgba_blender_normal(m_palette->getEntry(*m_srcAddress), m_color, m_opacity);
+    color_t c = *m_srcAddress;
+    if (c == m_maskColor)
+      c = m_palette->getEntry(c) & rgba_rgb_mask;  // Alpha = 0
+    else
+      c = m_palette->getEntry(c);
+
+    c = rgba_blender_normal(c, m_color, m_opacity);
     *m_dstAddress = m_rgbmap->mapColor(rgba_getr(c),
                                        rgba_getg(c),
-                                       rgba_getb(c));
+                                       rgba_getb(c),
+                                       rgba_geta(c));
   }
 
 private:
   const Palette* m_palette;
   const RgbMap* m_rgbmap;
   int m_opacity;
   color_t m_color;
+  color_t m_maskColor;
 };
 
 //////////////////////////////////////////////////////////////////////
@@ -387,24 +397,27 @@ class BlurInkProcessing<IndexedTraits> : public DoubleInkProcessing<BlurInkProce
     m_opacity(loop->getOpacity()),
     m_tiledMode(loop->getTiledMode()),
     m_srcImage(loop->getSrcImage()),
-    m_area(get_current_palette()) {
+    m_area(get_current_palette(),
+           loop->getLayer()->isBackground() ? -1: loop->sprite()->transparentColor()) {
   }
 
   void processPixel(int x, int y) {
     m_area.reset();
     get_neighboring_pixels<IndexedTraits>(m_srcImage, x, y, 3, 3, 1, 1, m_tiledMode, m_area);
 
-    if (m_area.count > 0 && m_area.a/9 >= 128) {
+    if (m_area.count > 0) {
       m_area.r /= m_area.count;
       m_area.g /= m_area.count;
       m_area.b /= m_area.count;
+      m_area.a /= 9;
 
       uint32_t color32 = m_palette->getEntry(*m_srcAddress);
       m_area.r = rgba_getr(color32) + (m_area.r-rgba_getr(color32)) * m_opacity / 255;
       m_area.g = rgba_getg(color32) + (m_area.g-rgba_getg(color32)) * m_opacity / 255;
       m_area.b = rgba_getb(color32) + (m_area.b-rgba_getb(color32)) * m_opacity / 255;
+      m_area.a = rgba_geta(color32) + (m_area.a-rgba_geta(color32)) * m_opacity / 255;
 
-      *m_dstAddress = m_rgbmap->mapColor(m_area.r, m_area.g, m_area.b);
+      *m_dstAddress = m_rgbmap->mapColor(m_area.r, m_area.g, m_area.b, m_area.a);
     }
     else {
       *m_dstAddress = *m_srcAddress;
@@ -415,20 +428,27 @@ class BlurInkProcessing<IndexedTraits> : public DoubleInkProcessing<BlurInkProce
   struct GetPixelsDelegate {
     const Palette* pal;
     int count, r, g, b, a;
+    color_t maskColor;
 
-    GetPixelsDelegate(const Palette* pal) : pal(pal) { }
+    GetPixelsDelegate(const Palette* pal,
+                      color_t maskColor)
+      : pal(pal), maskColor(maskColor) { }
 
     void reset() { count = r = g = b = a = 0; }
 
     void operator()(IndexedTraits::pixel_t color)
     {
-      a += (color == 0 ? 0: 255);
+      if (color == maskColor)
+        return;
 
       uint32_t color32 = pal->getEntry(color);
-      r += rgba_getr(color32);
-      g += rgba_getg(color32);
-      b += rgba_getb(color32);
-      count++;
+      if (rgba_geta(color32) > 0) {
+        r += rgba_getr(color32);
+        g += rgba_getg(color32);
+        b += rgba_getb(color32);
+        a += rgba_geta(color32);
+        ++count;
+      }
     }
   };
 
@@ -510,7 +530,7 @@ class ReplaceInkProcessing<IndexedTraits> : public DoubleInkProcessing<ReplaceIn
           m_palette->getEntry(*m_srcAddress), m_color2, m_opacity);
 
         *m_dstAddress = m_rgbmap->mapColor(
-          rgba_getr(c), rgba_getg(c), rgba_getb(c));
+          rgba_getr(c), rgba_getg(c), rgba_getb(c), rgba_geta(c));
       }
     }
   }
@@ -610,7 +630,8 @@ void JumbleInkProcessing<IndexedTraits>::processPixel(int x, int y)
   if (rgba_geta(c) >= 128)
     *m_dstAddress = m_rgbmap->mapColor(rgba_getr(c),
                                        rgba_getg(c),
-                                       rgba_getb(c));
+                                       rgba_getb(c),
+                                       rgba_geta(c));
   else
     *m_dstAddress = 0;
 }
@@ -692,7 +713,8 @@ class XorInkProcessing<IndexedTraits> : public DoubleInkProcessing<XorInkProcess
     color_t c = rgba_blender_neg_bw(m_palette->getEntry(*m_srcAddress), m_color, 255);
     *m_dstAddress = m_rgbmap->mapColor(rgba_getr(c),
                                        rgba_getg(c),
-                                       rgba_getb(c));
+                                       rgba_getb(c),
+                                       rgba_geta(c));
   }
 
 private:

src/doc/CMakeLists.txt

@@ -17,7 +17,6 @@ add_library(doc-lib
   cel_data_io.cpp
   cel_io.cpp
   cels_range.cpp
-  color_scales.cpp
   compressed_image.cpp
   context.cpp
   conversion_she.cpp

src/doc/algorithm/resize_image.cpp

@@ -18,7 +18,7 @@
 namespace doc {
 namespace algorithm {
 
-void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* pal, const RgbMap* rgbmap)
+void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* pal, const RgbMap* rgbmap, color_t maskColor)
 {
   switch (method) {
 
@@ -111,15 +111,23 @@ void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palet
               break;
             }
             case IMAGE_INDEXED: {
-              int r = int((rgba_getr(pal->getEntry(color[0]))*u2 + rgba_getr(pal->getEntry(color[1]))*u1)*v2 +
-                          (rgba_getr(pal->getEntry(color[2]))*u2 + rgba_getr(pal->getEntry(color[3]))*u1)*v1);
-              int g = int((rgba_getg(pal->getEntry(color[0]))*u2 + rgba_getg(pal->getEntry(color[1]))*u1)*v2 +
-                          (rgba_getg(pal->getEntry(color[2]))*u2 + rgba_getg(pal->getEntry(color[3]))*u1)*v1);
-              int b = int((rgba_getb(pal->getEntry(color[0]))*u2 + rgba_getb(pal->getEntry(color[1]))*u1)*v2 +
-                          (rgba_getb(pal->getEntry(color[2]))*u2 + rgba_getb(pal->getEntry(color[3]))*u1)*v1);
-              int a = int(((color[0] == 0 ? 0: 255)*u2 + (color[1] == 0 ? 0: 255)*u1)*v2 +
-                          ((color[2] == 0 ? 0: 255)*u2 + (color[3] == 0 ? 0: 255)*u1)*v1);
-              dst_color = a > 127 ? rgbmap->mapColor(r, g, b): 0;
+              // Convert index to RGBA values
+              for (int i=0; i<4; ++i) {
+                if (color[i] == maskColor)
+                  color[i] = pal->getEntry(color[i]) & rgba_rgb_mask; // Set alpha = 0
+                else
+                  color[i] = pal->getEntry(color[i]);
+              }
+
+              int r = int((rgba_getr(color[0])*u2 + rgba_getr(color[1])*u1)*v2 +
+                          (rgba_getr(color[2])*u2 + rgba_getr(color[3])*u1)*v1);
+              int g = int((rgba_getg(color[0])*u2 + rgba_getg(color[1])*u1)*v2 +
+                          (rgba_getg(color[2])*u2 + rgba_getg(color[3])*u1)*v1);
+              int b = int((rgba_getb(color[0])*u2 + rgba_getb(color[1])*u1)*v2 +
+                          (rgba_getb(color[2])*u2 + rgba_getb(color[3])*u1)*v1);
+              int a = int((rgba_geta(color[0])*u2 + rgba_geta(color[1])*u1)*v2 +
+                          (rgba_geta(color[2])*u2 + rgba_geta(color[3])*u1)*v1);
+              dst_color = rgbmap->mapColor(r, g, b, a);
               break;
             }
           }

src/doc/algorithm/resize_image.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
+// Copyright (c) 2001-2015 David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -8,6 +8,7 @@
 #define DOC_ALGORITHM_RESIZE_IMAGE_H_INCLUDED
 #pragma once
 
+#include "doc/color.h"
 #include "gfx/fwd.h"
 
 namespace doc {
@@ -27,7 +28,8 @@ namespace doc {
     // Warning: If you are using the RESIZE_METHOD_BILINEAR, it is
     // recommended to use 'fixup_image_transparent_colors' function
     // over the source image 'src' BEFORE using this routine.
-    void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* palette, const RgbMap* rgbmap);
+    void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* palette, const RgbMap* rgbmap,
+                      color_t maskColor);
 
     // It does not modify the image to the human eye, but internally
     // tries to fixup all colors that are completelly transparent