Removed fillOrphanNodes, now new colors are added to the octree color…

… arragement.

This gives more accuracy in the color picking criteria on new images pasted into an INDEXED sprite.

Also, added findMaskColor to fix the behavior reported: #3207

Both issues are related when RGBMAP is created. The 'mask color' and 'mask index' must be defined correctly to include/exclude during the table/octree map color search.

To do: Converting sprite to INDEXED should add 'mask color' to the palette when color count <256 and transparent color isn't in the palette.

src/app/cmd/set_pixel_format.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2020  Igara Studio S.A.
+// Copyright (C) 2019-2022  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -22,6 +22,7 @@
 #include "doc/document.h"
 #include "doc/layer.h"
 #include "doc/palette.h"
+#include "doc/rgbmap.h"
 #include "doc/sprite.h"
 #include "doc/tilesets.h"
 #include "render/quantization.h"
@@ -177,6 +178,12 @@ void SetPixelFormat::setFormat(PixelFormat format)
   Sprite* sprite = this->sprite();
 
   sprite->setPixelFormat(format);
+  if (format == IMAGE_INDEXED) {
+    int maskIndex = sprite->palette(0)->findMaskColor();
+    sprite->setTransparentColor(maskIndex == -1 ? 0 : maskIndex);
+  }
+  else
+    sprite->setTransparentColor(0);
   sprite->incrementVersion();
 
   // Regenerate extras
@@ -201,14 +208,25 @@ void SetPixelFormat::convertImage(doc::Sprite* sprite,
   ASSERT(oldImage);
   ASSERT(oldImage->pixelFormat() != IMAGE_TILEMAP);
 
+  // Making the RGBMap for Image->INDEXDED conversion.
+  // TODO: this is needed only when newImage
+  RgbMap* rgbmap;
+  int newMaskIndex = (isBackground ? -1 : 0);
+  if (m_newFormat == IMAGE_INDEXED) {
+    rgbmap = sprite->rgbMap(frame, sprite->rgbMapForSprite(), mapAlgorithm);
+    if (m_oldFormat == IMAGE_INDEXED)
+      newMaskIndex = sprite->transparentColor();
+    else
+      newMaskIndex = rgbmap->maskIndex();
+  }
   ImageRef newImage(
     render::convert_pixel_format
     (oldImage.get(), nullptr, m_newFormat,
      dithering,
-     sprite->rgbMap(frame, sprite->rgbMapForSprite(), mapAlgorithm),
+     rgbmap,
      sprite->palette(frame),
      isBackground,
-     oldImage->maskColor(),
+     newMaskIndex,
      toGray,
      delegate));
 

src/doc/octree_map.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (c) 2020-2021 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -36,64 +36,19 @@ void OctreeNode::addColor(color_t c, int level, OctreeNode* parent,
   (*m_children)[index].addColor(c, level + 1, this, paletteIndex, levelDeep);
 }
 
-void OctreeNode::fillOrphansNodes(const Palette* palette,
-                                  const color_t upstreamBranchColor,
-                                  const int level)
+int OctreeNode::mapColor(int  r, int g, int b, int a, int mask_index, const Palette* palette, int level) const
 {
-  for (int i=0; i<16; i++) {
-    OctreeNode& child = (*m_children)[i];
-
-    if (child.hasChildren()) {
-      child.fillOrphansNodes(
-        palette,
-        upstreamBranchColor + hextetToBranchColor(i, level),
-        level + 1);
-    }
-    else if (!(child.isLeaf())) {
-      // Here the node IS NOT a Leaf and HAS NOT children
-      // So, we must assign palette index to the current node
-      // to fill the "map holes" (i.e "death tree branches")
-      // BUT, if the level is low (a few bits to identify a color)
-      // 0, 1, 2, or 3, we need to create branchs/Leaves until
-      // the desired minimum color MSB bits.
-      if (level < MIN_LEVEL_OCTREE_DEEP) {
-        child.fillMostSignificantNodes(level);
-        i--;
-        continue;
-      }
-      int currentBranchColorAdd = hextetToBranchColor(i, level);
-      color_t branchColorMed = upstreamBranchColor |
-                               currentBranchColorAdd |
-                               ((level == 7) ? 0 : (0x01010101 << (6 - level))); // mid color adition
-      int indexMed = palette->findBestfit2(rgba_getr(branchColorMed),
-                                           rgba_getg(branchColorMed),
-                                           rgba_getb(branchColorMed),
-                                           rgba_geta(branchColorMed));
-      child.paletteIndex(indexMed);
-    }
+  // New behavior: if mapColor do not have an exact rgba match, it must calculate which
+  // color of the current palette is the bestfit and memorize the index in a octree leaf.
+  if (level >= 8) {
+    if (m_paletteIndex == -1)
+      m_paletteIndex = palette->findBestfit(r, g, b, a, mask_index);
+    return m_paletteIndex;
   }
-}
-
-void OctreeNode::fillMostSignificantNodes(int level)
-{
-  if (level < MIN_LEVEL_OCTREE_DEEP) {
+  int index = getHextet(r, g, b, a, level);
+  if (!m_children)
     m_children.reset(new std::array<OctreeNode, 16>());
-    level++;
-    for (int i=0; i<16; i++) {
-      OctreeNode& child = (*m_children)[i];
-
-      child.fillMostSignificantNodes(level);
-    }
-  }
-}
-
-int OctreeNode::mapColor(int r, int g, int b, int a, int level) const
-{
-  OctreeNode& child = (*m_children)[getHextet(rgba(r, g, b, a), level)];
-  if (child.hasChildren())
-    return child.mapColor(r, g, b, a, level+1);
-  else
-    return child.m_paletteIndex;
+  return (*m_children)[index].mapColor(r, g, b, a, mask_index, palette, level + 1);
 }
 
 void OctreeNode::collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex)
@@ -143,6 +98,14 @@ int OctreeNode::getHextet(color_t c, int level)
          ((c & (0x80000000 >> level)) ? 8 : 0);
 }
 
+int OctreeNode::getHextet(int r, int g, int b, int a, int level)
+{
+  return ((r & (0x80 >> level)) ? 1 : 0) |
+         ((g & (0x80 >> level)) ? 2 : 0) |
+         ((b & (0x80 >> level)) ? 4 : 0) |
+         ((a & (0x80 >> level)) ? 8 : 0);
+}
+
 // static
 color_t OctreeNode::hextetToBranchColor(int hextet, int level)
 {
@@ -261,11 +224,6 @@ bool OctreeMap::makePalette(Palette* palette,
   return true;
 }
 
-void OctreeMap::fillOrphansNodes(const Palette* palette)
-{
-  m_root.fillOrphansNodes(palette, 0, 0);
-}
-
 void OctreeMap::feedWithImage(const Image* image,
                               const bool withAlpha,
                               const color_t maskColor,
@@ -310,13 +268,12 @@ void OctreeMap::feedWithImage(const Image* image,
 
 int OctreeMap::mapColor(color_t rgba) const
 {
-  if (m_root.hasChildren())
-    return m_root.mapColor(rgba_getr(rgba),
-                           rgba_getg(rgba),
-                           rgba_getb(rgba),
-                           rgba_geta(rgba), 0);
-  else
-    return -1;
+  return m_root.mapColor(rgba_getr(rgba),
+                         rgba_getg(rgba),
+                         rgba_getb(rgba),
+                         rgba_geta(rgba),
+                         m_maskIndex,
+                         m_palette, 0);
 }
 
 void OctreeMap::regenerateMap(const Palette* palette, const int maskIndex)
@@ -354,7 +311,6 @@ void OctreeMap::regenerateMap(const Palette* palette, const int maskIndex)
     }
     m_root.addColor(palette->entry(i), 0, &m_root, i, 8);
   }
-  m_root.fillOrphansNodes(palette, 0, 0);
 
   m_palette = palette;
   m_modifications = palette->getModifications();

src/doc/octree_map.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (c) 2020-2021 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -93,13 +93,7 @@ class OctreeNode {
   void addColor(color_t c, int level, OctreeNode* parent,
                 int paletteIndex = 0, int levelDeep = 7);
 
-  void fillOrphansNodes(const Palette* palette,
-                        const color_t upstreamBranchColor,
-                        const int level);
-
-  void fillMostSignificantNodes(int level);
-
-  int mapColor(int r, int g, int b, int a, int level) const;
+  int mapColor(int  r, int g, int b, int a, int mask_index, const Palette* palette, int level) const;
 
   void collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex);
 
@@ -114,11 +108,12 @@ class OctreeNode {
   void paletteIndex(int index) { m_paletteIndex = index; }
 
   static int getHextet(color_t c, int level);
+  static int getHextet(int r, int g, int b, int a, int level);
   static color_t hextetToBranchColor(int hextet, int level);
 
   LeafColor m_leafColor;
-  int m_paletteIndex = 0;
-  std::unique_ptr<std::array<OctreeNode, 16>> m_children;
+  mutable int m_paletteIndex = -1;
+  mutable std::unique_ptr<std::array<OctreeNode, 16>> m_children;
   OctreeNode* m_parent = nullptr;
 };
 
@@ -142,12 +137,20 @@ class OctreeMap : public RgbMap {
   // RgbMap impl
   void regenerateMap(const Palette* palette, const int maskIndex) override;
   int mapColor(color_t rgba) const override;
+  int maskIndex() const override { return m_maskIndex; }
+  int mapColor(const int r, const int g,
+               const int b, const int a) const
+  {
+    ASSERT(r >= 0 && r < 256);
+    ASSERT(g >= 0 && g < 256);
+    ASSERT(b >= 0 && b < 256);
+    ASSERT(a >= 0 && a < 256);
+    return mapColor(rgba(r, g, b, a));
+  }
 
   int moodifications() const { return m_modifications; };
 
 private:
-  void fillOrphansNodes(const Palette* palette);
-
   OctreeNode m_root;
   OctreeNodes m_leavesVector;
   const Palette* m_palette = nullptr;

src/doc/palette.cpp

@@ -483,37 +483,14 @@ int Palette::findBestfit(int r, int g, int b, int a, int mask_index) const
   return bestfit;
 }
 
-int Palette::findBestfit2(int r, int g, int b, int a) const
+int Palette::findMaskColor() const
 {
-  ASSERT(r >= 0 && r <= 255);
-  ASSERT(g >= 0 && g <= 255);
-  ASSERT(b >= 0 && b <= 255);
-  ASSERT(a >= 0 && a <= 255);
-
-  int bestfit = 0;
-  int lowest = std::numeric_limits<int>::max();
   int size = m_colors.size();
-
-  for (int i=0; i<size; ++i) {
-    color_t rgb = m_colors[i];
-    int rDiff = r - rgba_getr(rgb);
-    int gDiff = g - rgba_getg(rgb);
-    int bDiff = b - rgba_getb(rgb);
-    int aDiff = a - rgba_geta(rgb);
-
-    // TODO We should have two different ways to calculate the
-    // distance between colors, like "Perceptual" and "Linear", or a
-    // way to configure these coefficients.
-    int diff = rDiff * rDiff * 900  +
-               gDiff * gDiff * 3481 +
-               bDiff * bDiff * 121  +
-               aDiff * aDiff * 900; // there is no scientific reason to choose this value.
-    if (diff < lowest) {
-      lowest = diff;
-      bestfit = i;
-    }
+  for (int i = 0; i < size; ++i) {
+    if (m_colors[i] == 0)
+      return i;
   }
-  return bestfit;
+  return -1;
 }
 
 void Palette::applyRemap(const Remap& remap)

src/doc/palette.h

@@ -101,7 +101,7 @@ namespace doc {
     int findExactMatch(int r, int g, int b, int a, int mask_index) const;
     bool findExactMatch(color_t color) const;
     int findBestfit(int r, int g, int b, int a, int mask_index) const;
-    int findBestfit2(int r, int g, int b, int a) const;
+    int findMaskColor() const;
 
     void applyRemap(const Remap& remap);
 

src/doc/rgbmap.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2020 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -25,6 +25,8 @@ namespace doc {
     // Should return the best index in a palette that matches the given RGBA values.
     virtual int mapColor(const color_t rgba) const = 0;
 
+    virtual int maskIndex() const = 0;
+
     int mapColor(const int r,
                  const int g,
                  const int b,

src/doc/rgbmap_rgb5a3.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2020 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 // Copyright (c) 2001-2016 David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -41,7 +41,7 @@ namespace doc {
       return (v & INVALID) ? generateEntry(i, r, g, b, a): v;
     }
 
-    int maskIndex() const { return m_maskIndex; }
+    int maskIndex() const override { return m_maskIndex; }
 
   private:
     int generateEntry(int i, int r, int g, int b, int a) const;

src/doc/sprite.cpp

@@ -395,9 +395,6 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
                        const RgbMapFor forLayer,
                        RgbMapAlgorithm mapAlgo) const
 {
-  int maskIndex = (forLayer == RgbMapFor::OpaqueLayer ?
-                   -1: transparentColor());
-
   if (!m_rgbMap || m_rgbMapAlgorithm != mapAlgo) {
     m_rgbMapAlgorithm = mapAlgo;
     switch (m_rgbMapAlgorithm) {
@@ -410,7 +407,14 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
         return nullptr;
     }
   }
-
+  int maskIndex;
+  if (forLayer == RgbMapFor::OpaqueLayer)
+    maskIndex = -1;
+  else {
+    maskIndex = palette(frame)->findMaskColor();
+    if (maskIndex == -1)
+      maskIndex = 0;
+  }
   m_rgbMap->regenerateMap(palette(frame), maskIndex);
   return m_rgbMap.get();
 }

src/render/quantization.cpp

@@ -1,5 +1,5 @@
 // Aseprite Render Library
-// Copyright (c) 2019-2021  Igara Studio S.A.
+// Copyright (c) 2019-2022  Igara Studio S.A.
 // Copyright (c) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -237,7 +237,7 @@ Image* convert_pixel_format(
             a = rgba_geta(c);
 
             if (a == 0)
-              *dst_it = new_mask_color;
+              *dst_it = (new_mask_color == -1? 0 : new_mask_color);
             else if (rgbmap)
               *dst_it = rgbmap->mapColor(c);
             else
@@ -296,7 +296,7 @@ Image* convert_pixel_format(
             c = graya_getv(c);
 
             if (a == 0)
-              *dst_it = new_mask_color;
+              *dst_it = (new_mask_color == -1? 0 : new_mask_color);
             else if (rgbmap)
               *dst_it = rgbmap->mapColor(c, c, c, a);
             else