Fixed Terrain Fill Mode for sets containing transitions to empty (#3774)

Mostly removing the special handling for erasing and changing the way filling
is implemented. This does mean that terrain sets now support only 254
colors (down from 255).

* Slightly simplified updateAdjacent (one less call to "oppositeIndex").

* In wangIdFromSurrounding, prefer colors over "no color".

* In WangFiller::wangIdFromSurroundings, don't let empty tiles contribute to
  the WangId.

* Don't require tiles on the border of the region to match their surroundings,
  but only prefer them to.

* Slightly optimized WangBrush rotational symmetry mode.

* Disabled placement of empty tiles by Stamp Brush in Terrain Fill mode.

* Added TileLayerWangEdit.erasingEnabled property in the scripting API.

* WangFiller: Don't try to make changes outside of a fixed map.

This avoids the problem, that tiles set outside of the map by the algorithm
will affect the tiles inside the map, which is not desirable.

Also, explicitly set tiles that are outside of the map to empty. This seems
like a good idea, since it means the preview will still highlight those areas,
but it will not affect the map once applied.

NEWS.md

@@ -10,6 +10,7 @@
 * Fixed updating of object selection outlines when changing parallax factor (#3669)
 * Fixed "Offset Map" action to offset all objects when choosing "Whole Map" as bounds
 * Fixed several issues with drawing ellipses (#3776)
+* Fixed Terrain Fill Mode for sets containing transitions to empty (#3774)
 * Godot 4 plugin: Export custom tile properties as Custom Data Layers (with Kevin Harrison, #3653)
 * AppImage: Updated to Sentry 0.6.4
 * Qt 6: Increased the image allocation limit from 1 GB to 4 GB (#3616)

docs/manual/terrain.rst

@@ -49,7 +49,7 @@ Mixed Set
 Based on the information in a terrain set, the :ref:`terrain-tool` can
 understand the map and automatically choose the right tiles when making edits.
 When necessary, it also adjusts neighboring tiles to make sure they correctly
-connect to the modified area. A terrain set can contain up to 255 terrains.
+connect to the modified area. A terrain set can contain up to 254 terrains.
 
 The :ref:`stamp-tool`, as well as the :ref:`bucket-fill-tool` and the
 :ref:`shape-fill-tool`, also have a mode where they can :ref:`fill an area with
@@ -335,7 +335,7 @@ your own project. A few things to keep in mind:
   the terrain overlay is displayed correctly, set up the *Orientation*,
   *Grid Width* and *Grid Height* in the tileset properties.
 
-- The tool will handle any number of terrains (up to 255) and each corner of a
+- The tool will handle any number of terrains (up to 254) and each corner of a
   tile can have a different type of terrain. Still, there are other ways of
   dealing with transitions that this tool can't handle. Also, it is not able
   to edit multiple layers at the same time. For a more flexible, but also more

docs/reference/tmx-map-format.rst

@@ -359,7 +359,7 @@ Defines a list of colors and any number of Wang tiles using these colors.
 
 Can contain at most one: :ref:`tmx-properties`
 
-Can contain up to 255: :ref:`tmx-wangcolor` (since Tiled 1.5)
+Can contain up to 254: :ref:`tmx-wangcolor` (255 since Tiled 1.5, 254 since Tiled 1.10.2)
 
 Can contain any number: :ref:`tmx-wangtile`
 
@@ -389,7 +389,7 @@ associating it with a certain Wang ID.
 
 -  **tileid:** The tile ID.
 -  **wangid:** The Wang ID, since Tiled 1.5 given by a comma-separated list of
-   indexes (0-255) referring to the Wang colors in the Wang set in the order:
+   indexes (0-254) referring to the Wang colors in the Wang set in the order:
    top, top-right, right, bottom-right, bottom, bottom-left, left, top-left.
    Index 0 means *unset* and index 1 refers to the first Wang color. Before
    Tiled 1.5, the Wang ID was saved as a 32-bit unsigned integer stored in the

docs/scripting-doc/index.d.ts

@@ -2965,6 +2965,12 @@ interface TileLayerWangEdit {
    */
   correctionsEnabled : boolean
 
+  /**
+   * Whether the empty tile is considered when looking for matching tiles.
+   * Defaults to `true`.
+   */
+  erasingEnabled : boolean
+
   /**
    * Sets the desired color for the given Wang index at the given location.
    *

src/libtiled/grid.h

@@ -60,6 +60,9 @@ class Grid
             mValues(CHUNK_SIZE * CHUNK_SIZE)
         {}
 
+        T &get(int x, int y) { return mValues[x + y * CHUNK_SIZE]; }
+        T &get(QPoint point) { return get(point.x(), point.y()); }
+
         const T &get(int x, int y) const { return mValues.at(x + y * CHUNK_SIZE); }
         const T &get(QPoint point) const { return get(point.x(), point.y()); }
 
@@ -102,6 +105,20 @@ class Grid
         return get(point.x(), point.y());
     }
 
+    T &add(int x, int y)
+    {
+        Chunk *chunk = findChunk(x, y);
+        if (!chunk)
+            chunk = &this->chunk(x, y);
+
+        return chunk->get(x & CHUNK_MASK, y & CHUNK_MASK);
+    }
+
+    T &add(QPoint point)
+    {
+        return add(point.x(), point.y());
+    }
+
     /**
      * Sets the value at the given coordinates.
      */

src/libtiled/wangset.cpp

@@ -718,60 +718,6 @@ QList<WangTile> WangSet::sortedWangTiles() const
     return wangTiles;
 }
 
-/**
- * Returns a WangId matching that of the provided \a surroundingWangIds.
- *
- * This is based off a provided array, { 0, 1, 2, 3, 4, 5, 6, 7 },
- * which corresponds to:
- *
- *      7|0|1
- *      6|X|2
- *      5|4|3
- */
-WangId WangSet::wangIdFromSurrounding(const WangId surroundingWangIds[])
-{
-    quint64 id = 0;
-
-    // Edges
-    for (int i = 0; i < WangId::NumEdges; ++i)
-        id |= quint64(surroundingWangIds[i*2].edgeColor((2 + i) % WangId::NumEdges)) << (i * WangId::BITS_PER_INDEX * 2);
-
-    // Corners
-    for (int i = 0; i < WangId::NumCorners; ++i) {
-        int color = surroundingWangIds[i*2 + 1].cornerColor((2 + i) % WangId::NumCorners);
-
-        if (!color)
-            color = surroundingWangIds[i*2].cornerColor((1 + i) % WangId::NumCorners);
-
-        if (!color)
-            color = surroundingWangIds[(i*2 + 2) % WangId::NumIndexes].cornerColor((3 + i) % WangId::NumCorners);
-
-        id |= quint64(color) << (WangId::BITS_PER_INDEX + i * WangId::BITS_PER_INDEX * 2);
-    }
-
-    return id;
-}
-
-/**
- * Returns a wangId matching that of the provided surrounding tiles.
- *
- * This is based off a provided array, { 0, 1, 2, 3, 4, 5, 6, 7 },
- * which corresponds to:
- *
- *      7|0|1
- *      6|X|2
- *      5|4|3
- */
-WangId WangSet::wangIdFromSurrounding(const Cell surroundingCells[]) const
-{
-    WangId wangIds[WangId::NumIndexes];
-
-    for (int i = 0; i < WangId::NumIndexes; ++i)
-        wangIds[i] = wangIdOfCell(surroundingCells[i]);
-
-    return wangIdFromSurrounding(wangIds);
-}
-
 /**
  * Returns the WangId of a given \a tile.
  *

src/libtiled/wangset.h

@@ -47,7 +47,7 @@ class TILEDSHARED_EXPORT WangId
     constexpr static unsigned BITS_PER_INDEX = 8;
     constexpr static quint64 INDEX_MASK = 0xFF;
     constexpr static quint64 FULL_MASK = Q_UINT64_C(0xFFFFFFFFFFFFFFFF);
-    constexpr static int MAX_COLOR_COUNT = (1 << BITS_PER_INDEX) - 1;
+    constexpr static int MAX_COLOR_COUNT = (1 << BITS_PER_INDEX) - 2;
 
     enum Index {
         Top         = 0,
@@ -74,6 +74,11 @@ class TILEDSHARED_EXPORT WangId
         MaskLeft        = INDEX_MASK << (BITS_PER_INDEX * Left),
         MaskTopLeft     = INDEX_MASK << (BITS_PER_INDEX * TopLeft),
 
+        MaskTopSide     = MaskTopLeft | MaskTop | MaskTopRight,
+        MaskRightSide   = MaskTopRight | MaskRight | MaskBottomRight,
+        MaskBottomSide  = MaskBottomLeft | MaskBottom | MaskBottomRight,
+        MaskLeftSide    = MaskTopLeft | MaskLeft | MaskBottomLeft,
+
         MaskEdges       = MaskTop | MaskRight | MaskBottom | MaskLeft,
         MaskCorners     = MaskTopRight | MaskBottomRight | MaskBottomLeft | MaskTopLeft,
     };
@@ -95,6 +100,7 @@ class TILEDSHARED_EXPORT WangId
     void setIndexColor(int index, unsigned value);
 
     void updateToAdjacent(WangId adjacent, int position);
+    void mergeWith(WangId wangId, quint64 mask);
 
     bool hasWildCards() const;
     bool hasCornerWildCards() const;
@@ -128,6 +134,11 @@ class TILEDSHARED_EXPORT WangId
     quint64 mId;
 };
 
+inline void WangId::mergeWith(WangId wangId, quint64 mask)
+{
+    mId = (mId & ~mask) | (wangId & mask);
+}
+
 inline WangId::Index WangId::oppositeIndex(int index)
 {
     return static_cast<Index>((index + 4) % NumIndexes);
@@ -278,9 +289,6 @@ class TILEDSHARED_EXPORT WangSet : public Object
 
     QList<WangTile> sortedWangTiles() const;
 
-    static WangId wangIdFromSurrounding(const WangId surroundingWangIds[]);
-    WangId wangIdFromSurrounding(const Cell surroundingCells[]) const;
-
     WangId wangIdOfTile(const Tile *tile) const;
     WangId wangIdOfCell(const Cell &cell) const;
 

src/tiled/abstracttilefilltool.cpp

@@ -286,9 +286,9 @@ void AbstractTileFillTool::wangFill(TileLayer &tileLayerToFill,
     if (!mWangSet)
         return;
 
-    WangFiller wangFiller(*mWangSet, mapDocument()->renderer());
+    WangFiller wangFiller(*mWangSet, backgroundTileLayer, mapDocument()->renderer());
     wangFiller.setRegion(region);
-    wangFiller.apply(tileLayerToFill, backgroundTileLayer);
+    wangFiller.apply(tileLayerToFill);
 }
 
 void AbstractTileFillTool::fillWithStamp(Map &map,

src/tiled/randompicker.h

@@ -57,6 +57,11 @@ class RandomPicker
         return mThresholds.isEmpty();
     }
 
+    qsizetype size() const
+    {
+        return mThresholds.size();
+    }
+
     const T &pick() const
     {
         Q_ASSERT(!isEmpty());

src/tiled/stampbrush.cpp

@@ -461,9 +461,10 @@ void StampBrush::drawPreviewLayer(const QVector<QPoint> &points)
             new TileLayer(QString(), bounds.topLeft(), bounds.size())
         };
 
-        WangFiller wangFiller(*mWangSet, mapDocument()->renderer());
+        WangFiller wangFiller(*mWangSet, *tileLayer, mapDocument()->renderer());
+        wangFiller.setErasingEnabled(false);
         wangFiller.setRegion(paintedRegion);
-        wangFiller.apply(*previewLayer, *tileLayer);
+        wangFiller.apply(*previewLayer);
 
         preview->addLayer(std::move(previewLayer));
         preview->addTileset(mWangSet->tileset()->sharedFromThis());

src/tiled/tilelayerwangedit.cpp

@@ -26,6 +26,7 @@
 #include "maprenderer.h"
 #include "scriptmanager.h"
 #include "tilelayer.h"
+#include "wangfiller.h"
 
 #include <QCoreApplication>
 
@@ -38,6 +39,7 @@ TileLayerWangEdit::TileLayerWangEdit(EditableTileLayer *tileLayer, EditableWangS
     , mMap(tileLayer->map()->map()->parameters())
     , mRenderer(MapRenderer::create(&mMap))
     , mWangFiller(std::make_unique<WangFiller>(*wangSet->wangSet(),
+                                               *mTargetLayer->tileLayer(),
                                                mRenderer.get()))
 {
     mTargetLayer->mActiveWangEdits.append(this);
@@ -62,6 +64,16 @@ void TileLayerWangEdit::setCorrectionsEnabled(bool correctionsEnabled)
     mWangFiller->setCorrectionsEnabled(correctionsEnabled);
 }
 
+bool TileLayerWangEdit::erasingEnabled() const
+{
+    return mWangFiller->erasingEnabled();
+}
+
+void TileLayerWangEdit::setErasingEnabled(bool erasingEnabled)
+{
+    mWangFiller->setErasingEnabled(erasingEnabled);
+}
+
 void TileLayerWangEdit::setWangIndex(QPoint pos, WangIndex::Value index, int color)
 {
     mWangFiller->setWangIndex(pos, static_cast<WangId::Index>(index), color);
@@ -90,7 +102,7 @@ void TileLayerWangEdit::setEdge(QPoint pos, WangIndex::Value edge, int color)
 EditableTileLayer *TileLayerWangEdit::generate()
 {
     auto changes = std::make_unique<TileLayer>();
-    mWangFiller->apply(*changes, *mTargetLayer->tileLayer());
+    mWangFiller->apply(*changes);
     return new EditableTileLayer(std::move(changes));
 }
 
@@ -102,7 +114,7 @@ void TileLayerWangEdit::apply()
 
     // Apply terrain changes
     TileLayer changes;
-    mWangFiller->apply(changes, *mTargetLayer->tileLayer());
+    mWangFiller->apply(changes);
     mTargetLayer->applyChangesFrom(&changes, mergeable);
 }
 

src/tiled/tilelayerwangedit.h

@@ -23,7 +23,6 @@
 
 #include "editablewangset.h"
 #include "map.h"
-#include "wangfiller.h"
 
 #include <QObject>
 
@@ -32,6 +31,8 @@
 namespace Tiled {
 
 class EditableTileLayer;
+class MapRenderer;
+class WangFiller;
 
 // Copy of WangId::Index, for exposing the enum to JS
 namespace WangIndex
@@ -63,6 +64,7 @@ class TileLayerWangEdit : public QObject
     Q_PROPERTY(Tiled::EditableWangSet *wangSet READ wangSet CONSTANT)
     Q_PROPERTY(bool mergeable READ isMergeable WRITE setMergeable)
     Q_PROPERTY(bool correctionsEnabled READ correctionsEnabled WRITE setCorrectionsEnabled)
+    Q_PROPERTY(bool erasingEnabled READ erasingEnabled WRITE setErasingEnabled)
 
 public:
     explicit TileLayerWangEdit(EditableTileLayer *tileLayer,
@@ -80,7 +82,10 @@ class TileLayerWangEdit : public QObject
     bool isMergeable() const;
 
     bool correctionsEnabled() const;
-    void setCorrectionsEnabled(bool newCorrectionsEnabled);
+    void setCorrectionsEnabled(bool correctionsEnabled);
+
+    bool erasingEnabled() const;
+    void setErasingEnabled(bool erasingEnabled);
 
     EditableTileLayer *target() const;
     EditableWangSet *wangSet() const;