Drop selection when we hide a layer that is being transformed (fix as…

…eprite#4179, fix aseprite#3254)

This fixes several problems in MovingPixelsState where hidden layers
were transformed anyway when we switched the visibility of a layer in
this state.

Other fix was tried before in aseprite#3254 but we needed the onBefore/After
layer visibility change notifications to make this work properly
(i.e. drop pixels when the visiblity of a layer is changed).

The only drawback at this moment is that changing the visibility of
the non-active layer when we are transforming multiple cels/timeline
range can be confused because we don't have aseprite#2144/aseprite#2865 implemented
yet.

This bug was originally reported here: https://community.aseprite.org/t/20621

src/app/commands/cmd_layer_visibility.cpp

@@ -1,4 +1,5 @@
 // Aseprite
+// Copyright (C) 2024  Igara Studio S.A.
 // Copyright (C) 2001-2017  David Capello
 //
 // This program is distributed under the terms of
@@ -8,7 +9,6 @@
 #include "config.h"
 #endif
 
-#include "app/app.h"
 #include "app/commands/command.h"
 #include "app/context_access.h"
 #include "app/modules/gui.h"
@@ -49,7 +49,7 @@ bool LayerVisibilityCommand::onChecked(Context* context)
     return false;
 
   SelectedLayers selLayers;
-  auto range = App::instance()->timeline()->range();
+  DocRange range = context->activeSite().range();
   if (range.enabled()) {
     selLayers = range.selectedLayers();
   }
@@ -67,24 +67,25 @@ bool LayerVisibilityCommand::onChecked(Context* context)
 void LayerVisibilityCommand::onExecute(Context* context)
 {
   ContextWriter writer(context);
+  Doc* doc = writer.document();
   SelectedLayers selLayers;
-  auto range = App::instance()->timeline()->range();
+  DocRange range = context->activeSite().range();
   if (range.enabled()) {
     selLayers = range.selectedLayers();
   }
   else {
     selLayers.insert(writer.layer());
   }
+
   bool anyVisible = false;
   for (auto layer : selLayers) {
     if (layer->isVisible())
       anyVisible = true;
   }
-  for (auto layer : selLayers) {
-    layer->setVisible(!anyVisible);
-  }
 
-  update_screen_for_document(writer.document());
+  const bool newState = !anyVisible;
+  for (auto layer : selLayers)
+    doc->setLayerVisibilityWithNotifications(layer, newState);
 }
 
 Command* CommandFactory::createLayerVisibilityCommand()

src/app/doc.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -193,6 +193,16 @@ color_t Doc::bgColor(Layer* layer) const
     return layer->sprite()->transparentColor();
 }
 
+//////////////////////////////////////////////////////////////////////
+// Modifications with notifications
+
+void Doc::setLayerVisibilityWithNotifications(Layer* layer, const bool visible)
+{
+  notifyBeforeLayerVisibilityChange(layer, visible);
+  layer->setVisible(visible);
+  notifyAfterLayerVisibilityChange(layer);
+}
+
 //////////////////////////////////////////////////////////////////////
 // Notifications
 
@@ -244,6 +254,20 @@ void Doc::notifyLayerMergedDown(Layer* srcLayer, Layer* targetLayer)
   notify_observers<DocEvent&>(&DocObserver::onLayerMergedDown, ev);
 }
 
+void Doc::notifyBeforeLayerVisibilityChange(Layer* layer, bool newState)
+{
+  DocEvent ev(this);
+  ev.layer(layer);
+  notify_observers<DocEvent&>(&DocObserver::onBeforeLayerVisibilityChange, ev, newState);
+}
+
+void Doc::notifyAfterLayerVisibilityChange(Layer* layer)
+{
+  DocEvent ev(this);
+  ev.layer(layer);
+  notify_observers<DocEvent&>(&DocObserver::onAfterLayerVisibilityChange, ev);
+}
+
 void Doc::notifyCelMoved(Layer* fromLayer, frame_t fromFrame, Layer* toLayer, frame_t toFrame)
 {
   DocEvent ev(this);

src/app/doc.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -107,6 +107,14 @@ namespace app {
 
     os::ColorSpaceRef osColorSpace() const { return m_osColorSpace; }
 
+    //////////////////////////////////////////////////////////////////////
+    // Modifications with notifications
+
+    // Use this function to change the layer visibility and notify all
+    // DocObservers about this change (e.g. so the Editor can be
+    // invalidated/redrawn, MovingPixelsState can drop pixels, etc.)
+    void setLayerVisibilityWithNotifications(Layer* layer, const bool visible);
+
     //////////////////////////////////////////////////////////////////////
     // Notifications
 
@@ -116,6 +124,8 @@ namespace app {
     void notifySpritePixelsModified(Sprite* sprite, const gfx::Region& region, frame_t frame);
     void notifyExposeSpritePixels(Sprite* sprite, const gfx::Region& region);
     void notifyLayerMergedDown(Layer* srcLayer, Layer* targetLayer);
+    void notifyBeforeLayerVisibilityChange(Layer* layer, bool newState);
+    void notifyAfterLayerVisibilityChange(Layer* layer);
     void notifyCelMoved(Layer* fromLayer, frame_t fromFrame, Layer* toLayer, frame_t toFrame);
     void notifyCelCopied(Layer* fromLayer, frame_t fromFrame, Layer* toLayer, frame_t toFrame);
     void notifySelectionChanged();

src/app/doc_observer.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -97,6 +97,10 @@ namespace app {
     // The collapsed/expanded flag of a specific layer changed.
     virtual void onLayerCollapsedChanged(DocEvent& ev) { }
 
+    // The visibility flag of a specific layer is going to change/changed.
+    virtual void onBeforeLayerVisibilityChange(DocEvent& ev, bool newState) { }
+    virtual void onAfterLayerVisibilityChange(DocEvent& ev) { }
+
     // The tileset was remapped (e.g. when tiles are re-ordered).
     virtual void onRemapTileset(DocEvent& ev, const doc::Remap& remap) { }
 

src/app/script/layer_class.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2018  David Capello
 //
 // This program is distributed under the terms of
@@ -329,7 +329,9 @@ int Layer_set_isEditable(lua_State* L)
 int Layer_set_isVisible(lua_State* L)
 {
   auto layer = get_docobj<Layer>(L, 1);
-  layer->setVisible(lua_toboolean(L, 2));
+  const bool newState = lua_toboolean(L, 2);
+  Doc* doc = static_cast<Doc*>(layer->sprite()->document());
+  doc->setLayerVisibilityWithNotifications(layer, newState);
   return 0;
 }
 

src/app/ui/doc_view.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -478,7 +478,16 @@ void DocView::onTotalFramesChanged(DocEvent& ev)
 
 void DocView::onLayerRestacked(DocEvent& ev)
 {
-  m_editor->invalidate();
+  if (hasContentInActiveFrame(ev.layer()))
+    m_editor->invalidate();
+}
+
+void DocView::onAfterLayerVisibilityChange(DocEvent& ev)
+{
+  // If there is no cel for this layer in the current frame, there is
+  // no need to redraw the editor
+  if (hasContentInActiveFrame(ev.layer()))
+    m_editor->invalidate();
 }
 
 void DocView::onTilesetChanged(DocEvent& ev)
@@ -653,4 +662,9 @@ void DocView::onCancel(Context* ctx)
   }
 }
 
+bool DocView::hasContentInActiveFrame(const doc::Layer* layer) const
+{
+  return (layer && layer->cel(m_editor->frame()));
+}
+
 } // namespace app

src/app/ui/doc_view.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2023  Igara Studio S.A.
+// Copyright (C) 2019-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -15,6 +15,10 @@
 #include "app/ui/workspace_view.h"
 #include "ui/box.h"
 
+namespace doc {
+  class Layer;
+}
+
 namespace ui {
   class View;
 }
@@ -86,6 +90,7 @@ namespace app {
     void onAfterRemoveCel(DocEvent& ev) override;
     void onTotalFramesChanged(DocEvent& ev) override;
     void onLayerRestacked(DocEvent& ev) override;
+    void onAfterLayerVisibilityChange(DocEvent& ev) override;
     void onTilesetChanged(DocEvent& ev) override;
 
     // InputChainElement impl
@@ -105,6 +110,8 @@ namespace app {
     bool onProcessMessage(ui::Message* msg) override;
 
   private:
+    bool hasContentInActiveFrame(const doc::Layer* layer) const;
+
     Type m_type;
     Doc* m_document;
     ui::View* m_view;

src/app/ui/editor/editor.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -2438,6 +2438,12 @@ void Editor::onRemoveSlice(DocEvent& ev)
   }
 }
 
+void Editor::onBeforeLayerVisibilityChange(DocEvent& ev, bool newState)
+{
+  if (m_state)
+    m_state->onBeforeLayerVisibilityChange(this, ev.layer(), newState);
+}
+
 void Editor::setCursor(const gfx::Point& mouseDisplayPos)
 {
   Rect vp = View::getView(this)->viewportBounds();

src/app/ui/editor/editor.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018-2023  Igara Studio S.A.
+// Copyright (C) 2018-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -351,6 +351,7 @@ namespace app {
     void onAddTag(DocEvent& ev) override;
     void onRemoveTag(DocEvent& ev) override;
     void onRemoveSlice(DocEvent& ev) override;
+    void onBeforeLayerVisibilityChange(DocEvent& ev, bool newState) override;
 
     // ActiveToolObserver impl
     void onActiveToolChange(tools::Tool* tool) override;

src/app/ui/editor/editor_state.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2023  Igara Studio S.A.
+// Copyright (C) 2019-2024  Igara Studio S.A.
 // Copyright (C) 2001-2016  David Capello
 //
 // This program is distributed under the terms of
@@ -26,6 +26,7 @@ namespace ui {
 }
 
 namespace doc {
+  class Layer;
   class Tag;
 }
 
@@ -145,6 +146,11 @@ namespace app {
     // collection.
     virtual void onBeforeRemoveLayer(Editor* editor) { }
 
+    // Called when the visibility of a specific layer is changed.
+    virtual void onBeforeLayerVisibilityChange(Editor* editor,
+                                               doc::Layer* layer,
+                                               bool newState) { }
+
   private:
     DISABLE_COPYING(EditorState);
   };

src/app/ui/editor/moving_pixels_state.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2023  Igara Studio S.A.
+// Copyright (C) 2019-2024  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -578,6 +578,25 @@ bool MovingPixelsState::acceptQuickTool(tools::Tool* tool)
      tool->getInk(0)->isZoom());
 }
 
+void MovingPixelsState::onBeforeLayerVisibilityChange(Editor* editor,
+                                                      doc::Layer* layer,
+                                                      bool newState)
+{
+  if (!isActiveDocument())
+    return;
+
+  // If the layer visibility of any selected layer changes, we just
+  // drop the pixels (it's the easiest way to avoid modifying hidden
+  // pixels).
+  if (m_pixelsMovement) {
+    const Site& site = m_pixelsMovement->site();
+    if (site.layer() == layer ||
+        site.range().contains(layer)) {
+      dropPixels();
+    }
+  }
+}
+
 // Before executing any command, we drop the pixels (go back to standby).
 void MovingPixelsState::onBeforeCommandExecution(CommandExecutionEvent& ev)
 {