fix!: for some layers hit_check() missed some blend method influence (#…

…2754)

Example:
 Layer_Bitmap and Layer_SolidColor ignored BLEND_BEHIND completely.

Gradient layers need some investigation
(Conical, Curve, Linear, Radial, Spiral, Noise).
They behave differently, also checking the alpha channel for some blend
methods.

BREAKING CHANGE: synfig API changed:
- add `Layer_Composite::basic_hit_check()` as a helper

synfig-core/src/modules/lyr_std/xorpattern.cpp

@@ -147,22 +147,11 @@ XORPattern::get_param_vocab()const
 Layer::Handle
 XORPattern::hit_check(Context context, const Point &getpos)const
 {
-	// if we have a zero amount
-	if(get_amount()==0.0)
-		// then the click passes down to our context
-		return context.hit_check(getpos);
-
-	Layer::Handle tmp;
-	// if we are behind the context, and the click hits something in the context
-	if(get_blend_method()==Color::BLEND_BEHIND && (tmp=context.hit_check(getpos)))
-		// then return the thing it hit in the context
-		return tmp;
+	bool check_myself_first;
+	auto layer = basic_hit_check(context, getpos, check_myself_first);
 
-	// if we're using an 'onto' blend method and the click missed the context
-	if(Color::is_onto(get_blend_method()) && !(tmp=context.hit_check(getpos)))
-		// then it misses everything
-		return 0;
+	if (!check_myself_first)
+		return layer;
 
-	// otherwise the click hit us, since we're the size of the whole plane
 	return const_cast<XORPattern*>(this);
 }

synfig-core/src/modules/mod_example/metaballs.cpp

@@ -161,16 +161,14 @@ Metaballs::hit_check(synfig::Context context, const synfig::Point &point)const
 {
 	Real density(totaldensity(point));
 
-	if (density <= 0 || density > 1 || get_amount() == 0)
+	if (density <= 0 || density > 1)
 		return context.hit_check(point);
 
-	synfig::Layer::Handle tmp;
+	bool check_myself_first;
+	auto layer = basic_hit_check(context, point, check_myself_first);
 
-	if (get_blend_method()==Color::BLEND_BEHIND && (tmp=context.hit_check(point)))
-		return tmp;
-
-	if (Color::is_onto(get_blend_method()) && !(context.hit_check(point)))
-		return 0;
+	if (!check_myself_first)
+		return layer;
 
 	return const_cast<Metaballs*>(this);
 }

synfig-core/src/modules/mod_geometry/checkerboard.cpp

@@ -294,15 +294,14 @@ CheckerBoard::get_param_vocab()const
 synfig::Layer::Handle
 CheckerBoard::hit_check(synfig::Context context, const synfig::Point &getpos)const
 {
-	if(get_amount()!=0.0 && point_test(getpos))
-	{
-		synfig::Layer::Handle tmp;
-		if(get_blend_method()==Color::BLEND_BEHIND && (tmp=context.hit_check(getpos)))
-			return tmp;
-		if(Color::is_onto(get_blend_method()) && !(tmp=context.hit_check(getpos)))
-			return 0;
+	bool check_myself_first;
+	auto layer = basic_hit_check(context, getpos, check_myself_first);
+
+	if (!check_myself_first)
+		return layer;
+
+	if (point_test(getpos))
 		return const_cast<CheckerBoard*>(this);
-	}
 	else
 		return context.hit_check(getpos);
 }

synfig-core/src/synfig/layers/layer_bitmap.cpp

@@ -190,6 +190,12 @@ Layer_Bitmap::get_param_vocab()const
 synfig::Layer::Handle
 Layer_Bitmap::hit_check(synfig::Context context, const synfig::Point &pos)const
 {
+	bool check_myself_first;
+	auto layer = basic_hit_check(context, pos, check_myself_first);
+
+	if (!check_myself_first)
+		return layer;
+
 	Point tl(param_tl.get(Point()));
 	Point br(param_br.get(Point()));
 	Point surface_pos;

synfig-core/src/synfig/layers/layer_composite.cpp

@@ -76,6 +76,32 @@ Layer_Composite::Layer_Composite(Real a, Color::BlendMethod bm):
 		SET_STATIC_DEFAULTS();
 	}
 
+Layer::Handle
+Layer_Composite::basic_hit_check(synfig::Context context, const synfig::Point &point, bool& check_myself_first) const
+{
+	check_myself_first = false;
+
+	// if we have a zero amount
+	if (get_amount() == 0.0)
+		// then the click passes down to our context
+		return context.hit_check(point);
+
+	Layer::Handle tmp;
+	// if we are behind the context, and the click hits something in the context
+	if (get_blend_method() == Color::BLEND_BEHIND && (tmp = context.hit_check(point)))
+		// then return the thing it hit in the context
+		return tmp;
+
+	// if we're using an 'onto' blend method and the click missed the context
+	if (Color::is_onto(get_blend_method()) && !(tmp = context.hit_check(point)))
+		// then it misses everything
+		return nullptr;
+
+	// otherwise the click may hit us: caller function must check if it does it me or not
+	check_myself_first = true;
+	return nullptr;
+}
+
 Rect
 Layer_Composite::get_full_bounding_rect(Context context)const
 {

synfig-core/src/synfig/layers/layer_composite.h

@@ -69,6 +69,24 @@ class Layer_Composite : public Layer
 	//!Old Straight plus transparent color seems to be the same new than alpha over.
 	bool transparent_color_;
 
+	/**
+	 * Convenient method to help to implement hit_check() for
+	 * Layer_Composite -based layers.
+	 *
+	 * It keep us from falling into the trap some blend methods may set.
+	 *
+	 * It's meant to be called in hit_check() implementations only.
+	 *
+	 * @param context Next layer down in canvas @see Layer::hit_check()
+	 * @param point Point to check if this layer is hittable
+	 * @param[out] check_myself_first If true, the caller layer must check
+	 *              if @a point is inside the layer. If so, return the layer itself.
+	 *              If not, return the context.hit_check()
+	 * @return The handle of the layer under @a point. If there is not
+	 *				a layer under @a point, then returns an empty handle.
+	 */
+	Layer::Handle basic_hit_check(synfig::Context context, const synfig::Point &point, bool& check_myself_first) const;
+
 public:
 	//! Gets the amount of the layer
 	float get_amount()const { return param_amount.get(Real()); }

synfig-core/src/synfig/layers/layer_shape.cpp

@@ -224,6 +224,12 @@ Layer_Shape::is_inside_contour(const Point& p, bool ignore_feather) const
 synfig::Layer::Handle
 Layer_Shape::hit_check(synfig::Context context, const synfig::Point &point) const
 {
+	bool check_myself_first;
+	auto layer = basic_hit_check(context, point, check_myself_first);
+
+	if (!check_myself_first)
+		return layer;
+
 	sync();
 
 	Color::BlendMethod blend_method = get_blend_method();
@@ -234,16 +240,6 @@ Layer_Shape::hit_check(synfig::Context context, const synfig::Point &point) cons
 		inside = is_inside_contour(point, false);
 
 	if (inside) {
-		if (blend_method == Color::BLEND_BEHIND) {
-			synfig::Layer::Handle layer = context.hit_check(point);
-			if (layer) return layer;
-		}
-
-		if (Color::is_onto(blend_method)) {
-			//if there's something in the lower layer then we're set...
-			if (context.hit_check(point))
-				return const_cast<Layer_Shape*>(this);
-		} else
 		if (blend_method == Color::BLEND_ALPHA_OVER) {
 			synfig::info("layer_shape::hit_check - we've got alphaover");
 			//if there's something in the lower layer then we're set...

synfig-core/src/synfig/layers/layer_solidcolor.cpp

@@ -123,15 +123,22 @@ synfig::Layer::Handle
 Layer_SolidColor::hit_check(synfig::Context context, const synfig::Point &point)const
 {
 	Color color=param_color.get(Color());
+
+	bool check_myself_first;
+	auto layer = basic_hit_check(context, point, check_myself_first);
+
+	if (!check_myself_first)
+		return layer;
+
 	if(get_blend_method()==Color::BLEND_STRAIGHT && get_amount()>=0.5)
 		return const_cast<Layer_SolidColor*>(this);
 	else
 	if(get_blend_method()==Color::BLEND_COMPOSITE && get_amount()*color.get_a()>=0.5)
 		return const_cast<Layer_SolidColor*>(this);
 
-	Layer::Handle layer(context.hit_check(point));
+	layer = context.hit_check(point);
 
-	return layer?layer:const_cast<Layer_SolidColor*>(this);
+	return layer ? layer : const_cast<Layer_SolidColor*>(this);
 }
 
 Color