renepay: accomodate fees in the payment flow

    Min. Cost Flow does not take into account fees when computing a flow
    with liquidity constraints.
    This is a work-around solution that reduces the amount on every route to
    respect the liquidity bound. The deficity in the delivered amount is
    solved by running MCF once again.

    Changes:

    1. the function `flow_complete` allocates amounts to send over the set of routes
       computed by the MCF algorithm, but it does not allocate more than liquidity
       bound of the route. For this reason `minflow` returns a set of routes that
       satisfy the liquidity bounds but it is not guaranteed that the total payment
       reaches the destination therefore there could a deficit in the delivery:
       `deficit = amount_to_deliver - delivering`.

    2. in the function `add_payflows` after `minflow` returns a set of routes we
       call `flows_fit_amount` that tries to a allocate the `deficit` in the routes
       that the MCF have computed.

    3. if the resulting flows pass all payment constraints then we update
        `amount_to_deliver = amount_to_deliver - delivering`, and the loop
        repeats as long as `amount_to_deliver` is not zero.

    In other words, the excess amount, beyond the liquidity bound,
    in the routes is removed and then we try to allocate it
    into known routes, otherwise we do a whole MCF again just for the
    remaining amount.

    Fixes issue #6599

plugins/renepay/flow.h

@@ -262,4 +262,14 @@ size_t commit_flowset(const tal_t *ctx, const struct gossmap *gossmap,
 		    struct chan_extra_map *chan_extra_map, struct flow **flows,
 		    char **fail);
 
+/* flows should be a set of optimal routes delivering an amount that is
+ * slighty less than amount_to_deliver. We will try to reallocate amounts in
+ * these flows so that it delivers the exact amount_to_deliver to the
+ * destination.
+ * Returns how much we are delivering at the end. */
+bool flows_fit_amount(const tal_t *ctx, struct amount_msat *amount_allocated,
+		      struct flow **flows, struct amount_msat amount_to_deliver,
+		      const struct gossmap *gossmap,
+		      struct chan_extra_map *chan_extra_map, char **fail);
+
 #endif /* LIGHTNING_PLUGINS_RENEPAY_FLOW_H */

plugins/renepay/mcf.c

@@ -457,6 +457,11 @@ static bool linearize_channel(const struct pay_parameters *params,
 		return false;
 	}
 
+	assert(
+	    amount_msat_less_eq(extra_half->htlc_total, extra_half->known_max));
+	assert(
+	    amount_msat_less_eq(extra_half->known_min, extra_half->known_max));
+
 	s64 h = extra_half->htlc_total.millisatoshis/1000; /* Raw: linearize_channel */
 	s64 a = extra_half->known_min.millisatoshis/1000, /* Raw: linearize_channel */
 	    b = 1 + extra_half->known_max.millisatoshis/1000; /* Raw: linearize_channel */
@@ -1234,7 +1239,10 @@ get_flow_paths(const tal_t *ctx, const struct gossmap *gossmap,
 
 	       // how many msats in excess we paid for not having msat accuracy
 	       // in the MCF solver
-	       struct amount_msat excess, char **fail)
+	       struct amount_msat excess,
+
+	       // error message
+	       char **fail)
 {
 	tal_t *this_ctx = tal(ctx,tal_t);
 	char *errmsg;
@@ -1386,10 +1394,11 @@ get_flow_paths(const tal_t *ctx, const struct gossmap *gossmap,
 		}
 	}
 
-	/* Stablish ownership. */
-	for(int i=0;i<tal_count(flows);++i)
+	/* Establish ownership. */
+	for(size_t i=0;i<tal_count(flows);++i)
 	{
 		flows[i] = tal_steal(flows,flows[i]);
+		assert(flows[i]);
 	}
 	tal_free(this_ctx);
 	return flows;
@@ -1644,6 +1653,7 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 
 		combine_cost_function(linear_network,residual_network,mu);
 
+		/* We solve a linear MCF problem. */
 		if(!optimize_mcf(this_ctx, dijkstra,linear_network,residual_network,
 				source_idx,target_idx,pay_amount_sats, &errmsg))
 		{
@@ -1655,6 +1665,9 @@ struct flow **minflow(const tal_t *ctx, struct gossmap *gossmap,
 		}
 
 		struct flow **flow_paths;
+		/* We dissect the solution of the MCF into payment routes.
+		 * Actual amounts considering fees are computed for every
+		 * channel in the routes. */
 		flow_paths = get_flow_paths(
 		    this_ctx, params->gossmap, params->chan_extra_map,
 		    linear_network, residual_network, excess, &errmsg);

plugins/renepay/pay_flow.c

@@ -10,6 +10,11 @@
 #include <plugins/renepay/pay.h>
 #include <plugins/renepay/pay_flow.h>
 
+// FIXME These macros are used in more than one place of the code, they could be
+// defined in a single header.
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+
 /* BOLT #7:
  *
  * If a route is computed by simply routing to the intended recipient and summing
@@ -445,18 +450,17 @@ static bool disable_htlc_violations(struct payment *payment,
 	return disabled_some;
 }
 
-const char *add_payflows(const tal_t *ctx,
-			 struct payment *p,
-			 struct amount_msat amount,
-			 struct amount_msat feebudget,
-			 bool is_entire_payment,
+const char *add_payflows(const tal_t *ctx, struct payment *p,
+			 struct amount_msat amount_to_deliver,
+			 struct amount_msat feebudget, bool is_entire_payment,
 			 enum jsonrpc_errcode *ecode)
 {
 	bitmap *disabled;
 	const struct gossmap_node *src, *dst;
-	char *errmsg;
+	char *errmsg, *fail = NULL;
 
-	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap, p->disabled_scids);
+	disabled = make_disabled_bitmap(tmpctx, pay_plugin->gossmap,
+					p->disabled_scids);
 	src = gossmap_find_node(pay_plugin->gossmap, &pay_plugin->my_id);
 	if (!src) {
 		*ecode = PAY_ROUTE_NOT_FOUND;
@@ -465,83 +469,102 @@ const char *add_payflows(const tal_t *ctx,
 	dst = gossmap_find_node(pay_plugin->gossmap, &p->destination);
 	if (!dst) {
 		*ecode = PAY_ROUTE_NOT_FOUND;
-		return tal_fmt(ctx, "Destination is unknown in the network gossip.");
+		return tal_fmt(ctx,
+			       "Destination is unknown in the network gossip.");
 	}
 
-	for (;;) {
-		struct flow **flows;
-		double prob;
-		struct amount_msat fee;
-		u64 delay;
-		bool too_expensive, too_delayed;
-		const u32 *final_cltvs;
-
-		flows = minflow(tmpctx, pay_plugin->gossmap, src, dst,
-				pay_plugin->chan_extra_map, disabled,
-				amount,
-				feebudget,
-				p->min_prob_success ,
-				p->delay_feefactor,
-				p->base_fee_penalty,
-				p->prob_cost_factor,
-				&errmsg);
+	/* probability "bugdet". We will prefer solutions whose probability of
+	 * success is above this value. */
+	double min_prob_success = p->min_prob_success;
+
+	while (!amount_msat_zero(amount_to_deliver)) {
+		struct flow **flows = minflow(
+		    tmpctx, pay_plugin->gossmap, src, dst,
+		    pay_plugin->chan_extra_map, disabled, amount_to_deliver,
+		    feebudget, min_prob_success, p->delay_feefactor,
+		    p->base_fee_penalty, p->prob_cost_factor, &errmsg);
 		if (!flows) {
 			*ecode = PAY_ROUTE_NOT_FOUND;
-			return tal_fmt(ctx,
-				       "minflow couldn't find a feasible flow for %s, %s",
-				       type_to_string(tmpctx,struct amount_msat,&amount),
-				       errmsg);
+
+			/* We fail to allocate a portion of the payment, cleanup
+			 * previous payflows. */
+			// FIXME wouldn't it be better to put these payflows
+			// into a tal ctx with a destructor?
+			fail = tal_fmt(
+			    ctx,
+			    "minflow couldn't find a feasible flow for %s, %s",
+			    type_to_string(tmpctx, struct amount_msat,
+					   &amount_to_deliver),
+			    errmsg);
+			goto function_fail;
+		}
+
+		/* `delivering` could be smaller than `amount_to_deliver`
+		 * because minflow does not count fees when constraining flows.
+		 * Try to redistribute the missing amount among the optimal
+		 * routes. */
+		struct amount_msat delivering;
+
+		if (!flows_fit_amount(tmpctx, &delivering, flows,
+				      amount_to_deliver, pay_plugin->gossmap,
+				      pay_plugin->chan_extra_map, &errmsg)) {
+			fail = tal_fmt(ctx,
+				       "(%s, line %d) flows_fit_amount failed "
+				       "with error: %s",
+				       __PRETTY_FUNCTION__, __LINE__, errmsg);
+			goto function_fail;
 		}
 
 		/* Are we unhappy? */
-		char *fail;
-		prob = flowset_probability(tmpctx, flows, pay_plugin->gossmap,
-					   pay_plugin->chan_extra_map, &fail);
-		if(prob<0)
-		{
+		double prob =
+		    flowset_probability(tmpctx, flows, pay_plugin->gossmap,
+					pay_plugin->chan_extra_map, &errmsg);
+		if (prob < 0) {
 			plugin_err(pay_plugin->plugin,
-				   "flow_set_probability failed: %s", fail);
+				   "flow_set_probability failed: %s", errmsg);
 		}
-		if(!flowset_fee(&fee,flows))
-		{
-			plugin_err(pay_plugin->plugin,
-				   "flowset_fee failed");
+		struct amount_msat fee;
+		if (!flowset_fee(&fee, flows)) {
+			plugin_err(pay_plugin->plugin, "flowset_fee failed");
 		}
-		delay = flows_worst_delay(flows) + p->final_cltv;
+		u64 delay = flows_worst_delay(flows) + p->final_cltv;
 
 		payment_note(p, LOG_INFORM,
-			      "we have computed a set of %ld flows with probability %.3lf, fees %s and delay %ld",
-			      tal_count(flows),
-			      prob,
-			      type_to_string(tmpctx,struct amount_msat,&fee),
-			      delay);
-
-		too_expensive = amount_msat_greater(fee, feebudget);
-		if (too_expensive)
-		{
+			     "we have computed a set of %ld flows with "
+			     "probability %.3lf, fees %s and delay %ld",
+			     tal_count(flows), prob,
+			     type_to_string(tmpctx, struct amount_msat, &fee),
+			     delay);
+
+		if (amount_msat_greater(fee, feebudget)) {
 			*ecode = PAY_ROUTE_TOO_EXPENSIVE;
-			return tal_fmt(ctx,
-				       "Fee exceeds our fee budget, "
-				       "fee = %s (maxfee = %s)",
-				       type_to_string(tmpctx, struct amount_msat, &fee),
-				       type_to_string(tmpctx, struct amount_msat, &feebudget));
+			fail = tal_fmt(
+			    ctx,
+			    "Fee exceeds our fee budget, "
+			    "fee = %s (maxfee = %s)",
+			    type_to_string(tmpctx, struct amount_msat, &fee),
+			    type_to_string(tmpctx, struct amount_msat,
+					   &feebudget));
+			goto function_fail;
 		}
-		too_delayed = (delay > p->maxdelay);
-		if (too_delayed) {
+		if (delay > p->maxdelay) {
 			/* FIXME: What is a sane limit? */
 			if (p->delay_feefactor > 1000) {
 				*ecode = PAY_ROUTE_TOO_EXPENSIVE;
-				return tal_fmt(ctx,
-					       "CLTV delay exceeds our CLTV budget, "
-					       "delay = %"PRIu64" (maxdelay = %u)",
-					       delay, p->maxdelay);
+				fail = tal_fmt(
+				    ctx,
+				    "CLTV delay exceeds our CLTV budget, "
+				    "delay = %" PRIu64 " (maxdelay = %u)",
+				    delay, p->maxdelay);
+				goto function_fail;
 			}
 
 			p->delay_feefactor *= 2;
 			payment_note(p, LOG_INFORM,
-				     "delay %"PRIu64" exceeds our max %u, so doubling delay_feefactor to %f",
-				     delay, p->maxdelay,
-				     p->delay_feefactor);
+				     "delay %" PRIu64
+				     " exceeds our max %u, so doubling "
+				     "delay_feefactor to %f",
+				     delay, p->maxdelay, p->delay_feefactor);
 
 			continue; // retry
 		}
@@ -552,24 +575,57 @@ const char *add_payflows(const tal_t *ctx,
 		 * are far better, since we can report min/max which
 		 * *actually* made us reconsider. */
 		if (disable_htlc_violations(p, flows, pay_plugin->gossmap,
-					    disabled))
-		{
+					    disabled)) {
 			continue; // retry
 		}
 
 		/* This can adjust amounts and final cltv for each flow,
 		 * to make it look like it's going elsewhere */
-		final_cltvs = shadow_additions(tmpctx, pay_plugin->gossmap,
-					       p, flows, is_entire_payment);
+		const u32 *final_cltvs = shadow_additions(
+		    tmpctx, pay_plugin->gossmap, p, flows, is_entire_payment);
 
 		/* OK, we are happy with these flows: convert to
 		 * pay_flows in the current payment, to outlive the
 		 * current gossmap. */
-		convert_and_attach_flows(p, pay_plugin->gossmap,
-					 flows, final_cltvs,
-					 &p->next_partid);
-		return NULL;
+		convert_and_attach_flows(p, pay_plugin->gossmap, flows,
+					 final_cltvs, &p->next_partid);
+		if (prob < 1e-10) {
+			// this last flow probability is too small for division
+			min_prob_success = 1.0;
+		} else {
+			/* prob here is a conditional probability, the next
+			 * round of flows will have a conditional probability
+			 * prob2 and we would like that
+			 *	prob*prob2 >= min_prob_success
+			 * hence min_prob_success/prob becomes the next
+			 * iteration's target. */
+			min_prob_success = MIN(1.0, min_prob_success / prob);
+		}
+		if (!amount_msat_sub(&feebudget, feebudget, fee)) {
+			plugin_err(
+			    pay_plugin->plugin,
+			    "%s: cannot substract feebudget (%s) - fee(%s)",
+			    __PRETTY_FUNCTION__,
+			    fmt_amount_msat(tmpctx, feebudget),
+			    fmt_amount_msat(tmpctx, fee));
+		}
+		if (!amount_msat_sub(&amount_to_deliver, amount_to_deliver,
+				     delivering)) {
+			// If we allow overpayment we might let some bugs
+			// get through.
+			plugin_err(pay_plugin->plugin,
+				   "%s: minflow has produced an overpayment, "
+				   "amount_to_deliver=%s delivering=%s",
+				   __PRETTY_FUNCTION__,
+				   fmt_amount_msat(tmpctx, amount_to_deliver),
+				   fmt_amount_msat(tmpctx, delivering));
+		}
 	}
+	return NULL;
+
+function_fail:
+	payment_remove_flows(p, PAY_FLOW_NOT_STARTED);
+	return fail;
 }
 
 const char *flow_path_to_str(const tal_t *ctx, const struct pay_flow *flow)

plugins/renepay/payment.c

@@ -5,7 +5,6 @@
 #include <common/json_stream.h>
 #include <common/memleak.h>
 #include <plugins/renepay/pay.h>
-#include <plugins/renepay/pay_flow.h>
 #include <plugins/renepay/payment.h>
 
 struct payment *payment_new(const tal_t *ctx,
@@ -458,3 +457,13 @@ void payment_reconsider(struct payment *payment)
 	if (errmsg)
 		payment_fail(payment, ecode, "%s", errmsg);
 }
+
+/* Remove all flows with the given state. */
+void payment_remove_flows(struct payment *p, enum pay_flow_state state)
+{
+	struct pay_flow *pf, *next;
+	list_for_each_safe(&p->flows, pf, next, list) {
+		if(pf->state == state)
+			list_del(&pf->list);
+	}
+}

plugins/renepay/payment.h

@@ -3,6 +3,10 @@
 #include "config.h"
 #include <common/gossmap.h>
 #include <plugins/libplugin.h>
+#include <plugins/renepay/pay_flow.h>
+
+/* FIXME: this shouldn't be here, the dependency tree is a little messed-up. */
+enum pay_flow_state;
 
 struct pay_flow;
 
@@ -173,6 +177,9 @@ void payment_disable_chan(struct payment *p,
 			  enum log_level lvl,
 			  const char *fmt, ...);
 
+/* Remove all flows with the given state. */
+void payment_remove_flows(struct payment *p, enum pay_flow_state state);
+
 struct command_result *payment_fail(
 	struct payment *payment,
 	enum jsonrpc_errcode code,