MEDIUM: http: Add a ruleset evaluated on all responses just before fo…

…rwarding

This patch introduces the 'http-after-response' rules. These rules are evaluated
at the end of the response analysis, just before the data forwarding, on ALL
HTTP responses, the server ones but also all responses generated by
HAProxy. Thanks to this ruleset, it is now possible for instance to add some
headers to the responses generated by the stats applet. Following actions are
supported :

   * allow
   * add-header
   * del-header
   * replace-header
   * replace-value
   * set-header
   * set-status
   * set-var
   * strict-mode
   * unset-var

doc/configuration.txt

@@ -2605,6 +2605,7 @@ filter                                    -          X         X         X
 fullconn                                  X          -         X         X
 grace                                     X          X         X         X
 hash-type                                 X          -         X         X
+http-after-response                       -          X         X         X
 http-check disable-on-404                 X          -         X         X
 http-check expect                         -          -         X         X
 http-check send-state                     X          -         X         X
@@ -4189,6 +4190,154 @@ hash-type <method> <function> <modifier>
   See also : "balance", "hash-balance-factor", "server"
 
 
+http-after-response <action> <options...> [ { if | unless } <condition> ]
+  Access control for all Layer 7 responses (server, applet/service and internal
+  ones).
+
+  May be used in sections:   defaults | frontend | listen | backend
+                                no    |    yes   |   yes  |   yes
+
+  The http-after-response statement defines a set of rules which apply to layer
+  7 processing. The rules are evaluated in their declaration order when they
+  are met in a frontend, listen or backend section. Any rule may optionally be
+  followed by an ACL-based condition, in which case it will only be evaluated
+  if the condition is true. Since these rules apply on responses, the backend
+  rules are applied first, followed by the frontend's rules.
+
+  Unlike http-response rules, these ones are applied on all responses, the
+  server ones but also to all responses generated by HAProxy. These rules are
+  evaluated at the end of the responses analysis, before the data forwarding.
+
+  The first keyword is the rule's action. The supported actions are described
+  below.
+
+  There is no limit to the number of http-after-response statements per
+  instance.
+
+  Example:
+    http-after-response set-header Strict-Transport-Security "max-age=31536000"
+    http-after-response set-header Cache-Control "no-store,no-cache,private"
+    http-after-response set-header Pragma "no-cache"
+
+http-after-response add-header <name> <fmt> [ { if | unless } <condition> ]
+
+  This appends an HTTP header field whose name is specified in <name> and whose
+  value is defined by <fmt> which follows the log-format rules (see Custom Log
+  Format in section 8.2.4). This may be used to send a cookie to a client for
+  example, or to pass some internal information.
+  This rule is not final, so it is possible to add other similar rules.
+  Note that header addition is performed immediately, so one rule might reuse
+  the resulting header from a previous rule.
+
+http-after-response allow [ { if | unless } <condition> ]
+
+  This stops the evaluation of the rules and lets the response pass the check.
+  No further "http-after-response" rules are evaluated.
+
+http-after-response del-header <name> [ { if | unless } <condition> ]
+
+  This removes all HTTP header fields whose name is specified in <name>.
+
+http-after-response replace-header <name> <regex-match> <replace-fmt>
+                                   [ { if | unless } <condition> ]
+
+  This works like "http-response replace-header".
+
+  Example:
+    http-after-response replace-header Set-Cookie (C=[^;]*);(.*) \1;ip=%bi;\2
+
+    # applied to:
+    Set-Cookie: C=1; expires=Tue, 14-Jun-2016 01:40:45 GMT
+
+    # outputs:
+    Set-Cookie: C=1;ip=192.168.1.20; expires=Tue, 14-Jun-2016 01:40:45 GMT
+
+    # assuming the backend IP is 192.168.1.20.
+
+http-after-response replace-value <name> <regex-match> <replace-fmt>
+                                 [ { if | unless } <condition> ]
+
+  This works like "http-response replace-value".
+
+  Example:
+    http-after-response replace-value Cache-control ^public$ private
+
+    # applied to:
+    Cache-Control: max-age=3600, public
+
+    # outputs:
+    Cache-Control: max-age=3600, private
+
+http-after-response set-header <name> <fmt> [ { if | unless } <condition> ]
+
+  This does the same as "add-header" except that the header name is first
+  removed if it existed. This is useful when passing security information to
+  the server, where the header must not be manipulated by external users.
+
+http-after-response set-status <status> [reason <str>]
+                               [ { if | unless } <condition> ]
+
+  This replaces the response status code with <status> which must be an integer
+  between 100 and 999. Optionally, a custom reason text can be provided defined
+  by <str>, or the default reason for the specified code will be used as a
+  fallback.
+
+  Example:
+    # return "431 Request Header Fields Too Large"
+    http-response set-status 431
+    # return "503 Slow Down", custom reason
+    http-response set-status 503 reason "Slow Down"
+
+http-after-response set-var(<var-name>) <expr> [ { if | unless } <condition> ]
+
+  This is used to set the contents of a variable. The variable is declared
+  inline.
+
+  Arguments:
+    <var-name>  The name of the variable starts with an indication about its
+                scope. The scopes allowed are:
+                  "proc" : the variable is shared with the whole process
+                  "sess" : the variable is shared with the whole session
+                  "txn"  : the variable is shared with the transaction
+                           (request and response)
+                  "req"  : the variable is shared only during request
+                           processing
+                  "res"  : the variable is shared only during response
+                           processing
+                This prefix is followed by a name. The separator is a '.'.
+                The name may only contain characters 'a-z', 'A-Z', '0-9', '.'
+                and '_'.
+
+    <expr>      Is a standard HAProxy expression formed by a sample-fetch
+                followed by some converters.
+
+  Example:
+    http-after-response set-var(sess.last_redir) res.hdr(location)
+
+http-after-response strict-mode { on | off }
+
+  This enables or disables the strict rewriting mode for following rules. It
+  does not affect rules declared before it and it is only applicable on rules
+  performing a rewrite on the responses. When the strict mode is enabled, any
+  rewrite failure triggers an internal error. Otherwise, such errors are
+  silently ignored. The purpose of the strict rewriting mode is to make some
+  rewrites optionnal while others must be performed to continue the response
+  processing.
+
+  By default, the strict rewriting mode is enabled. Its value is also reset
+  when a ruleset evaluation ends. So, for instance, if you change the mode on
+  the bacnkend, the default mode is restored when HAProxy starts the frontend
+  rules evaluation.
+
+http-after-response unset-var(<var-name>) [ { if | unless } <condition> ]
+
+  This is used to unset a variable. See "http-after-response set-var" for
+  details about <var-name>.
+
+  Example:
+    http-after-response unset-var(sess.last_redir)
+
+
 http-check disable-on-404
   Enable a maintenance mode upon HTTP/404 response to health-checks
   May be used in sections :   defaults | frontend | listen | backend

include/proto/http_ana.h

@@ -40,6 +40,7 @@ int http_process_res_common(struct stream *s, struct channel *rep, int an_bit, s
 int http_request_forward_body(struct stream *s, struct channel *req, int an_bit);
 int http_response_forward_body(struct stream *s, struct channel *res, int an_bit);
 int http_apply_redirect_rule(struct redirect_rule *rule, struct stream *s, struct http_txn *txn);
+int http_eval_after_res_rules(struct stream *s);
 int http_replace_hdrs(struct stream* s, struct htx *htx, struct ist name, const char *str, struct my_regex *re, int full);
 int http_req_replace_stline(int action, const char *replace, int len,
 			    struct proxy *px, struct stream *s);

include/proto/http_rules.h

@@ -29,9 +29,11 @@
 
 extern struct action_kw_list http_req_keywords;
 extern struct action_kw_list http_res_keywords;
+extern struct action_kw_list http_after_res_keywords;
 
 struct act_rule *parse_http_req_cond(const char **args, const char *file, int linenum, struct proxy *proxy);
 struct act_rule *parse_http_res_cond(const char **args, const char *file, int linenum, struct proxy *proxy);
+struct act_rule *parse_http_after_res_cond(const char **args, const char *file, int linenum, struct proxy *proxy);
 struct redirect_rule *http_parse_redirect_rule(const char *file, int linenum, struct proxy *curproxy,
                                                const char **args, char **errmsg, int use_fmt, int dir);
 
@@ -45,6 +47,11 @@ static inline void http_res_keywords_register(struct action_kw_list *kw_list)
 	LIST_ADDQ(&http_res_keywords.list, &kw_list->list);
 }
 
+static inline void http_after_res_keywords_register(struct action_kw_list *kw_list)
+{
+	LIST_ADDQ(&http_after_res_keywords.list, &kw_list->list);
+}
+
 #endif /* _PROTO_HTTP_RULES_H */
 
 /*

include/types/proxy.h

@@ -315,6 +315,7 @@ struct proxy {
 	struct list acl;                        /* ACL declared on this proxy */
 	struct list http_req_rules;		/* HTTP request rules: allow/deny/... */
 	struct list http_res_rules;		/* HTTP response rules: allow/deny/... */
+	struct list http_after_res_rules;	/* HTTP final response rules: set-header/del-header/... */
 	struct list redirect_rules;             /* content redirecting rules (chained) */
 	struct list switching_rules;            /* content switching rules (chained) */
 	struct list persist_rules;		/* 'force-persist' and 'ignore-persist' rules (chained) */

src/cfgparse-listen.c

@@ -1413,6 +1413,36 @@ int cfg_parse_listen(const char *file, int linenum, char **args, int kwm)
 
 		LIST_ADDQ(&curproxy->http_res_rules, &rule->list);
 	}
+	else if (!strcmp(args[0], "http-after-response")) {
+		struct act_rule *rule;
+
+		if (curproxy == &defproxy) {
+			ha_alert("parsing [%s:%d]: '%s' not allowed in 'defaults' section.\n", file, linenum, args[0]);
+			err_code |= ERR_ALERT | ERR_FATAL;
+			goto out;
+		}
+
+		if (!LIST_ISEMPTY(&curproxy->http_after_res_rules) &&
+		    !LIST_PREV(&curproxy->http_after_res_rules, struct act_rule *, list)->cond &&
+		    (LIST_PREV(&curproxy->http_after_res_rules, struct act_rule *, list)->flags & ACT_FLAG_FINAL)) {
+			ha_warning("parsing [%s:%d]: previous '%s' action is final and has no condition attached, further entries are NOOP.\n",
+				   file, linenum, args[0]);
+			err_code |= ERR_WARN;
+		}
+
+		rule = parse_http_after_res_cond((const char **)args + 1, file, linenum, curproxy);
+
+		if (!rule) {
+			err_code |= ERR_ALERT | ERR_ABORT;
+			goto out;
+		}
+
+		err_code |= warnif_cond_conflicts(rule->cond,
+	                                          (curproxy->cap & PR_CAP_BE) ? SMP_VAL_BE_HRS_HDR : SMP_VAL_FE_HRS_HDR,
+	                                          file, linenum);
+
+		LIST_ADDQ(&curproxy->http_after_res_rules, &rule->list);
+	}
 	else if (!strcmp(args[0], "http-send-name-header")) { /* send server name in request header */
 		/* set the header name and length into the proxy structure */
 		if (warnifnotcap(curproxy, PR_CAP_BE, file, linenum, args[0], NULL))

src/cfgparse.c

@@ -2841,6 +2841,16 @@ int check_config_validity()
 			}
 		}
 
+		/* check validity for 'http-after-response' layer 7 rules */
+		list_for_each_entry(arule, &curproxy->http_after_res_rules, list) {
+			err = NULL;
+			if (arule->check_ptr && !arule->check_ptr(arule, curproxy, &err)) {
+				ha_alert("Proxy '%s': %s.\n", curproxy->id, err);
+				free(err);
+				cfgerr++;
+			}
+		}
+
 		if (curproxy->table && curproxy->table->peers.name) {
 			struct peers *curpeers;
 

src/haproxy.c

@@ -2440,6 +2440,7 @@ void deinit(void)
 		deinit_act_rules(&p->tcp_req.l5_rules);
 		deinit_act_rules(&p->http_req_rules);
 		deinit_act_rules(&p->http_res_rules);
+		deinit_act_rules(&p->http_after_res_rules);
 
 		deinit_stick_rules(&p->storersp_rules);
 		deinit_stick_rules(&p->sticking_rules);

src/http_act.c

@@ -1875,6 +1875,22 @@ static struct action_kw_list http_res_actions = {
 
 INITCALL1(STG_REGISTER, http_res_keywords_register, &http_res_actions);
 
+static struct action_kw_list http_after_res_actions = {
+	.kw = {
+		{ "add-header",      parse_http_set_header,     0 },
+		{ "allow",           parse_http_allow,          0 },
+		{ "del-header",      parse_http_del_header,     0 },
+		{ "replace-header",  parse_http_replace_header, 0 },
+		{ "replace-value",   parse_http_replace_header, 0 },
+		{ "set-header",      parse_http_set_header,     0 },
+		{ "set-status",      parse_http_set_status,     0 },
+		{ "strict-mode",     parse_http_strict_mode,    0 },
+		{ NULL, NULL }
+	}
+};
+
+INITCALL1(STG_REGISTER, http_after_res_keywords_register, &http_after_res_actions);
+
 /*
  * Local variables:
  *  c-indent-level: 8

src/http_ana.c

@@ -1986,15 +1986,6 @@ int http_process_res_common(struct stream *s, struct channel *rep, int an_bit, s
 		cur_proxy = sess->fe;
 	}
 
-	/* After this point, this anayzer can't return yield, so we can
-	 * remove the bit corresponding to this analyzer from the list.
-	 *
-	 * Note that the intermediate returns and goto found previously
-	 * reset the analyzers.
-	 */
-	rep->analysers &= ~an_bit;
-	rep->analyse_exp = TICK_ETERNITY;
-
 	/* OK that's all we can do for 1xx responses */
 	if (unlikely(txn->status < 200 && txn->status != 101))
 		goto end;
@@ -2116,6 +2107,14 @@ int http_process_res_common(struct stream *s, struct channel *rep, int an_bit, s
 	}
 
   end:
+	/*
+	 * Evaluate after-response rules before forwarding the response. rules
+	 * from the backend are evaluated first, then one from the frontend if
+	 * it differs.
+	 */
+	if (!http_eval_after_res_rules(s))
+		goto return_int_err;
+
 	/* Always enter in the body analyzer */
 	rep->analysers &= ~AN_RES_FLT_XFER_DATA;
 	rep->analysers |= AN_RES_HTTP_XFER_BODY;
@@ -2130,10 +2129,9 @@ int http_process_res_common(struct stream *s, struct channel *rep, int an_bit, s
 		s->do_log(s);
 		s->logs.bytes_out = 0;
 	}
-	DBG_TRACE_LEAVE(STRM_EV_STRM_ANA|STRM_EV_HTTP_ANA, s, txn);
-	return 1;
 
  done:
+	DBG_TRACE_LEAVE(STRM_EV_STRM_ANA|STRM_EV_HTTP_ANA, s, txn);
 	rep->analysers &= ~an_bit;
 	rep->analyse_exp = TICK_ETERNITY;
 	return 1;
@@ -3120,6 +3118,31 @@ static enum rule_result http_res_get_intercept_rule(struct proxy *px, struct lis
 	return rule_ret;
 }
 
+/* Executes backend and frontend http-after-response rules for the stream <s>,
+ * in that order. it return 1 on success and 0 on error. It is the caller
+ * responsibility to catch error or ignore it. If it catches it, this function
+ * may be called a second time, for the internal error.
+ */
+int http_eval_after_res_rules(struct stream *s)
+{
+	struct session *sess = s->sess;
+	enum rule_result ret = HTTP_RULE_RES_CONT;
+
+	/* prune the request variables if not already done and swap to the response variables. */
+	if (s->vars_reqres.scope != SCOPE_RES) {
+		if (!LIST_ISEMPTY(&s->vars_reqres.head))
+			vars_prune(&s->vars_reqres, s->sess, s);
+		vars_init(&s->vars_reqres, SCOPE_RES);
+	}
+
+	ret = http_res_get_intercept_rule(s->be, &s->be->http_after_res_rules, s);
+	if ((ret == HTTP_RULE_RES_CONT || ret == HTTP_RULE_RES_STOP) && sess->fe != s->be)
+		ret = http_res_get_intercept_rule(sess->fe, &sess->fe->http_after_res_rules, s);
+
+	/* All other codes than CONTINUE, STOP or DONE are forbidden */
+	return (ret == HTTP_RULE_RES_CONT || ret == HTTP_RULE_RES_STOP || ret == HTTP_RULE_RES_DONE);
+}
+
 /*
  * Manage client-side cookie. It can impact performance by about 2% so it is
  * desirable to call it only when needed. This code is quite complex because
@@ -4534,6 +4557,8 @@ int http_forward_proxy_resp(struct stream *s, int final)
 
 	if (final) {
 		htx->flags |= HTX_FL_PROXY_RESP;
+		if (!http_eval_after_res_rules(s))
+			return 0;
 
 		channel_auto_read(req);
 		channel_abort(req);

src/http_htx.c

@@ -1308,7 +1308,7 @@ static int post_check_errors()
 		if (htx_free_data_space(htx) < global.tune.maxrewrite) {
 			ha_warning("config: errorfile '%s' runs over the buffer space"
 				   " reserved to headers rewritting. It may lead to internal errors if "
-				   " http-final-response rules are evaluated on this message.\n",
+				   " http-after-response rules are evaluated on this message.\n",
 				   (char *)node->key);
 			err_code |= ERR_WARN;
 		}