Sync with 2.4.10

Documentation/RelNotes/2.3.10.txt

@@ -0,0 +1,18 @@
+Git v2.3.10 Release Notes
+=========================
+
+Fixes since v2.3.9
+------------------
+
+ * xdiff code we use to generate diffs is not prepared to handle
+   extremely large files.  It uses "int" in many places, which can
+   overflow if we have a very large number of lines or even bytes in
+   our input files, for example.  Cap the input size to soemwhere
+   around 1GB for now.
+
+ * Some protocols (like git-remote-ext) can execute arbitrary code
+   found in the URL.  The URLs that submodules use may come from
+   arbitrary sources (e.g., .gitmodules files in a remote
+   repository), and can hurt those who blindly enable recursive
+   fetch.  Restrict the allowed protocols to well known and safe
+   ones.

Documentation/RelNotes/2.4.10.txt

@@ -0,0 +1,18 @@
+Git v2.4.10 Release Notes
+=========================
+
+Fixes since v2.4.9
+------------------
+
+ * xdiff code we use to generate diffs is not prepared to handle
+   extremely large files.  It uses "int" in many places, which can
+   overflow if we have a very large number of lines or even bytes in
+   our input files, for example.  Cap the input size to soemwhere
+   around 1GB for now.
+
+ * Some protocols (like git-remote-ext) can execute arbitrary code
+   found in the URL.  The URLs that submodules use may come from
+   arbitrary sources (e.g., .gitmodules files in a remote
+   repository), and can hurt those who blindly enable recursive
+   fetch.  Restrict the allowed protocols to well known and safe
+   ones.

Documentation/git.txt

@@ -51,9 +51,10 @@ Documentation for older releases are available here:
   link:RelNotes/2.5.1.txt[2.5.1],
   link:RelNotes/2.5.0.txt[2.5].
 
-* link:v2.4.9/git.html[documentation for release 2.4.9]
+* link:v2.4.10/git.html[documentation for release 2.4.10]
 
 * release notes for
+  link:RelNotes/2.4.10.txt[2.4.10],
   link:RelNotes/2.4.9.txt[2.4.9],
   link:RelNotes/2.4.8.txt[2.4.8],
   link:RelNotes/2.4.7.txt[2.4.7],
@@ -65,9 +66,10 @@ Documentation for older releases are available here:
   link:RelNotes/2.4.1.txt[2.4.1],
   link:RelNotes/2.4.0.txt[2.4].
 
-* link:v2.3.9/git.html[documentation for release 2.3.9]
+* link:v2.3.10/git.html[documentation for release 2.3.10]
 
 * release notes for
+  link:RelNotes/2.3.10.txt[2.3.10],
   link:RelNotes/2.3.9.txt[2.3.9],
   link:RelNotes/2.3.8.txt[2.3.8],
   link:RelNotes/2.3.7.txt[2.3.7],
@@ -1076,6 +1078,33 @@ GIT_ICASE_PATHSPECS::
 	an operation has touched every ref (e.g., because you are
 	cloning a repository to make a backup).
 
+`GIT_ALLOW_PROTOCOL`::
+	If set, provide a colon-separated list of protocols which are
+	allowed to be used with fetch/push/clone. This is useful to
+	restrict recursive submodule initialization from an untrusted
+	repository. Any protocol not mentioned will be disallowed (i.e.,
+	this is a whitelist, not a blacklist). If the variable is not
+	set at all, all protocols are enabled.  The protocol names
+	currently used by git are:
+
+	  - `file`: any local file-based path (including `file://` URLs,
+	    or local paths)
+
+	  - `git`: the anonymous git protocol over a direct TCP
+	    connection (or proxy, if configured)
+
+	  - `ssh`: git over ssh (including `host:path` syntax,
+	    `git+ssh://`, etc).
+
+	  - `rsync`: git over rsync
+
+	  - `http`: git over http, both "smart http" and "dumb http".
+	    Note that this does _not_ include `https`; if you want both,
+	    you should specify both as `http:https`.
+
+	  - any external helpers are named by their protocol (e.g., use
+	    `hg` to allow the `git-remote-hg` helper)
+
 
 Discussion[[Discussion]]
 ------------------------

builtin/blame.c

@@ -973,7 +973,10 @@ static void pass_blame_to_parent(struct scoreboard *sb,
 	fill_origin_blob(&sb->revs->diffopt, target, &file_o);
 	num_get_patch++;
 
-	diff_hunks(&file_p, &file_o, 0, blame_chunk_cb, &d);
+	if (diff_hunks(&file_p, &file_o, 0, blame_chunk_cb, &d))
+		die("unable to generate diff (%s -> %s)",
+		    sha1_to_hex(parent->commit->object.sha1),
+		    sha1_to_hex(target->commit->object.sha1));
 	/* The rest are the same as the parent */
 	blame_chunk(&d.dstq, &d.srcq, INT_MAX, d.offset, INT_MAX, parent);
 	*d.dstq = NULL;
@@ -1119,7 +1122,9 @@ static void find_copy_in_blob(struct scoreboard *sb,
 	 * file_p partially may match that image.
 	 */
 	memset(split, 0, sizeof(struct blame_entry [3]));
-	diff_hunks(file_p, &file_o, 1, handle_split_cb, &d);
+	if (diff_hunks(file_p, &file_o, 1, handle_split_cb, &d))
+		die("unable to generate diff (%s)",
+		    sha1_to_hex(parent->commit->object.sha1));
 	/* remainder, if any, all match the preimage */
 	handle_split(sb, ent, d.tlno, d.plno, ent->num_lines, parent, split);
 }

builtin/merge-file.c

@@ -75,7 +75,8 @@ int cmd_merge_file(int argc, const char **argv, const char *prefix)
 			names[i] = argv[i];
 		if (read_mmfile(mmfs + i, fname))
 			return -1;
-		if (buffer_is_binary(mmfs[i].ptr, mmfs[i].size))
+		if (mmfs[i].size > MAX_XDIFF_SIZE ||
+		    buffer_is_binary(mmfs[i].ptr, mmfs[i].size))
 			return error("Cannot merge binary files: %s",
 					argv[i]);
 	}

builtin/merge-tree.c

@@ -118,7 +118,8 @@ static void show_diff(struct merge_list *entry)
 	if (!dst.ptr)
 		size = 0;
 	dst.size = size;
-	xdi_diff(&src, &dst, &xpp, &xecfg, &ecb);
+	if (xdi_diff(&src, &dst, &xpp, &xecfg, &ecb))
+		die("unable to generate diff");
 	free(src.ptr);
 	free(dst.ptr);
 }

builtin/rerere.c

@@ -29,9 +29,10 @@ static int diff_two(const char *file1, const char *label1,
 	xdemitconf_t xecfg;
 	xdemitcb_t ecb;
 	mmfile_t minus, plus;
+	int ret;
 
 	if (read_mmfile(&minus, file1) || read_mmfile(&plus, file2))
-		return 1;
+		return -1;
 
 	printf("--- a/%s\n+++ b/%s\n", label1, label2);
 	fflush(stdout);
@@ -40,11 +41,11 @@ static int diff_two(const char *file1, const char *label1,
 	memset(&xecfg, 0, sizeof(xecfg));
 	xecfg.ctxlen = 3;
 	ecb.outf = outf;
-	xdi_diff(&minus, &plus, &xpp, &xecfg, &ecb);
+	ret = xdi_diff(&minus, &plus, &xpp, &xecfg, &ecb);
 
 	free(minus.ptr);
 	free(plus.ptr);
-	return 0;
+	return ret;
 }
 
 int cmd_rerere(int argc, const char **argv, const char *prefix)
@@ -104,7 +105,8 @@ int cmd_rerere(int argc, const char **argv, const char *prefix)
 		for (i = 0; i < merge_rr.nr; i++) {
 			const char *path = merge_rr.items[i].string;
 			const char *name = (const char *)merge_rr.items[i].util;
-			diff_two(rerere_path(name, "preimage"), path, path, path);
+			if (diff_two(rerere_path(name, "preimage"), path, path, path))
+				die("unable to generate diff for %s", name);
 		}
 	else
 		usage_with_options(rerere_usage, options);

combine-diff.c

@@ -419,8 +419,10 @@ static void combine_diff(const struct object_id *parent, unsigned int mode,
 	state.num_parent = num_parent;
 	state.n = n;
 
-	xdi_diff_outf(&parent_file, result_file, consume_line, &state,
-		      &xpp, &xecfg);
+	if (xdi_diff_outf(&parent_file, result_file, consume_line, &state,
+			  &xpp, &xecfg))
+		die("unable to generate combined diff for %s",
+		    oid_to_hex(parent));
 	free(parent_file.ptr);
 
 	/* Assign line numbers for this parent.

connect.c

@@ -9,6 +9,7 @@
 #include "url.h"
 #include "string-list.h"
 #include "sha1-array.h"
+#include "transport.h"
 
 static char *server_capabilities;
 static const char *parse_feature_value(const char *, const char *, int *);
@@ -694,6 +695,8 @@ struct child_process *git_connect(int fd[2], const char *url,
 		else
 			target_host = xstrdup(hostandport);
 
+		transport_check_allowed("git");
+
 		/* These underlying connection commands die() if they
 		 * cannot connect.
 		 */
@@ -727,6 +730,7 @@ struct child_process *git_connect(int fd[2], const char *url,
 			int putty, tortoiseplink = 0;
 			char *ssh_host = hostandport;
 			const char *port = NULL;
+			transport_check_allowed("ssh");
 			get_host_and_port(&ssh_host, &port);
 
 			if (!port)
@@ -781,6 +785,7 @@ struct child_process *git_connect(int fd[2], const char *url,
 			/* remove repo-local variables from the environment */
 			conn->env = local_repo_env;
 			conn->use_shell = 1;
+			transport_check_allowed("file");
 		}
 		argv_array_push(&conn->args, cmd.buf);
 

diff.c

@@ -1033,8 +1033,9 @@ static void diff_words_show(struct diff_words_data *diff_words)
 	xpp.flags = 0;
 	/* as only the hunk header will be parsed, we need a 0-context */
 	xecfg.ctxlen = 0;
-	xdi_diff_outf(&minus, &plus, fn_out_diff_words_aux, diff_words,
-		      &xpp, &xecfg);
+	if (xdi_diff_outf(&minus, &plus, fn_out_diff_words_aux, diff_words,
+			  &xpp, &xecfg))
+		die("unable to generate word diff");
 	free(minus.ptr);
 	free(plus.ptr);
 	if (diff_words->current_plus != diff_words->plus.text.ptr +
@@ -2441,8 +2442,9 @@ static void builtin_diff(const char *name_a,
 			xecfg.ctxlen = strtoul(v, NULL, 10);
 		if (o->word_diff)
 			init_diff_words_data(&ecbdata, o, one, two);
-		xdi_diff_outf(&mf1, &mf2, fn_out_consume, &ecbdata,
-			      &xpp, &xecfg);
+		if (xdi_diff_outf(&mf1, &mf2, fn_out_consume, &ecbdata,
+				  &xpp, &xecfg))
+			die("unable to generate diff for %s", one->path);
 		if (o->word_diff)
 			free_diff_words_data(&ecbdata);
 		if (textconv_one)
@@ -2519,8 +2521,9 @@ static void builtin_diffstat(const char *name_a, const char *name_b,
 		xpp.flags = o->xdl_opts;
 		xecfg.ctxlen = o->context;
 		xecfg.interhunkctxlen = o->interhunkcontext;
-		xdi_diff_outf(&mf1, &mf2, diffstat_consume, diffstat,
-			      &xpp, &xecfg);
+		if (xdi_diff_outf(&mf1, &mf2, diffstat_consume, diffstat,
+				  &xpp, &xecfg))
+			die("unable to generate diffstat for %s", one->path);
 	}
 
 	diff_free_filespec_data(one);
@@ -2566,8 +2569,9 @@ static void builtin_checkdiff(const char *name_a, const char *name_b,
 		memset(&xecfg, 0, sizeof(xecfg));
 		xecfg.ctxlen = 1; /* at least one context line */
 		xpp.flags = 0;
-		xdi_diff_outf(&mf1, &mf2, checkdiff_consume, &data,
-			      &xpp, &xecfg);
+		if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume, &data,
+				  &xpp, &xecfg))
+			die("unable to generate checkdiff for %s", one->path);
 
 		if (data.ws_rule & WS_BLANK_AT_EOF) {
 			struct emit_callback ecbdata;
@@ -4508,8 +4512,10 @@ static int diff_get_patch_id(struct diff_options *options, unsigned char *sha1)
 		xpp.flags = 0;
 		xecfg.ctxlen = 3;
 		xecfg.flags = 0;
-		xdi_diff_outf(&mf1, &mf2, patch_id_consume, &data,
-			      &xpp, &xecfg);
+		if (xdi_diff_outf(&mf1, &mf2, patch_id_consume, &data,
+				  &xpp, &xecfg))
+			return error("unable to generate patch-id diff for %s",
+				     p->one->path);
 	}
 
 	git_SHA1_Final(sha1, &ctx);

diffcore-pickaxe.c

@@ -62,8 +62,8 @@ static int diff_grep(mmfile_t *one, mmfile_t *two,
 	ecbdata.hit = 0;
 	xecfg.ctxlen = o->context;
 	xecfg.interhunkctxlen = o->interhunkcontext;
-	xdi_diff_outf(one, two, diffgrep_consume, &ecbdata,
-		      &xpp, &xecfg);
+	if (xdi_diff_outf(one, two, diffgrep_consume, &ecbdata, &xpp, &xecfg))
+		return 0;
 	return ecbdata.hit;
 }
 

git-submodule.sh

@@ -22,6 +22,15 @@ require_work_tree
 wt_prefix=$(git rev-parse --show-prefix)
 cd_to_toplevel
 
+# Restrict ourselves to a vanilla subset of protocols; the URLs
+# we get are under control of a remote repository, and we do not
+# want them kicking off arbitrary git-remote-* programs.
+#
+# If the user has already specified a set of allowed protocols,
+# we assume they know what they're doing and use that instead.
+: ${GIT_ALLOW_PROTOCOL=file:git:http:https:ssh}
+export GIT_ALLOW_PROTOCOL
+
 command=
 branch=
 force=

http.c

@@ -9,6 +9,7 @@
 #include "version.h"
 #include "pkt-line.h"
 #include "gettext.h"
+#include "transport.h"
 
 int active_requests;
 int http_is_verbose;
@@ -340,6 +341,7 @@ static void set_curl_keepalive(CURL *c)
 static CURL *get_curl_handle(void)
 {
 	CURL *result = curl_easy_init();
+	long allowed_protocols = 0;
 
 	if (!result)
 		die("curl_easy_init failed");
@@ -394,11 +396,27 @@ static CURL *get_curl_handle(void)
 	}
 
 	curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
+	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
 #if LIBCURL_VERSION_NUM >= 0x071301
 	curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
 #elif LIBCURL_VERSION_NUM >= 0x071101
 	curl_easy_setopt(result, CURLOPT_POST301, 1);
 #endif
+#if LIBCURL_VERSION_NUM >= 0x071304
+	if (is_transport_allowed("http"))
+		allowed_protocols |= CURLPROTO_HTTP;
+	if (is_transport_allowed("https"))
+		allowed_protocols |= CURLPROTO_HTTPS;
+	if (is_transport_allowed("ftp"))
+		allowed_protocols |= CURLPROTO_FTP;
+	if (is_transport_allowed("ftps"))
+		allowed_protocols |= CURLPROTO_FTPS;
+	curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS, allowed_protocols);
+#else
+	if (transport_restrict_protocols())
+		warning("protocol restrictions not applied to curl redirects because\n"
+			"your curl version is too old (>= 7.19.4)");
+#endif
 
 	if (getenv("GIT_CURL_VERBOSE"))
 		curl_easy_setopt(result, CURLOPT_VERBOSE, 1);

line-log.c

@@ -325,7 +325,7 @@ static int collect_diff_cb(long start_a, long count_a,
 	return 0;
 }
 
-static void collect_diff(mmfile_t *parent, mmfile_t *target, struct diff_ranges *out)
+static int collect_diff(mmfile_t *parent, mmfile_t *target, struct diff_ranges *out)
 {
 	struct collect_diff_cbdata cbdata = {NULL};
 	xpparam_t xpp;
@@ -340,7 +340,7 @@ static void collect_diff(mmfile_t *parent, mmfile_t *target, struct diff_ranges
 	xecfg.hunk_func = collect_diff_cb;
 	memset(&ecb, 0, sizeof(ecb));
 	ecb.priv = &cbdata;
-	xdi_diff(parent, target, &xpp, &xecfg, &ecb);
+	return xdi_diff(parent, target, &xpp, &xecfg, &ecb);
 }
 
 /*
@@ -1030,7 +1030,8 @@ static int process_diff_filepair(struct rev_info *rev,
 	}
 
 	diff_ranges_init(&diff);
-	collect_diff(&file_parent, &file_target, &diff);
+	if (collect_diff(&file_parent, &file_target, &diff))
+		die("unable to generate diff for %s", pair->one->path);
 
 	/* NEEDSWORK should apply some heuristics to prevent mismatches */
 	free(rg->path);

ll-merge.c

@@ -89,7 +89,10 @@ static int ll_xdl_merge(const struct ll_merge_driver *drv_unused,
 	xmparam_t xmp;
 	assert(opts);
 
-	if (buffer_is_binary(orig->ptr, orig->size) ||
+	if (orig->size > MAX_XDIFF_SIZE ||
+	    src1->size > MAX_XDIFF_SIZE ||
+	    src2->size > MAX_XDIFF_SIZE ||
+	    buffer_is_binary(orig->ptr, orig->size) ||
 	    buffer_is_binary(src1->ptr, src1->size) ||
 	    buffer_is_binary(src2->ptr, src2->size)) {
 		return ll_binary_merge(drv_unused, result,