Merge branch 'svn-fe' of git://repo.or.cz/git/jrn into jn/svn-fe

This simplifies svn-fe a great deal and fulfills a longstanding wish:
support for dumps with deltas in them, and incremental imports.

The cost is that commandline usage of the svn-fe tool becomes a little
more complicated since it no longer keeps state itself but instead reads
blobs back from fast-import in order to copy them between revisions and
apply deltas to them.

Also removes a couple of custom data structures and replaces them with
strbufs like other parts of Git.

* 'svn-fe' of git://repo.or.cz/git/jrn: (32 commits)
  vcs-svn: reset first_commit_done in fast_export_init
  vcs-svn: do not initialize report_buffer twice
  vcs-svn: avoid hangs from corrupt deltas
  vcs-svn: guard against overflow when computing preimage length
  vcs-svn: cap number of bytes read from sliding view
  test-svn-fe: split off "test-svn-fe -d" into a separate function
  vcs-svn: implement text-delta handling
  vcs-svn: let deltas use data from preimage
  vcs-svn: let deltas use data from postimage
  vcs-svn: verify that deltas consume all inline data
  vcs-svn: implement copyfrom_data delta instruction
  vcs-svn: read instructions from deltas
  vcs-svn: read inline data from deltas
  vcs-svn: read the preimage when applying deltas
  vcs-svn: parse svndiff0 window header
  vcs-svn: skeleton of an svn delta parser
  vcs-svn: make buffer_read_binary API more convenient
  vcs-svn: learn to maintain a sliding view of a file
  Makefile: list one vcs-svn/xdiff object or header per line
  vcs-svn: avoid using ls command twice
  ...

Conflicts:
	Makefile
	contrib/svn-fe/svn-fe.txt

LICENSE

@@ -1,8 +1,7 @@
 Copyright (C) 2010 David Barr <david.barr@cordelta.com>.
 All rights reserved.
 
-Copyright (C) 2008 Jason Evans <jasone@canonware.com>.
-All rights reserved.
+Copyright (C) 2010 Jonathan Nieder <jrnieder@gmail.com>.
 
 Copyright (C) 2005 Stefan Hegny, hydrografix Consulting GmbH,
 Frankfurt/Main, Germany

fast_export.c

@@ -4,34 +4,77 @@
  */
 
 #include "git-compat-util.h"
+#include "strbuf.h"
+#include "quote.h"
 #include "fast_export.h"
-#include "line_buffer.h"
 #include "repo_tree.h"
-#include "string_pool.h"
+#include "strbuf.h"
+#include "svndiff.h"
+#include "sliding_window.h"
+#include "line_buffer.h"
 
 #define MAX_GITSVN_LINE_LEN 4096
 
 static uint32_t first_commit_done;
+static struct line_buffer postimage = LINE_BUFFER_INIT;
+static struct line_buffer report_buffer = LINE_BUFFER_INIT;
+
+/* NEEDSWORK: move to fast_export_init() */
+static int init_postimage(void)
+{
+	static int postimage_initialized;
+	if (postimage_initialized)
+		return 0;
+	postimage_initialized = 1;
+	return buffer_tmpfile_init(&postimage);
+}
+
+void fast_export_init(int fd)
+{
+	first_commit_done = 0;
+	if (buffer_fdinit(&report_buffer, fd))
+		die_errno("cannot read from file descriptor %d", fd);
+}
+
+void fast_export_deinit(void)
+{
+	if (buffer_deinit(&report_buffer))
+		die_errno("error closing fast-import feedback stream");
+}
+
+void fast_export_reset(void)
+{
+	buffer_reset(&report_buffer);
+}
 
-void fast_export_delete(uint32_t depth, uint32_t *path)
+void fast_export_delete(const char *path)
 {
 	putchar('D');
 	putchar(' ');
-	pool_print_seq(depth, path, '/', stdout);
+	quote_c_style(path, NULL, stdout, 0);
 	putchar('\n');
 }
 
-void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode,
-			uint32_t mark)
+static void fast_export_truncate(const char *path, uint32_t mode)
+{
+	fast_export_modify(path, mode, "inline");
+	printf("data 0\n\n");
+}
+
+void fast_export_modify(const char *path, uint32_t mode, const char *dataref)
 {
 	/* Mode must be 100644, 100755, 120000, or 160000. */
-	printf("M %06"PRIo32" :%"PRIu32" ", mode, mark);
-	pool_print_seq(depth, path, '/', stdout);
+	if (!dataref) {
+		fast_export_truncate(path, mode);
+		return;
+	}
+	printf("M %06"PRIo32" %s ", mode, dataref);
+	quote_c_style(path, NULL, stdout, 0);
 	putchar('\n');
 }
 
 static char gitsvnline[MAX_GITSVN_LINE_LEN];
-void fast_export_commit(uint32_t revision, const char *author,
+void fast_export_begin_commit(uint32_t revision, const char *author,
 			const struct strbuf *log,
 			const char *uuid, const char *url,
 			unsigned long timestamp)
@@ -47,6 +90,7 @@ void fast_export_commit(uint32_t revision, const char *author,
 		*gitsvnline = '\0';
 	}
 	printf("commit refs/heads/master\n");
+	printf("mark :%"PRIu32"\n", revision);
 	printf("committer %s <%s@%s> %ld +0000\n",
 		   *author ? author : "nobody",
 		   *author ? author : "nobody",
@@ -57,32 +101,213 @@ void fast_export_commit(uint32_t revision, const char *author,
 	printf("%s\n", gitsvnline);
 	if (!first_commit_done) {
 		if (revision > 1)
-			printf("from refs/heads/master^0\n");
+			printf("from :%"PRIu32"\n", revision - 1);
 		first_commit_done = 1;
 	}
-	repo_diff(revision - 1, revision);
-	fputc('\n', stdout);
+}
 
+void fast_export_end_commit(uint32_t revision)
+{
 	printf("progress Imported commit %"PRIu32".\n\n", revision);
 }
 
+static void ls_from_rev(uint32_t rev, const char *path)
+{
+	/* ls :5 path/to/old/file */
+	printf("ls :%"PRIu32" ", rev);
+	quote_c_style(path, NULL, stdout, 0);
+	putchar('\n');
+	fflush(stdout);
+}
+
+static void ls_from_active_commit(const char *path)
+{
+	/* ls "path/to/file" */
+	printf("ls \"");
+	quote_c_style(path, NULL, stdout, 1);
+	printf("\"\n");
+	fflush(stdout);
+}
+
+static const char *get_response_line(void)
+{
+	const char *line = buffer_read_line(&report_buffer);
+	if (line)
+		return line;
+	if (buffer_ferror(&report_buffer))
+		die_errno("error reading from fast-import");
+	die("unexpected end of fast-import feedback");
+}
+
 static void die_short_read(struct line_buffer *input)
 {
 	if (buffer_ferror(input))
 		die_errno("error reading dump file");
 	die("invalid dump: unexpected end of file");
 }
 
-void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input)
+static int ends_with(const char *s, size_t len, const char *suffix)
+{
+	const size_t suffixlen = strlen(suffix);
+	if (len < suffixlen)
+		return 0;
+	return !memcmp(s + len - suffixlen, suffix, suffixlen);
+}
+
+static int parse_cat_response_line(const char *header, off_t *len)
+{
+	size_t headerlen = strlen(header);
+	uintmax_t n;
+	const char *type;
+	const char *end;
+
+	if (ends_with(header, headerlen, " missing"))
+		return error("cat-blob reports missing blob: %s", header);
+	type = memmem(header, headerlen, " blob ", strlen(" blob "));
+	if (!type)
+		return error("cat-blob header has wrong object type: %s", header);
+	n = strtoumax(type + strlen(" blob "), (char **) &end, 10);
+	if (end == type + strlen(" blob "))
+		return error("cat-blob header does not contain length: %s", header);
+	if (memchr(type + strlen(" blob "), '-', end - type - strlen(" blob ")))
+		return error("cat-blob header contains negative length: %s", header);
+	if (n == UINTMAX_MAX || n > maximum_signed_value_of_type(off_t))
+		return error("blob too large for current definition of off_t");
+	*len = n;
+	if (*end)
+		return error("cat-blob header contains garbage after length: %s", header);
+	return 0;
+}
+
+static void check_preimage_overflow(off_t a, off_t b)
+{
+	if (signed_add_overflows(a, b))
+		die("blob too large for current definition of off_t");
+}
+
+static long apply_delta(off_t len, struct line_buffer *input,
+			const char *old_data, uint32_t old_mode)
+{
+	long ret;
+	struct sliding_view preimage = SLIDING_VIEW_INIT(&report_buffer, 0);
+	FILE *out;
+
+	if (init_postimage() || !(out = buffer_tmpfile_rewind(&postimage)))
+		die("cannot open temporary file for blob retrieval");
+	if (old_data) {
+		const char *response;
+		printf("cat-blob %s\n", old_data);
+		fflush(stdout);
+		response = get_response_line();
+		if (parse_cat_response_line(response, &preimage.max_off))
+			die("invalid cat-blob response: %s", response);
+		check_preimage_overflow(preimage.max_off, 1);
+	}
+	if (old_mode == REPO_MODE_LNK) {
+		strbuf_addstr(&preimage.buf, "link ");
+		check_preimage_overflow(preimage.max_off, strlen("link "));
+		preimage.max_off += strlen("link ");
+		check_preimage_overflow(preimage.max_off, 1);
+	}
+	if (svndiff0_apply(input, len, &preimage, out))
+		die("cannot apply delta");
+	if (old_data) {
+		/* Read the remainder of preimage and trailing newline. */
+		assert(!signed_add_overflows(preimage.max_off, 1));
+		preimage.max_off++;	/* room for newline */
+		if (move_window(&preimage, preimage.max_off - 1, 1))
+			die("cannot seek to end of input");
+		if (preimage.buf.buf[0] != '\n')
+			die("missing newline after cat-blob response");
+	}
+	ret = buffer_tmpfile_prepare_to_read(&postimage);
+	if (ret < 0)
+		die("cannot read temporary file for blob retrieval");
+	strbuf_release(&preimage.buf);
+	return ret;
+}
+
+void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input)
 {
 	if (mode == REPO_MODE_LNK) {
 		/* svn symlink blobs start with "link " */
 		len -= 5;
 		if (buffer_skip_bytes(input, 5) != 5)
 			die_short_read(input);
 	}
-	printf("blob\nmark :%"PRIu32"\ndata %"PRIu32"\n", mark, len);
+	printf("data %"PRIu32"\n", len);
 	if (buffer_copy_bytes(input, len) != len)
 		die_short_read(input);
 	fputc('\n', stdout);
 }
+
+static int parse_ls_response(const char *response, uint32_t *mode,
+					struct strbuf *dataref)
+{
+	const char *tab;
+	const char *response_end;
+
+	assert(response);
+	response_end = response + strlen(response);
+
+	if (*response == 'm') {	/* Missing. */
+		errno = ENOENT;
+		return -1;
+	}
+
+	/* Mode. */
+	if (response_end - response < strlen("100644") ||
+	    response[strlen("100644")] != ' ')
+		die("invalid ls response: missing mode: %s", response);
+	*mode = 0;
+	for (; *response != ' '; response++) {
+		char ch = *response;
+		if (ch < '0' || ch > '7')
+			die("invalid ls response: mode is not octal: %s", response);
+		*mode *= 8;
+		*mode += ch - '0';
+	}
+
+	/* ' blob ' or ' tree ' */
+	if (response_end - response < strlen(" blob ") ||
+	    (response[1] != 'b' && response[1] != 't'))
+		die("unexpected ls response: not a tree or blob: %s", response);
+	response += strlen(" blob ");
+
+	/* Dataref. */
+	tab = memchr(response, '\t', response_end - response);
+	if (!tab)
+		die("invalid ls response: missing tab: %s", response);
+	strbuf_add(dataref, response, tab - response);
+	return 0;
+}
+
+int fast_export_ls_rev(uint32_t rev, const char *path,
+				uint32_t *mode, struct strbuf *dataref)
+{
+	ls_from_rev(rev, path);
+	return parse_ls_response(get_response_line(), mode, dataref);
+}
+
+int fast_export_ls(const char *path, uint32_t *mode, struct strbuf *dataref)
+{
+	ls_from_active_commit(path);
+	return parse_ls_response(get_response_line(), mode, dataref);
+}
+
+void fast_export_blob_delta(uint32_t mode,
+				uint32_t old_mode, const char *old_data,
+				uint32_t len, struct line_buffer *input)
+{
+	long postimage_len;
+	if (len > maximum_signed_value_of_type(off_t))
+		die("enormous delta");
+	postimage_len = apply_delta((off_t) len, input, old_data, old_mode);
+	if (mode == REPO_MODE_LNK) {
+		buffer_skip_bytes(&postimage, strlen("link "));
+		postimage_len -= strlen("link ");
+	}
+	printf("data %ld\n", postimage_len);
+	buffer_copy_bytes(&postimage, postimage_len);
+	fputc('\n', stdout);
+}

fast_export.h

@@ -1,16 +1,28 @@
 #ifndef FAST_EXPORT_H_
 #define FAST_EXPORT_H_
 
-#include "line_buffer.h"
 struct strbuf;
+struct line_buffer;
 
-void fast_export_delete(uint32_t depth, uint32_t *path);
-void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode,
-			uint32_t mark);
-void fast_export_commit(uint32_t revision, const char *author,
+void fast_export_init(int fd);
+void fast_export_deinit(void);
+void fast_export_reset(void);
+
+void fast_export_delete(const char *path);
+void fast_export_modify(const char *path, uint32_t mode, const char *dataref);
+void fast_export_begin_commit(uint32_t revision, const char *author,
 			const struct strbuf *log, const char *uuid,
 			const char *url, unsigned long timestamp);
-void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len,
-		      struct line_buffer *input);
+void fast_export_end_commit(uint32_t revision);
+void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input);
+void fast_export_blob_delta(uint32_t mode,
+			uint32_t old_mode, const char *old_data,
+			uint32_t len, struct line_buffer *input);
+
+/* If there is no such file at that rev, returns -1, errno == ENOENT. */
+int fast_export_ls_rev(uint32_t rev, const char *path,
+			uint32_t *mode_out, struct strbuf *dataref_out);
+int fast_export_ls(const char *path,
+			uint32_t *mode_out, struct strbuf *dataref_out);
 
 #endif

line_buffer.c

@@ -91,10 +91,10 @@ char *buffer_read_line(struct line_buffer *buf)
 	return buf->line_buffer;
 }
 
-void buffer_read_binary(struct line_buffer *buf,
-				struct strbuf *sb, uint32_t size)
+size_t buffer_read_binary(struct line_buffer *buf,
+				struct strbuf *sb, size_t size)
 {
-	strbuf_fread(sb, size, buf->infile);
+	return strbuf_fread(sb, size, buf->infile);
 }
 
 off_t buffer_copy_bytes(struct line_buffer *buf, off_t nbytes)

line_buffer.h

@@ -23,7 +23,7 @@ long buffer_tmpfile_prepare_to_read(struct line_buffer *buf);
 int buffer_ferror(struct line_buffer *buf);
 char *buffer_read_line(struct line_buffer *buf);
 int buffer_read_char(struct line_buffer *buf);
-void buffer_read_binary(struct line_buffer *buf, struct strbuf *sb, uint32_t len);
+size_t buffer_read_binary(struct line_buffer *buf, struct strbuf *sb, size_t len);
 /* Returns number of bytes read (not necessarily written). */
 off_t buffer_copy_bytes(struct line_buffer *buf, off_t len);
 off_t buffer_skip_bytes(struct line_buffer *buf, off_t len);