Merge branch 'master' into release

CI/tests.mk

@@ -149,7 +149,7 @@ hg.clonebundles.hg: hg.hg hg.incr.bundle
 hg.clonebundles-full.hg: hg.hg hg.full.bundle
 hg.clonebundles.hg hg.clonebundles-full.hg:
 	$(HG) clone -U $< $@
-	echo http://localhost:8080/$(word 2,$^) BUNDLESPEC=$(BUNDLESPEC) > $@/.hg/clonebundles.manifest
+	echo http://localhost:8080/$(word 2,$^) BUNDLESPEC=$(BUNDLESPEC) | tee $@/.hg/clonebundles.manifest
 
 hg.clonebundles.git: hg.clonebundles.hg hg.git
 hg.clonebundles-full.git: hg.clonebundles-full.hg hg.git
@@ -198,7 +198,7 @@ hg.cinnabarclone.git hg.cinnabarclone-full.git hg.cinnabarclone-graft.git hg.cin
 hg.cinnabarclone-bundle.git hg.cinnabarclone-bundle-full.git hg.cinnabarclone-graft-bundle.git: OTHER_SERVER=http
 hg.cinnabarclone.git hg.cinnabarclone-full.git hg.cinnabarclone-bundle.git hg.cinnabarclone-bundle-full.git hg.cinnabarclone-graft.git hg.cinnabarclone-graft-replace.git: hg.pure.hg
 	$(HG) clone -U $< $@.hg
-	($(if $(GIT_CINNABAR_OLD),,echo http://localhost:8888/$(word 2,$^) foo=1 ; )echo http://localhost:8080/$(word 2,$^)) > $@.hg/.hg/cinnabar.manifest
+	($(if $(GIT_CINNABAR_OLD),,echo http://localhost:8888/$(word 2,$^) foo=1 ; )echo http://localhost:8080/$(word 2,$^)) | tee $@.hg/.hg/cinnabar.manifest
 	$(if $(GIT_CINNABAR_OLD),env GIT_CINNABAR_EXPERIMENTS=$(GIT_CINNABAR_EXPERIMENTS:%=%,)git-clone) $(HG) -R $@.hg --config serve.other=$(OTHER_SERVER) --config extensions.x=$(TOPDIR)/CI/hg-serve-exec.py --config extensions.cinnabarclone=$(HG_CINNABARCLONE_EXT) serve-and-exec -- $(GIT) clone hg://localhost:8000.http/ $@
 	$(call COMPARE_REFS, $(or $(word 3,$^),$(word 2,$^)), $@)
 	$(GIT) -C $@ cinnabar fsck
@@ -209,8 +209,8 @@ hg.cinnabarclone-graft-bundle.git: hg.pure.hg
 	cp -r $(word 3,$^) $@
 	$(GIT) -C $@ cinnabar rollback 0000000000000000000000000000000000000000
 	$(GIT) -C $@ remote rename origin grafted
-	($(if $(GIT_CINNABAR_OLD),,echo http://localhost:8080/$(word 4,$^) graft=$$($(GIT) ls-remote $(CURDIR)/$(word 4,$^) refs/cinnabar/replace/* | awk -F/ '{print $$NF}'); )echo http://localhost:8080/$(word 2,$^)) > $@.hg/.hg/cinnabar.manifest
-	$(if $(GIT_CINNABAR_OLD),env GIT_CINNABAR_EXPERIMENTS=$(GIT_CINNABAR_EXPERIMENTS:%=%,)git-clone) $(HG) -R $@.hg --config serve.other=http --config extensions.x=$(TOPDIR)/CI/hg-serve-exec.py --config extensions.cinnabarclone=$(HG_CINNABARCLONE_EXT) serve-and-exec -- $(GIT) -C $@ fetch hg://localhost:8000.http/ refs/heads/*:refs/remotes/origin/*
+	(echo http://localhost:8080/$(word 2,$^)$(if $(GIT_CINNABAR_OLD),,; echo http://localhost:8080/$(word 4,$^) graft=$$($(GIT) ls-remote $(CURDIR)/$(word 4,$^) refs/cinnabar/replace/* | awk -F/ '{print $$NF}'))) | tee $@.hg/.hg/cinnabar.manifest
+	$(if $(GIT_CINNABAR_OLD),env GIT_CINNABAR_EXPERIMENTS=$(GIT_CINNABAR_EXPERIMENTS:%=%,)git-clone) $(HG) -R $@.hg --config serve.other=http --config extensions.x=$(TOPDIR)/CI/hg-serve-exec.py --config extensions.cinnabarclone=$(HG_CINNABARCLONE_EXT) serve-and-exec -- $(GIT) -c cinnabar.graft=true -C $@ fetch hg://localhost:8000.http/ refs/heads/*:refs/remotes/origin/*
 	$(call COMPARE_REFS, $(or $(word 3,$^),$(word 2,$^)), $@)
 	$(GIT) -C $@ cinnabar fsck
 	$(GIT) -C $@ cinnabar fsck --full

CI/tools.py

@@ -222,7 +222,7 @@ def old_compatible_python():
         min_version = None
         for l in fh:
             if l.startswith('#define MIN_CMD_VERSION'):
-                min_version = l.rstrip().split()[-1][:2]
+                min_version = l.rstrip().split()[-1]
                 break
         if not min_version:
             raise Exception('Cannot find MIN_CMD_VERSION')
@@ -314,9 +314,7 @@ def prefix(p, s):
         elif not os.startswith('osx'):
             make_flags.append('USE_LIBPCRE1=YesPlease')
             make_flags.append('USE_LIBPCRE2=')
-            make_flags.append(
-                'CFLAGS+="-DCURLOPT_PROXY_CAINFO=246 -DCURL_STATICLIB"')
-            make_flags.append('CURL_LDFLAGS="$(curl-config --static-libs)"')
+            make_flags.append('CFLAGS+=-DCURLOPT_PROXY_CAINFO=246')
 
         hash = hash or helper_hash()
 

Makefile

@@ -113,6 +113,9 @@ PYTHON_LIBS_DIRS := $(sort $(dir $(PYTHON_LIBS)))
 $(SHELL_SCRIPTS):
 	ln -s ../$@ $@
 
+clean-cinnabarscripts:
+	rm $(SHELL_SCRIPTS)
+
 install-cinnabarscripts:
 	$(INSTALL) $(SHELL_SCRIPTS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 
@@ -174,12 +177,15 @@ endif
 
 ifndef NO_CURL
 ifeq (,$(filter http.c.patch,$(PATCHES)))
-git-cinnabar-helper$X: http.o
+libcinnabar.a: http.o
 endif
 endif
-git-cinnabar-helper$X: $(CINNABAR_OBJECTS) GIT-LDFLAGS $(GITLIBS)
-	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
-		$(CURL_LIBCURL) $(LIBS)
+libcinnabar.a: $(CINNABAR_OBJECTS) $(filter-out $(PATCHES:%.c.patch=%.o) run-command.o compat/mingw.o,$(LIB_OBJS)) $(XDIFF_OBJS)
+	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
+
+git-cinnabar-helper$X: libcinnabar.a GIT-LDFLAGS
+	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) libcinnabar.a \
+		$(CURL_LIBCURL) $(EXTLIBS)
 
 cinnabar-helper.o: EXTRA_CPPFLAGS=-DHELPER_HASH=$(shell python ../git-cinnabar --version=helper 2> /dev/null | awk -F/ '{print $$NF}')
 cinnabar-helper.o: $(addprefix ../helper/,$(PATCHES) $(CINNABAR_OBJECTS:%.o=%.c))

cinnabar/__init__.py

@@ -1 +1 @@
-VERSION = '0.5.3'
+VERSION = '0.5.4'

cinnabar/cmd/fsck.py

@@ -20,7 +20,6 @@
     fsdecode,
     interval_expired,
     iteritems,
-    itervalues,
     Progress,
     progress_iter,
 )
@@ -200,7 +199,7 @@ def get_checked_metadata(num):
     # it's close to an order of magnitude difference on the "Checking
     # manifests" loop.
     depths = {}
-    roots = {}
+    roots = set()
     manifest_queue = []
     revs = []
     revs.append(b'%s^@' % manifests)
@@ -210,20 +209,10 @@ def get_checked_metadata(num):
             'Loading {} manifests', GitHgHelper.rev_list(
                 b'--topo-order', b'--reverse', b'--full-history', *revs)):
         manifest_queue.append((m, parents))
-        if parents:
-            depth = {}
-            for p in parents:
-                for root, num in iteritems(depths.get(p, {})):
-                    if root in depth:
-                        depth[root] = max(depth[root], num + 1)
-                    else:
-                        depth[root] = num + 1
-            if depth:
-                depths[m] = depth
-                del depth
-                continue
-        depths[m] = {m: 0}
-        roots[m] = parents
+        for p in parents:
+            if p not in depths:
+                roots.add(p)
+            depths[m] = max(depths.get(p, 0) + 1, depths.get(m, 0))
 
     if status('broken'):
         return 1
@@ -233,7 +222,7 @@ def get_checked_metadata(num):
     manifests_commit = GitCommit(manifests)
     checked_commit = get_checked_metadata(2)
     depths = [
-        ([depths[p].get(r, 0) for r in roots], p)
+        (depths.get(p, 0), p)
         for p in manifests_commit.parents
         if not checked_commit or p not in checked_commit.parents
     ]
@@ -278,17 +267,16 @@ def get_checked_metadata(num):
     # Don't check files that were already there in the previously checked
     # manifests.
     previous = None
-    for parents in itervalues(roots):
-        for p in parents:
-            if previous:
-                for _, _, before, after, d, path in GitHgHelper.diff_tree(
-                        previous, p):
-                    if d in b'AM' and before != after:
-                        all_interesting.discard((path, after))
-            else:
-                for _, t, sha1, path in GitHgHelper.ls_tree(p, recursive=True):
-                    all_interesting.discard((path, sha1))
-            previous = p
+    for r in roots:
+        if previous:
+            for _, _, before, after, d, path in GitHgHelper.diff_tree(
+                    previous, r):
+                if d in b'AM' and before != after:
+                    all_interesting.discard((path, after))
+        else:
+            for _, t, sha1, path in GitHgHelper.ls_tree(r, recursive=True):
+                all_interesting.discard((path, sha1))
+        previous = r
 
     progress = Progress('Checking {} files')
     while all_interesting and manifest_queue:

cinnabar/hg/repo.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 import os
+import re
 import ssl
 import sys
 try:
@@ -740,6 +741,15 @@ class dummy(object):
                          version)
             continue
 
+        params_dict = {}
+        for p in params.split(b':'):
+            k, _, v = p.partition(b'=')
+            params_dict[k] = v
+
+        if 'stream' in params_dict:
+            logger.debug('Skip because stream bundles are not supported')
+            continue
+
         return url
 
 
@@ -854,6 +864,9 @@ def enumerate_files(iter):
                 logging.warn('Cannot graft %s, not importing.', cs.node)
 
 
+SHA1_RE = re.compile(b'[0-9a-fA-F]{1,40}$')
+
+
 def do_cinnabarclone(repo, manifest, store):
     url = None
     for line in manifest.splitlines():
@@ -873,16 +886,25 @@ def do_cinnabarclone(repo, manifest, store):
             if not graft:
                 continue
             graft = graft.split(b',')
-            revs = list(Git.iter('rev-parse', '--revs-only', *graft))
+            graft_u = []
+            for g in graft:
+                if SHA1_RE.match(g):
+                    graft_u.append(g.decode('ascii'))
+            if len(graft) != len(graft_u):
+                continue
+            revs = list(Git.iter('rev-parse', '--revs-only', *graft_u))
             if len(revs) != len(graft):
                 continue
             # We apparently have all the grafted revisions locally, ensure
             # they're actually reachable.
             if not any(Git.iter(
                     'rev-list', '--branches', '--tags', '--remotes',
                     '--max-count=1', '--ancestry-path', '--stdin',
-                    stdin=('^{}^@'.format(c) for c in graft))):
+                    stdin=(b'^%s^@' % c for c in graft))):
                 continue
+        elif graft:
+            # When not grafting, ignore lines with a graft revision.
+            continue
         url, _, branch = spec.partition(b'#')
         url, branch = (url.split(b'#', 1) + [None])[:2]
         if url:

cinnabar/util.py

@@ -151,7 +151,7 @@ def __call__(self, name):
 )
 
 
-def interval_expired(config_key, interval):
+def interval_expired(config_key, interval, globl=False):
     from .git import Git
     config_key = 'cinnabar.{}'.format(config_key)
     try:
@@ -162,7 +162,12 @@ def interval_expired(config_key, interval):
     if last:
         if last + interval > now:
             return False
-    Git.run('config', '--global', config_key, str(int(now)))
+    # cinnabar.fsck used to be global and is now local.
+    # Remove the global value.
+    if globl is not True and config_key == 'cinnabar.fsck':
+        Git.run('config', '--global', '--unset', config_key)
+    Git.run('config', '--global' if globl else '--local',
+            config_key, str(int(now)))
     return bool(last)
 
 
@@ -742,7 +747,7 @@ def __init__(self, *args, **kwargs):
                 self._stdin.write(stdin)
             elif isinstance(stdin, Iterable):
                 for line in stdin:
-                    self._stdin.write('%s\n' % line)
+                    self._stdin.write(b'%s\n' % line)
             if proc_stdin != stdin:
                 self._proc.stdin.close()
 
@@ -858,7 +863,7 @@ def run(self):
         parent_dir = os.path.dirname(os.path.dirname(__file__))
         if not os.path.exists(os.path.join(parent_dir, '.git')) or \
                 check_enabled('no-version-check') or \
-                not interval_expired('version-check', 86400):
+                not interval_expired('version-check', 86400, globl=True):
             return
         REPO = 'https://github.com/glandium/git-cinnabar'
         devnull = open(os.devnull, 'wb')

helper/cinnabar-helper.c

@@ -49,6 +49,7 @@
 #include <string.h>
 
 #include "cache.h"
+#include "attr.h"
 #include "blob.h"
 #include "commit.h"
 #include "config.h"
@@ -84,7 +85,7 @@
 #endif
 
 #define CMD_VERSION 3003
-#define MIN_CMD_VERSION 3000
+#define MIN_CMD_VERSION 3003
 
 static const char NULL_NODE[] = "0000000000000000000000000000000000000000";
 
@@ -2421,11 +2422,39 @@ static void init_git_config()
 	strbuf_release(&path);
 }
 
-int cmd_main(int argc, const char *argv[])
+static void restore_sigpipe_to_default(void)
+{
+	sigset_t unblock;
+
+	sigemptyset(&unblock);
+	sigaddset(&unblock, SIGPIPE);
+	sigprocmask(SIG_UNBLOCK, &unblock, NULL);
+	signal(SIGPIPE, SIG_DFL);
+}
+
+int main(int argc, const char *argv[])
 {
 	int initialized = 0;
 	struct strbuf buf = STRBUF_INIT;
 
+	// Initialization from common-main.c.
+	trace2_initialize_clock();
+
+	sanitize_stdfds();
+	restore_sigpipe_to_default();
+
+	git_resolve_executable_dir(argv[0]);
+
+	git_setup_gettext();
+
+	initialize_the_repository();
+
+	attr_start();
+
+	trace2_initialize();
+	trace2_cmd_start(argv);
+	trace2_collect_process_info(TRACE2_PROCESS_INFO_STARTUP);
+
 	if (argc > 1) {
 		if (argc > 2)
 			die("Too many arguments");
@@ -2525,6 +2554,5 @@ int cmd_main(int argc, const char *argv[])
 
 	oidset_clear(&hg2git_seen);
 	hashmap_free_entries(&git_tree_cache, struct oid_map_entry, ent);
-
 	return 0;
 }

helper/which.c

@@ -1,3 +1,5 @@
+#include "run-command.c"
+
 #ifdef _WIN32
 #include "compat/mingw.c"
 
@@ -6,7 +8,6 @@ char *which(const char *file) {
 }
 
 #else
-#include "run-command.c"
 
 char *which(const char *file) {
 	return locate_in_PATH(file);