Merge pull request #2727 from PastaPastaPasta/backports-0.15-pr7

Backports 0.15 pr7

contrib/devtools/github-merge.py

@@ -78,24 +78,53 @@ def get_symlink_files():
             ret.append(f.decode('utf-8').split("\t")[1])
     return ret
 
-def tree_sha512sum():
-    files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', '--name-only', 'HEAD']).splitlines())
+def tree_sha512sum(commit='HEAD'):
+    # request metadata for entire tree, recursively
+    files = []
+    blob_by_name = {}
+    for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
+        name_sep = line.index(b'\t')
+        metadata = line[:name_sep].split() # perms, 'blob', blobid
+        assert(metadata[1] == b'blob')
+        name = line[name_sep+1:]
+        files.append(name)
+        blob_by_name[name] = metadata[2]
+
+    files.sort()
+    # open connection to git-cat-file in batch mode to request data for all blobs
+    # this is much faster than launching it per file
+    p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
     overall = hashlib.sha512()
     for f in files:
+        blob = blob_by_name[f]
+        # request blob
+        p.stdin.write(blob + b'\n')
+        p.stdin.flush()
+        # read header: blob, "blob", size
+        reply = p.stdout.readline().split()
+        assert(reply[0] == blob and reply[1] == b'blob')
+        size = int(reply[2])
+        # hash the blob data
         intern = hashlib.sha512()
-        fi = open(f, 'rb')
-        while True:
-            piece = fi.read(65536)
-            if piece:
+        ptr = 0
+        while ptr < size:
+            bs = min(65536, size - ptr)
+            piece = p.stdout.read(bs)
+            if len(piece) == bs:
                 intern.update(piece)
             else:
-                break
-        fi.close()
+                raise IOError('Premature EOF reading git cat-file output')
+            ptr += bs
         dig = intern.hexdigest()
+        assert(p.stdout.read(1) == b'\n') # ignore LF that follows blob data
+        # update overall hash with file hash
         overall.update(dig.encode("utf-8"))
         overall.update("  ".encode("utf-8"))
         overall.update(f)
         overall.update("\n".encode("utf-8"))
+    p.stdin.close()
+    if p.wait():
+        raise IOError('Non-zero return value executing git cat-file')
     return overall.hexdigest()
 
 

contrib/gitian-descriptors/gitian-win.yml

@@ -179,6 +179,7 @@ script: |
     make ${MAKEOPTS} -C src check-security
     make deploy
     make install DESTDIR=${INSTALLPATH}
+    rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe
     cp -f dashcore-*setup*.exe $OUTDIR/
     cd installed
     mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/
@@ -192,9 +193,11 @@ script: |
     cd ../../
     rm -rf distsrc-${i}
   done
-  cd $OUTDIR
-  rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe
-  find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz
+  cp -rf contrib/windeploy $BUILD_DIR
+  cd $BUILD_DIR/windeploy
+  mkdir unsigned
+  cp $OUTDIR/dashcore-*setup-unsigned.exe unsigned/
+  find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz
   mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip
   mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip
   mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip

contrib/macdeploy/detached-sig-create.sh

@@ -10,7 +10,7 @@ BUNDLE="${ROOTDIR}/Dash-Qt.app"
 CODESIGN=codesign
 TEMPDIR=sign.temp
 TEMPLIST=${TEMPDIR}/signatures.txt
-OUT=signature.tar.gz
+OUT=signature-osx.tar.gz
 OUTROOT=osx
 
 if [ ! -n "$1" ]; then

contrib/verify-commits/trusted-sha512-root-commit

@@ -1 +1 @@
-f7ec7cfd38b543ba81ac7bed5b77f9a19739460b
+309bf16257b2395ce502017be627186b749ee749

contrib/verify-commits/verify-commits.sh

@@ -3,9 +3,6 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-# Not technically POSIX-compliant due to use of "local", but almost every
-# shell anyone uses today supports it, so its probably fine
-
 DIR=$(dirname "$0")
 [ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0")
 
@@ -16,14 +13,36 @@ VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit")
 REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits")
 
 HAVE_FAILED=false
-IS_SIGNED () {
-	if [ $1 = $VERIFIED_ROOT ]; then
-		return 0;
+
+HAVE_GNU_SHA512=1
+[ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0
+
+if [ x"$1" = "x" ]; then
+	CURRENT_COMMIT="HEAD"
+else
+	CURRENT_COMMIT="$1"
+fi
+
+if [ "${CURRENT_COMMIT#* }" != "$CURRENT_COMMIT" ]; then
+	echo "Commit must not contain spaces?" > /dev/stderr
+	exit 1
+fi
+
+VERIFY_TREE=0
+if [ x"$2" = "x--tree-checks" ]; then
+	VERIFY_TREE=1
+fi
+
+NO_SHA1=1
+PREV_COMMIT=""
+
+while true; do
+	if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then
+		echo "There is a valid path from "$CURRENT_COMMIT" to $VERIFIED_ROOT where all commits are signed!"
+		exit 0;
 	fi
 
-	VERIFY_TREE=$2
-	NO_SHA1=$3
-	if [ $1 = $VERIFIED_SHA512_ROOT ]; then
+	if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then
 		if [ "$VERIFY_TREE" = "1" ]; then
 			echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr
 		fi
@@ -37,91 +56,77 @@ IS_SIGNED () {
 		export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1
 	fi
 
-	if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then
+	if [ "${REVSIG_ALLOWED#*$CURRENT_COMMIT}" != "$REVSIG_ALLOWED" ]; then
 		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1
 	else
 		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0
 	fi
 
-	if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null; then
-		return 1;
+	if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit "$CURRENT_COMMIT" > /dev/null; then
+		if [ "$PREV_COMMIT" != "" ]; then
+			echo "No parent of $PREV_COMMIT was signed with a trusted key!" > /dev/stderr
+			echo "Parents are:" > /dev/stderr
+			PARENTS=$(git show -s --format=format:%P $PREV_COMMIT)
+			for PARENT in $PARENTS; do
+				git show -s $PARENT > /dev/stderr
+			done
+		else
+			echo "$CURRENT_COMMIT was not signed with a trusted key!" > /dev/stderr
+		fi
+		exit 1
 	fi
 
-	# We set $4 to 1 on the first call, always verifying the top of the tree
-	if [ "$VERIFY_TREE" = 1 -o "$4" = "1" ]; then
+	# We always verify the top of the tree
+	if [ "$VERIFY_TREE" = 1 -o "$PREV_COMMIT" = "" ]; then
 		IFS_CACHE="$IFS"
 		IFS='
 '
-		for LINE in $(git ls-tree --full-tree -r $1); do
+		for LINE in $(git ls-tree --full-tree -r "$CURRENT_COMMIT"); do
 			case "$LINE" in
 				"12"*)
 					echo "Repo contains symlinks" > /dev/stderr
 					IFS="$IFS_CACHE"
-					return 1
+					exit 1
 					;;
 			esac
 		done
 		IFS="$IFS_CACHE"
 
 		FILE_HASHES=""
-		for FILE in $(git ls-tree --full-tree -r --name-only $1 | LC_ALL=C sort); do
-			HASH=$(git cat-file blob $1:"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } )
+		for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do
+			if [ "$HAVE_GNU_SHA512" = 1 ]; then
+				HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } )
+			else
+				HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST OTHER; echo $FIRST; } )
+			fi
 			[ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"'
 '
 			FILE_HASHES="$FILE_HASHES$HASH  $FILE"
 		done
+
+		if [ "$HAVE_GNU_SHA512" = 1 ]; then
+			TREE_HASH="$(echo "$FILE_HASHES" | sha512sum)"
+		else
+			TREE_HASH="$(echo "$FILE_HASHES" | shasum -a 512)"
+		fi
 		HASH_MATCHES=0
-		MSG="$(git show -s --format=format:%B $1 | tail -n1)"
+		MSG="$(git show -s --format=format:%B "$CURRENT_COMMIT" | tail -n1)"
 
 		case "$MSG  -" in
-			"Tree-SHA512: $(echo "$FILE_HASHES" | sha512sum)")
+			"Tree-SHA512: $TREE_HASH")
 				HASH_MATCHES=1;;
 		esac
 
 		if [ "$HASH_MATCHES" = "0" ]; then
-			echo "Tree-SHA512 did not match for commit $1" > /dev/stderr
-			HAVE_FAILED=true
-			return 1
+			echo "Tree-SHA512 did not match for commit $CURRENT_COMMIT" > /dev/stderr
+			exit 1
 		fi
 	fi
 
-	local PARENTS
-	PARENTS=$(git show -s --format=format:%P $1)
+	PARENTS=$(git show -s --format=format:%P "$CURRENT_COMMIT")
 	for PARENT in $PARENTS; do
-		if IS_SIGNED $PARENT $VERIFY_TREE $NO_SHA1 0; then
-			return 0;
-		fi
+		PREV_COMMIT="$CURRENT_COMMIT"
+		CURRENT_COMMIT="$PARENT"
+		break
 	done
-	if ! "$HAVE_FAILED"; then
-		echo "No parent of $1 was signed with a trusted key!" > /dev/stderr
-		echo "Parents are:" > /dev/stderr
-		for PARENT in $PARENTS; do
-			git show -s $PARENT > /dev/stderr
-		done
-		HAVE_FAILED=true
-	fi
-	return 1;
-}
-
-if [ x"$1" = "x" ]; then
-	TEST_COMMIT="HEAD"
-else
-	TEST_COMMIT="$1"
-fi
-
-DO_CHECKOUT_TEST=0
-if [ x"$2" = "x--tree-checks" ]; then
-	DO_CHECKOUT_TEST=1
-fi
-
-IS_SIGNED "$TEST_COMMIT" "$DO_CHECKOUT_TEST" 1 1
-RES=$?
-if [ "$RES" = 1 ]; then
-	if ! "$HAVE_FAILED"; then
-		echo "$TEST_COMMIT was not signed with a trusted key!"
-	fi
-else
-	echo "There is a valid path from $TEST_COMMIT to $VERIFIED_ROOT where all commits are signed!"
-fi
-
-exit $RES
+done

contrib/windeploy/detached-sig-create.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+# Copyright (c) 2014-2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+if [ -z "$OSSLSIGNCODE" ]; then
+  OSSLSIGNCODE=osslsigncode
+fi
+
+if [ ! -n "$1" ]; then
+  echo "usage: $0 <osslcodesign args>"
+  echo "example: $0 -key codesign.key"
+  exit 1
+fi
+
+OUT=signature-win.tar.gz
+SRCDIR=unsigned
+WORKDIR=./.tmp
+OUTDIR="${WORKDIR}/out"
+OUTSUBDIR="${OUTDIR}/win"
+TIMESERVER=http://timestamp.comodoca.com
+CERTFILE="win-codesign.cert"
+
+mkdir -p "${OUTSUBDIR}"
+basename -a `ls -1 "${SRCDIR}"/*-unsigned.exe` | while read UNSIGNED; do
+  echo Signing "${UNSIGNED}"
+  "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@"
+  "${OSSLSIGNCODE}" extract-signature -pem -in "${WORKDIR}/${UNSIGNED}" -out "${OUTSUBDIR}/${UNSIGNED}.pem" && rm "${WORKDIR}/${UNSIGNED}"
+done
+
+rm -f "${OUT}"
+tar -C "${OUTDIR}" -czf "${OUT}" .
+rm -rf "${WORKDIR}"
+echo "Created ${OUT}"