/
verify-commits.py
executable file
·204 lines (180 loc) · 9.56 KB
/
verify-commits.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
#!/usr/bin/env python3
# Copyright (c) 2018-2022 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Verify commits against a trusted keys list."""
import argparse
import hashlib
import logging
import os
import subprocess
import sys
import time
GIT = os.getenv('GIT', 'git')
def tree_sha512sum(commit='HEAD'):
"""Calculate the Tree-sha512 for the commit.
This is copied from github-merge.py. See https://github.com/bitcoin-core/bitcoin-maintainer-tools."""
# request metadata for entire tree, recursively
files = []
blob_by_name = {}
for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
name_sep = line.index(b'\t')
metadata = line[:name_sep].split() # perms, 'blob', blobid
assert metadata[1] == b'blob'
name = line[name_sep + 1:]
files.append(name)
blob_by_name[name] = metadata[2]
files.sort()
# open connection to git-cat-file in batch mode to request data for all blobs
# this is much faster than launching it per file
p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
overall = hashlib.sha512()
for f in files:
blob = blob_by_name[f]
# request blob
p.stdin.write(blob + b'\n')
p.stdin.flush()
# read header: blob, "blob", size
reply = p.stdout.readline().split()
assert reply[0] == blob and reply[1] == b'blob'
size = int(reply[2])
# hash the blob data
intern = hashlib.sha512()
ptr = 0
while ptr < size:
bs = min(65536, size - ptr)
piece = p.stdout.read(bs)
if len(piece) == bs:
intern.update(piece)
else:
raise IOError('Premature EOF reading git cat-file output')
ptr += bs
dig = intern.hexdigest()
assert p.stdout.read(1) == b'\n' # ignore LF that follows blob data
# update overall hash with file hash
overall.update(dig.encode("utf-8"))
overall.update(" ".encode("utf-8"))
overall.update(f)
overall.update("\n".encode("utf-8"))
p.stdin.close()
if p.wait():
raise IOError('Non-zero return value executing git cat-file')
return overall.hexdigest()
def main():
# Enable debug logging if running in CI
if 'CI' in os.environ and os.environ['CI'].lower() == "true":
logging.getLogger().setLevel(logging.DEBUG)
# Parse arguments
parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]')
parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check')
parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER')
parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>')
args = parser.parse_args()
# get directory of this program and read data files
dirname = os.path.dirname(os.path.abspath(__file__))
print("Using verify-commits data from " + dirname)
with open(dirname + "/trusted-git-root", "r", encoding="utf8") as f:
verified_root = f.read().splitlines()[0]
with open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8") as f:
verified_sha512_root = f.read().splitlines()[0]
with open(dirname + "/allow-revsig-commits", "r", encoding="utf8") as f:
revsig_allowed = f.read().splitlines()
with open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf8") as f:
unclean_merge_allowed = f.read().splitlines()
with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f:
incorrect_sha512_allowed = f.read().splitlines()
with open(dirname + "/trusted-keys", "r", encoding="utf8") as f:
trusted_keys = f.read().splitlines()
# Set commit and variables
current_commit = args.commit
if ' ' in current_commit:
print("Commit must not contain spaces", file=sys.stderr)
sys.exit(1)
verify_tree = args.verify_tree
no_sha1 = True
prev_commit = ""
initial_commit = current_commit
# Iterate through commits
while True:
# Log a message to prevent Travis from timing out
logging.debug("verify-commits: [in-progress] processing commit {}".format(current_commit[:8]))
if current_commit == verified_root:
print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root))
sys.exit(0)
else:
# Make sure this commit isn't older than trusted roots
check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_root, current_commit])
if check_root_older_res.returncode != 0:
print(f"\"{current_commit}\" predates the trusted root, stopping!")
sys.exit(0)
if verify_tree:
if current_commit == verified_sha512_root:
print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr)
verify_tree = False
no_sha1 = False
else:
# Skip the tree check if we are older than the trusted root
check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_sha512_root, current_commit])
if check_root_older_res.returncode != 0:
print(f"\"{current_commit}\" predates the trusted SHA512 root, disabling tree verification.")
verify_tree = False
no_sha1 = False
os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1"
allow_revsig = current_commit in revsig_allowed
# Check that the commit (and parents) was signed with a trusted key
valid_sig = False
verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], capture_output=True)
for line in verify_res.stderr.decode().splitlines():
if line.startswith("[GNUPG:] VALIDSIG "):
key = line.split(" ")[-1]
valid_sig = key in trusted_keys
elif (line.startswith("[GNUPG:] REVKEYSIG ") or line.startswith("[GNUPG:] EXPKEYSIG ")) and not allow_revsig:
valid_sig = False
break
if not valid_sig:
if prev_commit != "":
print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
print("Parents are:", file=sys.stderr)
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ')
for parent in parents:
subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
else:
print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr)
sys.exit(1)
# Check the Tree-SHA512
if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
tree_hash = tree_sha512sum(current_commit)
if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines():
print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
sys.exit(1)
# Merge commits should only have two parents
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ')
if len(parents) > 2:
print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
sys.exit(1)
# Check that the merge commit is clean
commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0])
check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days
allow_unclean = current_commit in unclean_merge_allowed
if len(parents) == 2 and check_merge and not allow_unclean:
current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0]
# This merge-tree functionality requires git >= 2.38. The
# --write-tree option was added in order to opt-in to the new
# behavior. Older versions of git will not recognize the option and
# will instead exit with code 128.
try:
recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]]).decode('utf8').splitlines()[0]
except subprocess.CalledProcessError as e:
if e.returncode == 128:
print("git v2.38+ is required for this functionality.", file=sys.stderr)
sys.exit(1)
else:
raise e
if current_tree != recreated_tree:
print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
subprocess.call([GIT, 'diff', recreated_tree, current_tree])
sys.exit(1)
prev_commit = current_commit
current_commit = parents[0]
if __name__ == '__main__':
main()