forked from Bitcoin-ABC/bitcoin-abc
/
gitian-build.py
executable file
·378 lines (324 loc) · 18 KB
/
gitian-build.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
#!/usr/bin/env python3
import argparse
import inspect
import multiprocessing
import os
import subprocess
import sys
# Codes for bold red warning formatting are no-ops unless on POSIX
BOLD, RED = ('', ''), ('', ''),
if os.name == 'posix':
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ('\033[0m', '\033[1m')
RED = ('\033[0m', '\033[0;31m')
is_mac = sys.platform == 'darwin'
def setup():
global args, workdir
if not is_mac:
programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget']
if args.kvm:
programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils']
elif args.docker:
dockers = ['docker.io', 'docker-ce']
for i in dockers:
return_code = subprocess.call(
['sudo', 'apt-get', 'install', '-qq', i])
if return_code == 0:
break
if return_code != 0:
print('Cannot find any way to install docker', file=sys.stderr)
exit(1)
else:
programs += ['lxc', 'debootstrap']
subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs)
else:
if args.docker:
print("Warning: macOS support is experimental and requires docker be already installed", file=sys.stderr)
else:
sys.exit("macOS support only works in --docker mode")
if not os.path.isdir('gitian-builder'):
subprocess.check_call(
['git', 'clone', 'https://github.com/devrandom/gitian-builder.git'])
if not os.path.isdir('bitcoin-cash-node'):
subprocess.check_call(
['git', 'clone', 'https://gitlab.com/bitcoin-cash-node/bitcoin-cash-node.git'])
os.chdir('gitian-builder')
make_image_prog = ['bin/make-base-vm',
'--distro', 'debian', '--suite', 'buster', '--arch', 'amd64']
if args.docker:
make_image_prog += ['--docker']
elif not args.kvm:
make_image_prog += ['--lxc']
subprocess.check_call(make_image_prog)
os.chdir(workdir)
if args.is_bionic and not args.kvm and not args.docker:
subprocess.check_call(
['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net'])
print('Reboot is required')
exit(0)
def build():
global args, workdir
base_output_dir = 'bitcoin-binaries/' + args.version
os.makedirs(base_output_dir + '/src', exist_ok=True)
print('\nBuilding Dependencies\n')
os.chdir('gitian-builder')
os.makedirs('inputs', exist_ok=True)
subprocess.check_call(['make', '-C', '../bitcoin-cash-node/depends',
'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common'])
output_dir_src = '../' + base_output_dir + '/src'
if args.linux:
print('\nCompiling ' + args.version + ' Linux')
subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin=' + args.commit,
'--url', 'bitcoin=' + args.url, '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-linux.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version +
'-linux', '--destination', '../gitian.sigs/', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-linux.yml'])
output_dir_linux = '../' + base_output_dir + '/linux'
os.makedirs(output_dir_linux, exist_ok=True)
subprocess.check_call(
'mv build/out/bitcoin-*.tar.gz ' + output_dir_linux, shell=True)
subprocess.check_call(
'mv build/out/src/bitcoin-*.tar.gz ' + output_dir_src, shell=True)
subprocess.check_call(
'mv result/bitcoin-*-linux-res.yml ' + output_dir_linux, shell=True)
if args.windows:
print('\nCompiling ' + args.version + ' Windows')
subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin=' + args.commit,
'--url', 'bitcoin=' + args.url, '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-win.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version +
'-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-win.yml'])
output_dir_win = '../' + base_output_dir + '/win'
os.makedirs(output_dir_win, exist_ok=True)
subprocess.check_call(
'mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/', shell=True)
subprocess.check_call(
'mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ' + output_dir_win, shell=True)
subprocess.check_call(
'mv build/out/src/bitcoin-*.tar.gz ' + output_dir_src, shell=True)
subprocess.check_call(
'mv result/bitcoin-*-win-res.yml ' + output_dir_win, shell=True)
if args.macos:
print('\nCompiling ' + args.version + ' MacOS')
subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin=' + args.commit,
'--url', 'bitcoin=' + args.url, '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-osx.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version +
'-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-osx.yml'])
output_dir_osx = '../' + base_output_dir + '/osx'
os.makedirs(output_dir_osx, exist_ok=True)
subprocess.check_call(
'mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/', shell=True)
subprocess.check_call(
'mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ' + output_dir_osx, shell=True)
subprocess.check_call(
'mv build/out/src/bitcoin-*.tar.gz ' + output_dir_src, shell=True)
subprocess.check_call(
'mv result/bitcoin-*-osx-res.yml ' + output_dir_osx, shell=True)
os.chdir(workdir)
if args.commit_files:
print('\nCommitting ' + args.version + ' Unsigned Sigs\n')
os.chdir('gitian.sigs')
subprocess.check_call(
['git', 'add', args.version + '-linux/' + args.signer])
subprocess.check_call(
['git', 'add', args.version + '-win-unsigned/' + args.signer])
subprocess.check_call(
['git', 'add', args.version + '-osx-unsigned/' + args.signer])
subprocess.check_call(
['git', 'commit', '-m', 'Add ' + args.version + ' unsigned sigs for ' + args.signer])
os.chdir(workdir)
def sign():
global args, workdir
os.chdir('gitian-builder')
if args.windows:
print('\nSigning ' + args.version + ' Windows')
subprocess.check_call('cp inputs/bitcoin-' + args.version +
'-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True)
subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature=' + args.commit,
'../bitcoin-cash-node/contrib/gitian-descriptors/gitian-win-signer.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-win-signed',
'--destination', '../gitian.sigs/', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-win-signer.yml'])
subprocess.check_call(
'mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/' + args.version, shell=True)
if args.macos:
print('\nSigning ' + args.version + ' MacOS')
subprocess.check_call('cp inputs/bitcoin-' + args.version +
'-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True)
subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature=' + args.commit,
'../bitcoin-cash-node/contrib/gitian-descriptors/gitian-osx-signer.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version + '-osx-signed',
'--destination', '../gitian.sigs/', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-osx-signer.yml'])
subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/' +
args.version + '/bitcoin-' + args.version + '-osx.dmg', shell=True)
os.chdir(workdir)
if args.commit_files:
print('\nCommitting ' + args.version + ' Signed Sigs\n')
os.chdir('gitian.sigs')
subprocess.check_call(
['git', 'add', args.version + '-win-signed/' + args.signer])
subprocess.check_call(
['git', 'add', args.version + '-osx-signed/' + args.signer])
subprocess.check_call(['git', 'commit', '-a', '-m', 'Add ' +
args.version + ' signed binary sigs for ' + args.signer])
os.chdir(workdir)
def verify():
global args, workdir
os.chdir('gitian-builder')
print('\nVerifying v' + args.version + ' Linux\n')
subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version +
'-linux', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-linux.yml'])
print('\nVerifying v' + args.version + ' Windows\n')
subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version +
'-win-unsigned', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-win.yml'])
print('\nVerifying v' + args.version + ' MacOS\n')
subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version +
'-osx-unsigned', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-osx.yml'])
print('\nVerifying v' + args.version + ' Signed Windows\n')
subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version +
'-win-signed', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-win-signer.yml'])
print('\nVerifying v' + args.version + ' Signed MacOS\n')
subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version +
'-osx-signed', '../bitcoin-cash-node/contrib/gitian-descriptors/gitian-osx-signer.yml'])
os.chdir(workdir)
def main():
global args, workdir
num_cpus = multiprocessing.cpu_count()
parser = argparse.ArgumentParser(description='Script for running full Gitian builds.')
parser.add_argument('-c', '--commit', action='store_true', dest='commit',
help='Indicate that the version argument is for a commit or branch')
parser.add_argument('-R', '--merge-request', action='store_true', dest='merge_request',
help='Indicate that the version argument is the number of a GitLab merge request')
parser.add_argument('-u', '--url', dest='url', default='https://gitlab.com/bitcoin-cash-node/bitcoin-cash-node.git',
help='Specify the URL of the repository. Default is %(default)s')
parser.add_argument('-v', '--verify', action='store_true',
dest='verify', help='Verify the Gitian build')
parser.add_argument('-b', '--build', action='store_true',
dest='build', help='Do a Gitian build')
parser.add_argument('-s', '--sign', action='store_true', dest='sign',
help='Make signed binaries for Windows and MacOS')
parser.add_argument('-B', '--buildsign', action='store_true',
dest='buildsign', help='Build both signed and unsigned binaries')
parser.add_argument('-o', '--os', dest='os', default='lwm',
help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS')
parser.add_argument('-j', '--jobs', dest='jobs', default=str(num_cpus),
help='Number of processes to use. Default %(default)s')
parser.add_argument('-m', '--memory', dest='memory', default='3500',
help='Memory to allocate in MiB. Default %(default)s')
parser.add_argument('-k', '--kvm', action='store_true',
dest='kvm', help='Use KVM instead of LXC')
parser.add_argument('-d', '--docker', action='store_true',
dest='docker', help='Use Docker instead of LXC')
parser.add_argument('-S', '--setup', action='store_true', dest='setup',
help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)')
parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign',
help='Create the assert file for detached signing. Will not commit anything.')
parser.add_argument('-n', '--no-commit', action='store_false',
dest='commit_files', help='Do not commit anything to git')
parser.add_argument(
'signer', nargs='?', help='GPG signer to sign each build assert file')
parser.add_argument(
'version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified')
args = parser.parse_args()
workdir = os.getcwd()
args.is_bionic = not is_mac and b'bionic' in subprocess.check_output(
['lsb_release', '-cs'])
if args.kvm and args.docker:
raise Exception('Error: cannot have both kvm and docker')
# Set environment variable USE_LXC or USE_DOCKER, let gitian-builder know
# that we use lxc or docker
if args.docker:
os.environ.pop('USE_LXC', None)
os.environ['USE_DOCKER'] = '1'
elif args.kvm:
os.environ.pop('USE_LXC', None)
os.environ.pop('USE_DOCKER', None)
else:
os.environ['USE_LXC'] = '1'
os.environ.pop('USE_DOCKER', None)
if 'GITIAN_HOST_IP' not in os.environ.keys():
os.environ['GITIAN_HOST_IP'] = '10.0.3.1'
if 'LXC_GUEST_IP' not in os.environ.keys():
os.environ['LXC_GUEST_IP'] = '10.0.3.5'
if args.setup:
setup()
if args.buildsign:
args.build = True
args.sign = True
if not args.build and not args.sign and not args.verify:
sys.exit(0)
args.linux = 'l' in args.os
args.windows = 'w' in args.os
args.macos = 'm' in args.os
# Disable for MacOS if no SDK found
if args.macos and not os.path.isfile(
'gitian-builder/inputs/MacOSX10.15.sdk.tar.xz'):
print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
args.macos = False
args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
script_name = os.path.basename(sys.argv[0])
# Signer and version shouldn't be empty
if not args.signer:
print(script_name + ': Missing signer.')
print('Try ' + script_name + ' --help for more information')
exit(1)
if not args.version:
print(script_name + ': Missing version.')
print('Try ' + script_name + ' --help for more information')
exit(1)
# Add leading 'v' for tags
if args.commit and args.merge_request:
raise Exception('Cannot have both commit and merge request')
args.commit = ('' if args.commit else 'v') + args.version
# If this is the first time you run gitian_build you never had a chance of initialize
# the project repo inside gitian-builder directory, in fact this is usually done by
# gitian-builder/bin/gbuild ruby script.
if not os.path.isdir('gitian-builder/inputs/bitcoin'):
subprocess.check_call(['mkdir', '-p', 'gitian-builder/inputs'])
os.chdir('gitian-builder/inputs')
subprocess.check_call(['git', 'clone', args.url, 'bitcoin'])
# Move to bitcoin-cash-node folder
os.chdir(os.path.join(workdir, 'bitcoin-cash-node'))
# If merge request argument, the fetch the MR and get the head commit
if args.merge_request:
subprocess.check_call(
['git', 'fetch', args.url, 'merge-requests/' + args.version + '/head:mr-' + args.version])
args.commit = subprocess.check_output(
['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip()
# Still in bitcoin-cash-node/ ...
subprocess.check_call(['git', 'fetch'])
subprocess.check_call(['git', 'checkout', args.commit])
# Compare our own source code to bitcoin-cash-node version of gitian-build.py
# and raise a warning if it differs.
# Need to add newlines back in order to diff against file.
our_source = [line + '\n' for line in
inspect.getsource(inspect.getmodule(inspect.currentframe())).splitlines()]
with open('contrib/gitian-build.py', 'r', encoding='utf-8') as script_file:
script_source = script_file.readlines()
if our_source != script_source:
print(BOLD[1] + RED[1] + '*' * 70)
print("WARNING: gitian-build.py you are running differs from the one in\n"
" the source you are building!\n"
" If this is unexpected, please check whether you need to\n"
" copy the source tree's version to your top level build\n"
" directory.")
print('*' * 70 + RED[0] + BOLD[0])
# Update inputs/bitcoin/ to the commit to build
os.chdir(os.path.join(workdir, 'gitian-builder/inputs/bitcoin'))
subprocess.check_call(['git', 'fetch'])
if args.merge_request:
subprocess.check_call(
['git', 'fetch', args.url, 'merge-requests/' + args.version + '/head:mr-' + args.version])
args.version = 'mr-' + args.version
subprocess.check_call(['git', 'checkout', args.commit])
# Head back to work dir, show some diagnostics before kicking off the build.
os.chdir(workdir)
print("cwd:", os.getcwd())
print("commit:", args.commit)
if args.build:
build()
if args.sign:
sign()
if args.verify:
verify()
os.chdir(workdir)
if __name__ == '__main__':
main()