diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 034ddbe12f49d4..5d9e9b05705ca1 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -1,5 +1,15 @@ #!/usr/bin/env python3 - +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bytz Core Developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# how to use and guide: https://github.com/wagerr/docs/blob/master/gitian-building.md +# Setup: gitian-build.py --setup --build $SIGNER $VERSION 192.168.1.37 SHA256 wagerr-binaries +#usage: gitian-build.py [-h] [-c] [-p] [-u URL] [-v] [-b] [-s] [-B] [-o OS] +# [-j JOBS] [-m MEMORY] [-k] [-d] [-S] [-D] [-n] [-z] +# [-x SERVER] [-l] [-f UPLOADFOLDER] [-y HASH] +# [signer] [version] import argparse import os import subprocess @@ -7,10 +17,8 @@ def setup(): global args, workdir - programs = ['ruby', 'git', 'make', 'wget'] - if args.lxc: - programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] - elif args.kvm: + programs = ['ruby', 'git', 'make', 'wget', 'curl'] + if args.kvm: programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'): dockers = ['docker.io', 'docker-ce'] @@ -19,29 +27,31 @@ def setup(): if return_code == 0: break if return_code != 0: - print('Cannot find any way to install Docker', file=sys.stderr) - exit(1) + print('Cannot find any way to install Docker.', file=sys.stderr) + sys.exit(1) + else: + programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) if not os.path.isdir('gitian.sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/wagerr/gitian.sigs.git']) + subprocess.check_call(['git', 'clone', 'https://github.com/wagerr/gitian.sigs.git', 'gitian.sigs']) if not os.path.isdir('wagerr-detached-sigs'): subprocess.check_call(['git', 'clone', 'https://github.com/wagerr/wagerr-detached-sigs.git']) if not os.path.isdir('gitian-builder'): subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) if not os.path.isdir('wagerr'): - subprocess.check_call(['git', 'clone', 'https://github.com/wagerr/wagerr.git']) + subprocess.check_call(['git', 'clone', '--recurse-submodules', 'https://github.com/wagerr/wagerr.git']) os.chdir('gitian-builder') make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] if args.docker: make_image_prog += ['--docker'] - elif args.lxc: + elif not args.kvm: make_image_prog += ['--lxc'] subprocess.check_call(make_image_prog) os.chdir(workdir) if args.is_bionic and not args.kvm and not args.docker: subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') - exit(0) + sys.exit(0) def build(): global args, workdir @@ -51,7 +61,15 @@ def build(): os.chdir('gitian-builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) + if args.macos and not os.path.isfile('inputs/MacOSX10.11.sdk.tar.xz'): + subprocess.check_call(['wget', '-O', 'inputs/MacOSX10.11.sdk.tar.xz', '-N', '-P', 'inputs', 'https://github.com/wagerr/macosx-sdks/downloads/MacOSX10.11.sdk.tar.xz']) + + if args.macos and not os.path.isfile('inputs/MacOSX10.11.sdk.tar.xz'): + subprocess.check_call(['wget', '-O', 'inputs/MacOSX10.11.sdk.tar.xz', '-N', '-P', 'inputs', 'https://github.com/gitianuser/MacOSX-SDKs/releases/download/MacOSX10.11.sdk/MacOSX10.11.sdk.tar.xz']) + + if not os.path.isfile('inputs/osslsigncode-2.0.tar.gz'): + subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) + subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', '../wagerr/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) @@ -65,20 +83,25 @@ def build(): print('\nCompiling ' + args.version + ' Windows') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'wagerr='+args.commit, '--url', 'wagerr='+args.url, '../wagerr/contrib/gitian-descriptors/gitian-win.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../wagerr/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call('mv build/out/wagerr-*-win-unsigned.tar.gz inputs/wagerr-win-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/wagerr-*-win-unsigned.tar.gz inputs/', shell=True) subprocess.check_call('mv build/out/wagerr-*.zip build/out/wagerr-*.exe ../wagerr-binaries/'+args.version, shell=True) if args.macos: print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz']) - subprocess.check_output(["echo 'bec9d089ebf2e2dd59b1a811a38ec78ebd5da18cbbcd6ab39d1e59f64ac5033f inputs/MacOSX10.11.sdk.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'wagerr='+args.commit, '--url', 'wagerr='+args.url, '../wagerr/contrib/gitian-descriptors/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../wagerr/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call('mv build/out/wagerr-*-osx-unsigned.tar.gz inputs/wagerr-osx-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/wagerr-*-osx-unsigned.tar.gz inputs/', shell=True) subprocess.check_call('mv build/out/wagerr-*.tar.gz build/out/wagerr-*.dmg ../wagerr-binaries/'+args.version, shell=True) os.chdir(workdir) + if args.hash: + os.chdir('wagerr-binaries/'+args.version) + subprocess.check_call('sha'+args.hash+'sum wagerr* > SHA'+args.hash+'SUMS', shell=True) + subprocess.check_call('gpg -u '+args.signer+' --digest-algo sha'+args.hash+' --clearsign SHA'+args.hash+'SUMS', shell=True) + + os.chdir(workdir) + if args.commit_files: print('\nCommitting '+args.version+' Unsigned Sigs\n') os.chdir('gitian.sigs') @@ -88,12 +111,15 @@ def build(): subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) os.chdir(workdir) + os.chdir(workdir) + def sign(): global args, workdir os.chdir('gitian-builder') if args.windows: print('\nSigning ' + args.version + ' Windows') + subprocess.check_call('cp inputs/wagerr-' + args.version + '-win-unsigned.tar.gz inputs/wagerr-win-unsigned.tar.gz', shell=True) subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../wagerr/contrib/gitian-descriptors/gitian-win-signer.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../wagerr/contrib/gitian-descriptors/gitian-win-signer.yml']) subprocess.check_call('mv build/out/wagerr-*win64-setup.exe ../wagerr-binaries/'+args.version, shell=True) @@ -101,6 +127,7 @@ def sign(): if args.macos: print('\nSigning ' + args.version + ' MacOS') + subprocess.check_call('cp inputs/wagerr-' + args.version + '-osx-unsigned.tar.gz inputs/wagerr-osx-unsigned.tar.gz', shell=True) subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../wagerr/contrib/gitian-descriptors/gitian-osx-signer.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../wagerr/contrib/gitian-descriptors/gitian-osx-signer.yml']) subprocess.check_call('mv build/out/wagerr-osx-signed.dmg ../wagerr-binaries/'+args.version+'/wagerr-'+args.version+'-osx.dmg', shell=True) @@ -115,27 +142,99 @@ def sign(): subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) os.chdir(workdir) +def createhashes(): + global args, workdir + os.chdir(workdir) + os.chdir('wagerr-binaries/'+args.version) + subprocess.check_call('sha'+args.hash+'sum wagerr* > SHA'+args.hash+'SUMS', shell=True) + subprocess.check_call('gpg -u '+args.signer+' --digest-algo sha'+args.hash+' --clearsign SHA'+args.hash+'SUMS', shell=True) + subprocess.check_call('rm', '-f', '/SHA'+args.hash+'SUMS', shell=True) + os.chdir(workdir) + +def sshupload(): + global args, workdir + os.chdir(workdir) + print('\n'+args.server+': Start uploading all files to the uploadserver.\n') + subprocess.check_call(['ssh', args.server, 'mkdir', '-p', args.uploadfolder+'/'+args.version]) + subprocess.check_call(['scp', '-r', args.uploadfolder+'/'+args.version, args.server+':'+args.uploadfolder+'/'+args.version]) + os.chdir(workdir) + +def logsupload(): + global args, workdir + os.chdir(workdir) + + if args.uploadlogs: + print('\n'+args.server+': Start uploading logs to the uploadserver.\n') + subprocess.check_call(['ssh', args.server, 'mkdir', '-p', args.uploadfolder+'/'+args.version+'/logs']) + subprocess.check_call(['ssh', args.server, 'mkdir', '-p', args.uploadfolder+'/'+args.version+'/result']) + subprocess.check_call(['scp', '-r', 'gitian-builder/var', args.server+':'+args.uploadfolder+'/'+args.version+'/logs']) + subprocess.check_call(['scp', 'gitian-builder/result/wagerr-*.yml', args.server+':'+args.uploadfolder+'/'+args.version+'/result']) + + if args.createreleasenotes: + print('\n'+args.server+': Start uploading release notes to the uploadserver.\n') + subprocess.check_call(['ssh', args.server, 'mkdir', '-p', args.uploadfolder+'/'+args.version+'/release-notes']) + subprocess.check_call(['scp', '-r', args.uploadfolder+'/'+args.version+'/release-notes', args.server+':'+args.uploadfolder+'/'+args.version+'/release-notes']) + + os.chdir(workdir) + +def releasenotes(): + global args, workdir + os.chdir(workdir) + + if not os.path.isdir(args.uploadfolder+'/'+args.version+'/release-notes'): + subprocess.check_call('mkdir -p '+args.uploadfolder+'/'+args.version+'/release-notes', shell=True) + + os.chdir('wagerr') + + if args.previousver: + # Create shortlog notes + subprocess.check_call('git shortlog --no-merges v'+args.previousver+'..v'+args.version+' > ../wagerr-binaries/'+args.version+'/release-notes/shortlog_v'+args.previousver+'..v'+args.version+'.md', shell=True) + # Create changes notes + subprocess.check_call('git log --oneline v'+args.previousver+'..v'+args.version+' > ../wagerr-binaries/'+args.version+'/release-notes/changes_'+args.previousver+'-v'+args.version+'.md', shell=True) + # Create authors notes + subprocess.check_call('git log --format='"'- %aN' v'+args.previousver+'..v'+args.version+' | sort -fiu > ../ion-binaries/"+args.version+'/release-notes/authors_'+args.previousver+'-v'+args.version+'.md', shell=True) + # Create detailed notes + subprocess.check_call('git log v'+args.previousver+'..v'+args.version+' --pretty="format:%at %C(yellow)commit %H%Creset\nAuthor: %an <%ae>\nDate: %aD\n\n %s\n" | sort -r | cut -d" " -f2- | sed -e "s/\\\n/\\`echo -e '+"'"+"\n\r'`/g"+'" | tr -d '+"'\15\32'"+' | less -R > ../wagerr-binaries/'+args.version+'/release-notes/detailedlog_'+args.previousver+'-v'+args.version+'.md', shell=True) + + os.chdir(workdir) + def verify(): global args, workdir + rc = 0 os.chdir('gitian-builder') print('\nVerifying v'+args.version+' Linux\n') - subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../wagerr/contrib/gitian-descriptors/gitian-linux.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../wagerr/contrib/gitian-descriptors/gitian-linux.yml']): + print('Verifying v'+args.version+' Linux FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' Windows\n') - subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../wagerr/contrib/gitian-descriptors/gitian-win.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../wagerr/contrib/gitian-descriptors/gitian-win.yml']): + print('Verifying v'+args.version+' Windows FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' MacOS\n') - subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../wagerr/contrib/gitian-descriptors/gitian-osx.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../wagerr/contrib/gitian-descriptors/gitian-osx.yml']): + print('Verifying v'+args.version+' MacOS FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' Signed Windows\n') - subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../wagerr/contrib/gitian-descriptors/gitian-win-signer.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../wagerr/contrib/gitian-descriptors/gitian-win-signer.yml']): + print('Verifying v'+args.version+' Signed Windows FAILED\n') + rc = 1 + print('\nVerifying v'+args.version+' Signed MacOS\n') - subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../wagerr/contrib/gitian-descriptors/gitian-osx-signer.yml']) + if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../wagerr/contrib/gitian-descriptors/gitian-osx-signer.yml']): + print('Verifying v'+args.version+' Signed MacOS FAILED\n') + rc = 1 os.chdir(workdir) + return rc def main(): global args, workdir - parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser = argparse.ArgumentParser(description='Script for running full Gitian builds.') parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') parser.add_argument('-u', '--url', dest='url', default='https://github.com/wagerr/wagerr', help='Specify the URL of the repository. Default is %(default)s') @@ -146,84 +245,97 @@ def main(): parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') - parser.add_argument('-V', '--virtualization', dest='virtualization', default='docker', help='Specify virtualization technology to use: lxc for LXC, kvm for KVM, docker for Docker. Default is %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)') parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', help='GPG signer to sign each build assert file') - parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') - + parser.add_argument('-z', '--no-upload', action='store_false', dest='upload', help='If upload is enabled, files will be uploaded to defined server. If not specified, upload is enabled by default') + parser.add_argument('-x', '--server', dest='server', default='defaultuploadserver', help='Use scp to upload file to the server, defines in .ssh as uploadserver, pass serverIp and path to ssh private key. Default is which is configured in your ~/.ssh/config file') + parser.add_argument('-l', '--uploadlogs', action='store_true', dest='uploadlogs', help='Upload logs and scripts (var folder)') + parser.add_argument('-f', '--uploadfolder', dest='uploadfolder', default='ion-binaries', help='Upload folder on uploadserver') + parser.add_argument('-y', '--hash', dest='hash', default='256', help='Create SHA hashes, choose beetwen SHA1, SHA256, SHA512') + parser.add_argument('-r', '--createreleasenotes', action='store_true', dest='createreleasenotes', help='Create release notes and changes to previous version, previous version variable must be set') + parser.add_argument('-P', '--previousver', dest='previousver', default='unset', help='Previous version for release notes, authors etc.') + parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') + parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') args = parser.parse_args() workdir = os.getcwd() - args.linux = 'l' in args.os - args.windows = 'w' in args.os - args.macos = 'm' in args.os - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) - if args.buildsign: - args.build = True - args.sign = True - - args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' - - args.lxc = (args.virtualization == 'lxc') - args.kvm = (args.virtualization == 'kvm') - args.docker = (args.virtualization == 'docker') + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') - script_name = os.path.basename(sys.argv[0]) - # Set all USE_* environment variables for gitian-builder: USE_LXC, USE_DOCKER and USE_VBOX + # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they + # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm). + os.environ['USE_LXC'] = '' os.environ['USE_VBOX'] = '' - if args.lxc: + os.environ['USE_DOCKER'] = '' + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: os.environ['USE_LXC'] = '1' - os.environ['USE_DOCKER'] = '' if 'GITIAN_HOST_IP' not in os.environ.keys(): os.environ['GITIAN_HOST_IP'] = '10.0.3.1' if 'LXC_GUEST_IP' not in os.environ.keys(): os.environ['LXC_GUEST_IP'] = '10.0.3.5' - elif args.kvm: - os.environ['USE_LXC'] = '' - os.environ['USE_DOCKER'] = '' - elif args.docker: - os.environ['USE_LXC'] = '' - os.environ['USE_DOCKER'] = '1' - else: - print(script_name+': Wrong virtualization option.') - print('Try '+script_name+' --help for more information') - exit(1) - # Signer and version shouldn't be empty - if args.signer == '': - print(script_name+': Missing signer.') + # Script will fail to automaticly download all resources if inputs folder does not exist + subprocess.check_call(['mkdir', '-p', 'gitian-builder/inputs']) + + if args.setup: + setup() + + if args.buildsign: + args.build = True + args.sign = True + + if not args.build and not args.sign and not args.verify: + sys.exit(0) + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.xz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + script_name = os.path.basename(sys.argv[0]) + if not args.signer: + print(script_name+': Missing signer') print('Try '+script_name+' --help for more information') - exit(1) - if args.version == '': - print(script_name+': Missing version.') + sys.exit(1) + if not args.version: + print(script_name+': Missing version') print('Try '+script_name+' --help for more information') - exit(1) - + sys.exit(1) # Add leading 'v' for tags if args.commit and args.pull: raise Exception('Cannot have both commit and pull') args.commit = ('' if args.commit else 'v') + args.version - if args.setup: - setup() - - if not args.build and not args.sign and not args.verify: - exit(0) - os.chdir('wagerr') if args.pull: subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + subprocess.check_call(['git', 'submodule', 'update', '--init', '--recursive']) os.chdir('../gitian-builder/inputs/wagerr') subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() args.version = 'pull-' + args.version + subprocess.check_call(['git', 'submodule', 'update', '--init', '--recursive']) print(args.commit) subprocess.check_call(['git', 'fetch']) subprocess.check_call(['git', 'checkout', args.commit]) + subprocess.check_call(['git', 'submodule', 'update', '--init', '--recursive']) + os.chdir(workdir) + + os.chdir('gitian-builder') + subprocess.check_call(['git', 'pull']) os.chdir(workdir) if args.build: @@ -233,7 +345,22 @@ def main(): sign() if args.verify: - verify() + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'pull']) + os.chdir(workdir) + sys.exit(verify()) + + if args.hash: + createhashes() + + if args.createreleasenotes: + releasenotes() + + if args.uploadlogs: + createhashes() + + if args.upload: + sshupload() if __name__ == '__main__': main() diff --git a/contrib/gitian-descriptors/README.md b/contrib/gitian-descriptors/README.md index ae894bdffd5136..6753a52bd70387 100644 --- a/contrib/gitian-descriptors/README.md +++ b/contrib/gitian-descriptors/README.md @@ -1,4 +1,4 @@ -### Gavin's notes on getting Gitian builds up and running using KVM +### Notes on getting Gitian builds up and running These instructions distilled from [https://help.ubuntu.com/community/KVM/Installation](https://help.ubuntu.com/community/KVM/Installation). @@ -45,11 +45,11 @@ If your main machine is a 64-bit Mac or PC with a few gigabytes of memory and at least 10 gigabytes of free disk space, you can `gitian-build` using LXC running inside a virtual machine. -Here's a description of Gavin's setup on OSX 10.6: +Here's a description of a setup on OSX 10.6: 1. Download and install VirtualBox from [https://www.virtualbox.org/](https://www.virtualbox.org/) -2. Download the 64-bit Ubuntu Desktop 12.04 LTS .iso CD image from +2. Download the 64-bit Ubuntu Desktop 20.04 LTS .iso CD image from [http://www.ubuntu.com/](http://www.ubuntu.com/) 3. Run VirtualBox and create a new virtual machine, using the Ubuntu .iso (see the [VirtualBox documentation](https://www.virtualbox.org/wiki/Documentation) for details). Create it with at least 2 gigabytes of memory and a disk that is at least 20 gigabytes big. diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 215633b09b0024..c28041858a7901 100755 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,6 +1,7 @@ --- -name: "wagerr-linux-4.0" +name: "wagerr-linux-4.0.99" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -8,15 +9,27 @@ architectures: packages: - "curl" - "g++-aarch64-linux-gnu" -- "g++-7-aarch64-linux-gnu" -- "gcc-7-aarch64-linux-gnu" +- "g++-8-aarch64-linux-gnu" +- "gcc-8-aarch64-linux-gnu" - "binutils-aarch64-linux-gnu" - "g++-arm-linux-gnueabihf" -- "g++-7-arm-linux-gnueabihf" -- "gcc-7-arm-linux-gnueabihf" +- "g++-8-arm-linux-gnueabihf" +- "gcc-8-arm-linux-gnueabihf" - "binutils-arm-linux-gnueabihf" -- "g++-7-multilib" -- "gcc-7-multilib" +- "g++-riscv64-linux-gnu" +- "g++-8-riscv64-linux-gnu" +- "gcc-8-riscv64-linux-gnu" +- "binutils-riscv64-linux-gnu" +- "g++-powerpc64le-linux-gnu" +- "g++-8-powerpc64le-linux-gnu" +- "gcc-8-powerpc64le-linux-gnu" +- "binutils-powerpc64le-linux-gnu" +- "g++-i686-linux-gnu" +- "g++-8-i686-linux-gnu" +- "gcc-8-i686-linux-gnu" +- "binutils-i686-linux-gnu" +- "g++-8-multilib" +- "gcc-8-multilib" - "binutils-gold" - "git" - "pkg-config" @@ -26,20 +39,21 @@ packages: - "faketime" - "bsdmainutils" - "ca-certificates" -- "python" - "python3" - "libxkbcommon0" -- "ccache" +- "linux-libc-dev" +- "doxygen" remotes: - "url": "https://github.com/wagerr/wagerr.git" "dir": "wagerr" files: [] script: | + set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" + HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu" # powerpc64le-linux-gnu riscv64-linux-gnu s390x-linux-gnu powerpc64-linux-gnu mipsel-linux-gnu mips-linux-gnu CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks" - FAKETIME_HOST_PROGS="" + FAKETIME_HOST_PROGS="gcc g++" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" @@ -54,54 +68,33 @@ script: | mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE}/depends + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - - # Setup ccache to use correct cache directories and fix the compiler check of ccache - CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache" - export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache - # As we later wrap the gcc binaries, this is fast - export CCACHE_COMPILERCHECK="content" - if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then - pushd ${GBUILD_PACKAGE_CACHE} - tar xf ccache.tar - rm ccache.tar - popd - fi - # instead of compressing ccache.tar, we let ccache handle it by itself - # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use - export CCACHE_COMPRESS=1 - else - CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache" fi - # We include the GCC version in all wrappers so that ccache can detect compiler upgrades when hashing the wrappers - GCCVERSION=`gcc --version | head -1` - function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog} + if which ${i}-${prog}-8 + then + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + fi done done } @@ -127,8 +120,7 @@ script: | rm -f ${WRAP_DIR}/${prog} cat << EOF > ${WRAP_DIR}/${prog} #!/usr/bin/env bash - # GCCVERSION=${GCCVERSION} - REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`" + REAL="`which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1`" for var in "\$@" do if [ "\$var" = "-m32" ]; then @@ -140,7 +132,6 @@ script: | \$REAL \$@ EOF chmod +x ${WRAP_DIR}/${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog} done cd wagerr @@ -192,27 +183,22 @@ script: | mkdir src/obj cp ../src/obj/build.h src/obj/ - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - - #TODO: This is a quick hack that disables symbol checking for arm. - # Instead, we should investigate why these are popping up. - # For aarch64, we'll need to bump up the min GLIBC version, as the abi - # support wasn't introduced until 2.17. case $i in aarch64-*) : ;; arm-*) : ;; *) make ${MAKEOPTS} -C src check-symbols ;; esac - make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + #find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + cp ../doc/README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ @@ -220,11 +206,3 @@ script: | done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src - - # Compress ccache (otherwise the assert file will get too huge) - if [ "$CCACHE_DIR" != "" ]; then - pushd ${GBUILD_PACKAGE_CACHE} - tar cf ccache.tar ccache - rm -rf ccache - popd - fi diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 775e3760190e93..4c16a63f01b7c9 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -1,5 +1,6 @@ --- name: "wagerr-dmg-signer" +distro: "ubuntu" suites: - "bionic" architectures: @@ -12,24 +13,22 @@ remotes: files: - "wagerr-osx-unsigned.tar.gz" script: | + set -e -o pipefail WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} - export PATH=`pwd`:$PATH + export PATH="$PWD":$PATH FAKETIME_PROGS="dmg genisoimage" - # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done - UNSIGNED=wagerr-osx-unsigned.tar.gz SIGNED=wagerr-osx-signed.dmg - tar -xf ${UNSIGNED} OSX_VOLNAME="$(cat osx_volname)" ./detached-sig-apply.sh ${UNSIGNED} signature/osx diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 91395611e12340..1c1a3a15374a40 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,6 +1,7 @@ --- -name: "wagerr-osx-4.0" +name: "wagerr-osx-4.0.99" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -22,22 +23,22 @@ packages: - "libcap-dev" - "libz-dev" - "libbz2-dev" -- "python" -- "python-dev" -- "python3" - "python3-dev" - "python3-setuptools" +- "python3-biplist" - "fonts-tuffy" -- "ccache" +- "doxygen" remotes: - "url": "https://github.com/wagerr/wagerr.git" "dir": "wagerr" files: -- "MacOSX10.11.sdk.tar.gz" +- "MacOSX10.11.sdk.tar.xz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin14" - CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks" + HOSTS="x86_64-apple-darwin16" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" @@ -50,23 +51,8 @@ script: | mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE}/depends + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - - # Setup ccache to use correct cache directories - CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache" - export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache - if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then - pushd ${GBUILD_PACKAGE_CACHE} - tar xf ccache.tar - rm ccache.tar - popd - fi - # instead of compressing ccache.tar, we let ccache handle it by itself - # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use - export CCACHE_COMPRESS=1 - else - CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache" fi export ZERO_AR_DATE=1 @@ -79,7 +65,6 @@ script: | echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog} done } @@ -92,7 +77,6 @@ script: | echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog} done done } @@ -107,7 +91,7 @@ script: | BASEPREFIX=`pwd`/depends mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xJvf ${BUILD_DIR}/MacOSX10.11.sdk.tar.xz # Build dependencies for each host for i in $HOSTS; do @@ -152,9 +136,8 @@ script: | mkdir src/obj cp ../src/obj/build.h src/obj/ - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} - make -C src osx_debug make install-strip DESTDIR=${INSTALLPATH} make osx_volname @@ -179,20 +162,9 @@ script: | find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find .. -name *.dSYM -exec cp -ra {} ${DISTNAME}/bin \; - find ${DISTNAME} -not -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz + find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz cd ../../ done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin14.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin14-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz - - # Compress ccache (otherwise the assert file will get too huge) - if [ "$CCACHE_DIR" != "" ]; then - pushd ${GBUILD_PACKAGE_CACHE} - tar cf ccache.tar ccache - rm -rf ccache - popd - fi + mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 4dbf69dc6f5492..ad941ab6e063ae 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -1,5 +1,6 @@ --- name: "wagerr-win-signer" +distro: "ubuntu" suites: - "bionic" architectures: @@ -17,7 +18,9 @@ files: - "osslsigncode-2.0.tar.gz" - "wagerr-win-unsigned.tar.gz" script: | - BUILD_DIR=`pwd` + set -e -o pipefail + + BUILD_DIR="$PWD" SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned @@ -34,6 +37,6 @@ script: | make find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do INFILE="`basename "${i}"`" - OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`" + OUTFILE="${INFILE/-unsigned}" ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml old mode 100755 new mode 100644 index f8cfcfeae1a393..dd39dd1d0e7edb --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,6 +1,7 @@ --- -name: "wagerr-win-4.0" +name: "wagerr-win-4.0.99" enable_cache: true +distro: "ubuntu" suites: - "bionic" architectures: @@ -20,18 +21,19 @@ packages: - "nsis" - "zip" - "ca-certificates" -- "python" - "python3" - "rename" -- "ccache" +- "doxygen" remotes: - "url": "https://github.com/wagerr/wagerr.git" "dir": "wagerr" files: [] script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests --enable-crash-hooks" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g" @@ -46,40 +48,18 @@ script: | mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE}/depends + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - - # Setup ccache to use correct cache directories and fix the compiler check of ccache - CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache" - export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache - # As we later wrap the gcc binaries, this is fast - export CCACHE_COMPILERCHECK="content" - if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then - pushd ${GBUILD_PACKAGE_CACHE} - tar xf ccache.tar - rm ccache.tar - popd - fi - # instead of compressing ccache.tar, we let ccache handle it by itself - # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use - export CCACHE_COMPRESS=1 - else - CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache" fi - # We include the GCC version in all wrappers so that ccache can detect compiler upgrades when hashing the wrappers - GCCVERSION=`gcc --version | head -1` - function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${prog} done } @@ -87,13 +67,11 @@ script: | for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog} done done } @@ -105,25 +83,19 @@ script: | mkdir -p ${WRAP_DIR}/${i} for prog in collect2; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} - echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}/${prog} REAL=$(${i}-gcc -print-prog-name=${prog}) echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} chmod +x ${WRAP_DIR}/${i}/${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}/${prog} done for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "# GCCVERSION=${GCCVERSION}" >> ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo '# Add the gcc version to the wrapper so that ccache takes this into account (we use CCACHE_COMPILERCHECK=content)' >> ${WRAP_DIR}/${i}-${prog} - echo "# `${prog} --version | head -1`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} - touch -d "${REFERENCE_DATETIME}" ${WRAP_DIR}/${i}-${prog} done done } @@ -183,7 +155,7 @@ script: | mkdir src/obj cp ../src/obj/build.h src/obj/ - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make deploy @@ -191,12 +163,12 @@ script: | rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe cp -f wagerr-*setup*.exe $OUTDIR/ cd installed - mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete + # mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ # temporarily disabled for Zerocoin + # find . -name "lib*.la" -delete # temporarily disabled for Zerocoin + # find . -name "lib*.a" -delete # temporarily disabled for Zerocoin rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; - find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; + # find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip cd ../../ @@ -206,16 +178,8 @@ script: | cd $BUILD_DIR/windeploy mkdir unsigned cp $OUTDIR/wagerr-*setup-unsigned.exe unsigned/ - find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz + find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9 > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip - - # Compress ccache (otherwise the assert file will get too huge) - if [ "$CCACHE_DIR" != "" ]; then - pushd ${GBUILD_PACKAGE_CACHE} - tar cf ccache.tar ccache - rm -rf ccache - popd - fi