Skip to content

Commit

Permalink
Bootstrappable Builds
Browse files Browse the repository at this point in the history
  • Loading branch information
tobtoht committed Feb 24, 2024
1 parent 50b0744 commit 008cf5e
Show file tree
Hide file tree
Showing 48 changed files with 3,623 additions and 1,580 deletions.
49 changes: 0 additions & 49 deletions .github/workflows/gitian.yml

This file was deleted.

143 changes: 143 additions & 0 deletions .github/workflows/guix.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
name: ci/gh-actions/guix

on: [push, pull_request]

env:
APT_SET_CONF: |
echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
jobs:
cache-sources:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: depends sources cache
id: cache
uses: actions/cache@v4
with:
path: contrib/depends/sources
key: sources-${{ hashFiles('contrib/depends/packages/*') }}
- name: download depends sources
if: steps.cache.outputs.cache-hit != 'true'
run: make -C contrib/depends download

cache-guix:
runs-on: ubuntu-latest
needs: [cache-sources]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: guix cache
id: cache
uses: actions/cache@v4
with:
path: |
gnu/store
guix_db.sqlite
key: guix-${{ hashFiles('contrib/guix/manifest.scm') }}
- name: move guix store
if: steps.cache.outputs.cache-hit != 'true'
run: |
if [[ -e "guix_db.sqlite" ]]; then
sudo mkdir -p /var/guix/db
sudo mv gnu /gnu
sudo cp guix_db.sqlite /var/guix/db/db.sqlite
sudo chmod 1775 /gnu/store
sudo chown 0644 /var/guix/db/db.sqlite
sudo chown -R root:root /gnu/store /var/guix/db/db.sqlite
fi
- name: depends sources cache
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/restore@v4
with:
path: contrib/depends/sources
key: sources-${{ hashFiles('contrib/depends/packages/*') }}
- name: set apt conf
if: steps.cache.outputs.cache-hit != 'true'
run: ${{env.APT_SET_CONF}}
- name: install dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: sudo apt update; sudo apt -y install guix git ca-certificates
- name: dry run
if: steps.cache.outputs.cache-hit != 'true'
run: DRY_RUN=1 SUBSTITUTE_URLS='http://ci.guix.gnu.org' JOBS=2 ./contrib/guix/guix-build
- name: prepare guix store for caching
if: steps.cache.outputs.cache-hit != 'true'
run: |
sudo systemctl stop guix-daemon
sudo mv /gnu gnu
sudo mv /var/guix/db/db.sqlite guix_db.sqlite
build-guix:
runs-on: ubuntu-latest
needs: [cache-guix]
strategy:
fail-fast: false
matrix:
toolchain:
- target: "x86_64-linux-gnu"
- target: "aarch64-linux-gnu"
- target: "arm-linux-gnueabihf"
- target: "riscv64-linux-gnu"
- target: "i686-linux-gnu"
- target: "x86_64-w64-mingw32"
- target: "i686-w64-mingw32"
- target: "x86_64-apple-darwin"
- target: "aarch64-apple-darwin"
- target: "aarch64-linux-android"
- target: "arm-linux-androideabi"

name: ${{ matrix.toolchain.target }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
- name: remove bundled packages
run: sudo rm -rf /usr/local
- name: guix cache
uses: actions/cache/restore@v4
with:
path: |
gnu/store
guix_db.sqlite
key: guix-${{ hashFiles('contrib/guix/manifest.scm') }}
- name: move guix store
run: |
if [[ -e "guix_db.sqlite" ]]; then
sudo mkdir -p /var/guix/db
sudo mv gnu /gnu
sudo cp guix_db.sqlite /var/guix/db/db.sqlite
sudo chmod 1775 /gnu/store
sudo chown 0644 /var/guix/db/db.sqlite
sudo chown -R root:root /gnu/store /var/guix/db/db.sqlite
fi
- name: depends cache
uses: actions/cache@v4
with:
path: contrib/depends/built
key: depends-${{ matrix.toolchain.target }}-${{ hashFiles('contrib/depends/packages/*') }}
- name: depends sources cache
uses: actions/cache/restore@v4
with:
path: contrib/depends/sources
key: sources-${{ hashFiles('contrib/depends/packages/*') }}
- name: set apt conf
run: ${{env.APT_SET_CONF}}
- name: install dependencies
run: sudo apt update; sudo apt -y install guix git ca-certificates
- name: build
run: SUBSTITUTE_URLS='http://ci.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" JOBS=2 ./contrib/guix/guix-build
- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.toolchain.target }}
path: |
guix/guix-build-*/output/${{ matrix.toolchain.target }}/*
guix/guix-build-*/logs/${{ matrix.toolchain.target }}/*
6 changes: 2 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@ miniupnpcstrings.h
version/
ClangBuildAnalyzerSession.txt

# gitian
contrib/gitian/builder/
contrib/gitian/docker/
contrib/gitian/sigs/
# guix
/guix

# Created by https://www.gitignore.io

Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1113,7 +1113,7 @@ if(MINGW)
elseif(APPLE OR OPENBSD OR ANDROID)
set(EXTRA_LIBRARIES "")
elseif(FREEBSD)
set(EXTRA_LIBRARIES execinfo)
set(EXTRA_LIBRARIES execinfo elf)
elseif(DRAGONFLY)
find_library(COMPAT compat)
set(EXTRA_LIBRARIES execinfo ${COMPAT})
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers.
- [Release staging schedule and protocol](#release-staging-schedule-and-protocol)
- [Compiling Monero from source](#compiling-monero-from-source)
- [Dependencies](#dependencies)
- [Gitian builds](#gitian-builds)
- [Guix builds](#guix-builds)
- [Internationalization](#Internationalization)
- [Using Tor](#using-tor)
- [Pruning](#Pruning)
Expand Down Expand Up @@ -599,9 +599,9 @@ USE_DEVICE_TREZOR=OFF make release

For more information, please check out Trezor [src/device_trezor/README.md](src/device_trezor/README.md).

### Gitian builds
### Guix builds

See [contrib/gitian/README.md](contrib/gitian/README.md).
See [contrib/guix/README.md](contrib/guix/README.md).

## Installing Monero from a package

Expand Down
28 changes: 12 additions & 16 deletions contrib/depends/Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
.NOTPARALLEL :

# Pattern rule to print variables, e.g. make print-all_packages
print-%: FORCE
@echo '$($*)'

SOURCES_PATH ?= $(BASEDIR)/sources
BASE_CACHE ?= $(BASEDIR)/built
FALLBACK_DOWNLOAD_PATH ?= https://downloads.getmonero.org/depends-sources
Expand Down Expand Up @@ -81,19 +85,8 @@ include builders/$(build_os).mk
include builders/default.mk
include packages/packages.mk

build_id_string:=$(BUILD_ID_SALT)
build_id_string+=$(shell $(build_CC) --version 2>/dev/null)
build_id_string+=$(shell $(build_AR) --version 2>/dev/null)
build_id_string+=$(shell $(build_CXX) --version 2>/dev/null)
build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null)
build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null)

$(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT)
$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null)
$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null)
$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR) 'NM='$(build_NM)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' NM='$(host_NM)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')

packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages)
native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
Expand All @@ -102,12 +95,13 @@ all_packages = $(packages) $(native_packages)

meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk

$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils)
$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)

include funcs.mk

toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
final_build_id_long+=$(shell $(build_SHA256SUM) toolchain.cmake.in)
final_build_id_long+=:[sha256sum]:$(shell $(build_SHA256SUM) toolchain.cmake.in)
final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
$(AT)rm -rf $(@D)
Expand All @@ -120,8 +114,8 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
$(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(final_build_id)
$(AT)@mkdir -p $(@D)
$(AT)sed -e 's|@HOST@|$(host)|' \
-e 's|@CC@|$(toolchain_path)$(host_CC)|' \
-e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
-e 's|@CC@|$(host_CC)|' \
-e 's|@CXX@|$(host_CXX)|' \
-e 's|@AR@|$(toolchain_path)$(host_AR)|' \
-e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \
-e 's|@NM@|$(toolchain_path)$(host_NM)|' \
Expand All @@ -139,6 +133,7 @@ $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_
-e 's|@cmake_system_name@|$($(host_os)_cmake_system)|' \
-e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\
-e 's|@arch@|$(host_arch)|'\
-e 's|@guix_ldflags@|$(guix_ldflags)|'\
$< > $@
$(AT)touch $@

Expand Down Expand Up @@ -184,3 +179,4 @@ download: download-osx download-linux download-win download-freebsd download-and
$(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package))))

.PHONY: install cached download-one download-osx download-linux download-win download-freebsd download-android download check-packages check-sources
.PHONY: FORCE
45 changes: 27 additions & 18 deletions contrib/depends/funcs.mk
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
define int_vars
#Set defaults for vars which may be overridden per-package
$(1)_cc=$($($(1)_type)_CC)
$(1)_cxx=$($($(1)_type)_CXX)
$(1)_objc=$($($(1)_type)_OBJC)
$(1)_objcxx=$($($(1)_type)_OBJCXX)
$(1)_ar=$($($(1)_type)_AR)
$(1)_ranlib=$($($(1)_type)_RANLIB)
$(1)_libtool=$($($(1)_type)_LIBTOOL)
$(1)_nm=$($($(1)_type)_NM)
$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS)
$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS)
$(1)_arflags=$($($(1)_type)_ARFLAGS) $($($(1)_type)_$(release_type)_ARFLAGS)
$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib
$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include
$(1)_cc=$$($$($(1)_type)_CC)
$(1)_cxx=$$($$($(1)_type)_CXX)
$(1)_objc=$$($$($(1)_type)_OBJC)
$(1)_objcxx=$$($$($(1)_type)_OBJCXX)
$(1)_ar=$$($$($(1)_type)_AR)
$(1)_ranlib=$$($$($(1)_type)_RANLIB)
$(1)_libtool=$$($$($(1)_type)_LIBTOOL)
$(1)_nm=$$($$($(1)_type)_NM)
$(1)_cflags=$$($$($(1)_type)_CFLAGS) \
$$($$($(1)_type)_$$(release_type)_CFLAGS)
$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \
$$($$($(1)_type)_$$(release_type)_CXXFLAGS)
$(1)_arflags=$$($$($(1)_type)_ARFLAGS) \
$$($$($(1)_type)_$(release_type)_ARFLAGS)
$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \
$$($$($(1)_type)_$$(release_type)_LDFLAGS) \
-L$$($($(1)_type)_prefix)/lib
$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \
$$($$($(1)_type)_$$(release_type)_CPPFLAGS) \
-I$$($$($(1)_type)_prefix)/include
$(1)_recipe_hash:=
endef

Expand All @@ -37,16 +44,17 @@ endef

define int_get_build_recipe_hash
$(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1))
final_build_id_long+=:[$(1)_all_file_checksums]$(foreach checksum,$($(1)_all_file_checksums),$(shell echo ":$(checksum)")):
$(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1))
endef

define int_get_build_id
$(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
$(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string))
$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id))
$(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
final_build_id_long+=$($(package)_build_id_long)
final_build_id_long+=:[recipe]:$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type):[deps]$(foreach dep,$($(1)_build_id_deps),$(shell echo ":$(dep)")):[$($(1)_type)_id]:$($($(1)_type)_id):

#compute package-specific paths
$(1)_build_subdir?=.
Expand Down Expand Up @@ -252,7 +260,8 @@ $(foreach package,$(packages),$(eval $(package)_type=$(host_arch)_$(host_os)))
$(foreach package,$(all_packages),$(eval $(call int_vars,$(package))))

#include package files
$(foreach package,$(all_packages),$(eval include packages/$(package).mk))
$(foreach native_package,$(native_packages),$(eval include packages/$(native_package).mk))
$(foreach package,$(packages),$(eval include packages/$(package).mk))

#compute a hash of all files that comprise this package's build recipe
$(foreach package,$(all_packages),$(eval $(call int_get_build_recipe_hash,$(package))))
Expand All @@ -267,4 +276,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
$(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))

#special exception: if a toolchain package exists, all non-native packages depend on it
$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))
$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) ))

0 comments on commit 008cf5e

Please sign in to comment.