Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

depends: Explicitly download and vendor Rust dependencies #3096

Merged
merged 1 commit into from Mar 28, 2018
Merged

depends: Explicitly download and vendor Rust dependencies #3096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion depends/Makefile
View file
Open in desktop
Expand Up @@ -13,6 +13,7 @@ BASEDIR = $(CURDIR)
HASH_LENGTH:=11
DOWNLOAD_CONNECT_TIMEOUT:=10
DOWNLOAD_RETRIES:=3
CRATE_REGISTRY:=vendored-sources

host:=$(BUILD)
ifneq ($(HOST),)
Expand Down Expand Up @@ -79,7 +80,7 @@ packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(rust_pa
native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
all_packages = $(packages) $(native_packages)

meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk cargo-checksum.sh

$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)

Expand Down
13 changes: 13 additions & 0 deletions depends/cargo-checksum.sh
View file
Open in desktop
@@ -0,0 +1,13 @@
echo "{\"files\":{$(
find . -type f | # Get list of file paths
grep -v $1 | # Exclude Makefile hashes
grep -v .stamp_ | # Exclude Makefile stamps
sed 's|^./||' | # Remove leading ./
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Spotted a bug post-merge: this is supposed to match only leading "./", but actually matches any leading character followed by "/" as the second character.

$ echo 'X/foo' |sed 's|^./||'  # incorrect
foo
$ echo 'X/foo' |sed 's|^[.]/||'  # correct
X/foo
$ echo './foo' |sed 's|^[.]/||'  # correct
foo

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similarly, the previous line should be grep -v '[.]stamp_' |.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Opened #3131.

sort | # Sort (for uniformity)
xargs $2 | # Get SHA256 hashes (assumes standard 'H(A) A' format)
awk -v OFS='":"' '{print $2, $1}' | # 'H(A) A' -> 'A":"H(A)'
sed 's|^|"|' | # 'A":"H(A)' -> '"A":"H(A)'
sed 's|$|"|' | # '"A":"H(A)' -> '"A":"H(A)"'
tr '\n' ',' | # Concatenate lines with commas
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think backslash-escaping within backticks is nonportable (POSIX allows the backslash sequence to be expanded early).

http://pubs.opengroup.org/onlinepubs/9699919799/xrat/V4_xcu_chap02.html#tag_23_02_06_03

Copy link
Contributor

@daira daira Mar 23, 2018 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See also https://stackoverflow.com/questions/43838007/posix-shell-escaping-line-continuations-in-backquote-command-substitutions (the question isn't quite the same but the answer is relevant).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used backticks originally because of the Makefile dollar-symbol issue. I'll change it now.

sed 's|,$||' # Remove any trailing comma (to fit JSON spec)
)},\"package\":\"$3\"}" > .cargo-checksum.json
12 changes: 12 additions & 0 deletions depends/funcs.mk
View file
Open in desktop
Expand Up @@ -30,6 +30,18 @@ define fetch_file
rm -rf $$($(1)_download_dir) ))
endef

define generate_crate_checksum
rm .gitignore && \
$(BASEDIR)/cargo-checksum.sh "$($(1)_file_name)" "$(build_SHA256SUM)" "$($(1)_sha256_hash)"
endef

define vendor_crate_source
mkdir -p $($(1)_staging_prefix_dir)/$(CRATE_REGISTRY) && \
cp -r $($(1)_extract_dir) $($(1)_staging_prefix_dir)/$(CRATE_REGISTRY)/$($(1)_crate_name) && \
cd $($(1)_staging_prefix_dir)/$(CRATE_REGISTRY)/$($(1)_crate_versioned_name) && \
rm -r `basename $($(1)_patch_dir)` .stamp_* .$($(1)_file_name).hash
endef

define int_get_build_recipe_hash
$(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1))
$(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1))
Expand Down
15 changes: 15 additions & 0 deletions depends/packages/crate_libc.mk
View file
Open in desktop
@@ -0,0 +1,15 @@
package=crate_libc
$(package)_crate_name=libc
$(package)_version=0.2.21
$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name)
$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate
$(package)_sha256_hash=88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135
$(package)_crate_versioned_name=$($(package)_crate_name)

define $(package)_preprocess_cmds
$(call generate_crate_checksum,$(package))
endef

define $(package)_stage_cmds
$(call vendor_crate_source,$(package))
endef
10 changes: 8 additions & 2 deletions depends/packages/librustzcash.mk
View file
Open in desktop
Expand Up @@ -5,10 +5,16 @@ $(package)_file_name=$(package)-$($(package)_git_commit).tar.gz
$(package)_download_file=$($(package)_git_commit).tar.gz
$(package)_sha256_hash=a5760a90d4a1045c8944204f29fa2a3cf2f800afee400f88bf89bbfe2cce1279
$(package)_git_commit=91348647a86201a9482ad4ad68398152dc3d635e
$(package)_dependencies=rust
$(package)_dependencies=rust $(rust_crates)
$(package)_patches=cargo.config

define $(package)_preprocess_cmds
mkdir .cargo && \
cat $($(package)_patch_dir)/cargo.config | sed 's|CRATE_REGISTRY|$(host_prefix)/$(CRATE_REGISTRY)|' > .cargo/config
endef

define $(package)_build_cmds
cargo build --release
cargo build --frozen --release
endef

define $(package)_stage_cmds
Expand Down
3 changes: 2 additions & 1 deletion depends/packages/packages.mk
View file
Open in desktop
@@ -1,4 +1,5 @@
rust_packages := rust librustzcash
rust_crates := crate_libc
rust_packages := rust $(rust_crates) librustzcash
proton_packages := proton
zcash_packages := libgmp libsodium
packages := boost openssl libevent zeromq $(zcash_packages) googletest
Expand Down
5 changes: 5 additions & 0 deletions depends/patches/librustzcash/cargo.config
View file
Open in desktop
@@ -0,0 +1,5 @@
[source.crates-io]
replace-with = "vendored-sources"

[source.vendored-sources]
directory = "CRATE_REGISTRY"