From f49f245675f3b9aa51c002977835859dfd1d5da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Tue, 26 Dec 2023 22:05:00 +0200 Subject: [PATCH 1/8] Undo breaking changes of (v4) -> (v5). --- Dockerfile | 8 ++++---- multiversx_sdk_rust_contract_builder/constants.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1b26fb6..6d6d03a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,11 @@ FROM ubuntu:22.04 # Constants ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v5.3.0" -ARG VERSION_RUST="nightly-2023-05-26" -ARG VERSION_BINARYEN="version_112" +ARG VERSION_RUST="nightly-2022-10-16" +ARG VERSION_BINARYEN="version_105" ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz" ARG VERSION_WABT="1.0.27-1" -ARG VERSION_SC_META="0.43.3" +ARG VERSION_SC_META="0.39.5" ARG TARGETPLATFORM # Install system dependencies @@ -39,7 +39,7 @@ RUN wget -O rustup.sh https://sh.rustup.rs && \ rm -rf /rust/registry # Install sc-tool -RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install multiversx-sc-meta --version ${VERSION_SC_META} && \ +RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install multiversx-sc-meta --locked --version ${VERSION_SC_META} && \ rm -rf /rust/registry COPY "multiversx_sdk_rust_contract_builder" "/multiversx_sdk_rust_contract_builder" diff --git a/multiversx_sdk_rust_contract_builder/constants.py b/multiversx_sdk_rust_contract_builder/constants.py index b6e135b..1c476ae 100644 --- a/multiversx_sdk_rust_contract_builder/constants.py +++ b/multiversx_sdk_rust_contract_builder/constants.py @@ -1,7 +1,7 @@ from pathlib import Path # The default value is the legacy one, to avoid breaking reproducibility of previous builds. -DEFAULT_BUILD_ROOT = "/tmp/project" +DEFAULT_BUILD_ROOT = "/tmp/elrond-contract-rust" HARDCODED_UNWRAP_FOLDER = Path("/tmp/unwrapped") ONE_KB_IN_BYTES: int = 1024 MAX_PACKAGED_SOURCE_CODE_SIZE: int = 2 * ONE_KB_IN_BYTES * 1024 From f3f37df8a4abc26d23d72df7680ebc04f3317df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 10:00:35 +0200 Subject: [PATCH 2/8] Fix inclusion of all files in project (if applicable). --- multiversx_sdk_rust_contract_builder/source_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiversx_sdk_rust_contract_builder/source_code.py b/multiversx_sdk_rust_contract_builder/source_code.py index 0273432..6157b94 100644 --- a/multiversx_sdk_rust_contract_builder/source_code.py +++ b/multiversx_sdk_rust_contract_builder/source_code.py @@ -50,7 +50,7 @@ def get_source_code_files( files_related_to_contract = set(file.path for file in source_code_files) if include_unrelated_to_contract: - all_files = get_all_files(contract_folder, _is_source_code_file) + all_files = get_all_files(project_folder, _is_source_code_file) for file in all_files: if file not in files_related_to_contract: source_code_files.append(SourceCodeFile(file, contract_folder, sys.maxsize)) From 0fd997b9d7e720c192d7f59f2bcfc3c233896b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 10:03:39 +0200 Subject: [PATCH 3/8] Add support for the flag "--package-whole-project-src" (in the wrapper script). --- build_with_docker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_with_docker.py b/build_with_docker.py index 07446a9..a6422ca 100644 --- a/build_with_docker.py +++ b/build_with_docker.py @@ -20,6 +20,7 @@ def main(cli_args: List[str]): parser.add_argument("--packaged-src", type=str, help="source code packaged in a JSON file") parser.add_argument("--contract", type=str) parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output") + parser.add_argument("--package-whole-project-src", type=bool, default=False, help="include all project files in *.source.json (default: %(default)s)") parser.add_argument("--cargo-target-dir", help="deprecated parameter, not used anymore") parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)") parser.add_argument("--build-root", type=str, required=False, help="root path (within container) for the build (default: %(default)s)") @@ -34,6 +35,7 @@ def main(cli_args: List[str]): packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None contract_path = parsed_args.contract output_path = Path(parsed_args.output).expanduser().resolve() + package_whole_project_src = parsed_args.package_whole_project_src no_wasm_opt = parsed_args.no_wasm_opt build_root = Path(parsed_args.build_root) if parsed_args.build_root else None cargo_verbose = parsed_args.cargo_verbose @@ -95,6 +97,9 @@ def main(cli_args: List[str]): if build_root: entrypoint_args.extend(["--build-root", str(build_root)]) + if package_whole_project_src: + entrypoint_args.append("--package-whole-project-src") + # Run docker container args = docker_general_args + docker_mount_args + docker_env_args + [image] + entrypoint_args logger.info(f"Running docker: {args}") From 8e16b0fbbd2d080b254fb63bce1288759a3fb043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 10:48:18 +0200 Subject: [PATCH 4/8] Add some integration tests. --- integration_tests/previous_builds.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/integration_tests/previous_builds.py b/integration_tests/previous_builds.py index 0b533ea..0111cdd 100644 --- a/integration_tests/previous_builds.py +++ b/integration_tests/previous_builds.py @@ -47,5 +47,18 @@ def __init__(self, name: str, "multisig-view": "ebaf987b041fcda297da71291d76736e4e98a1e449e5ec37908cdc0198e8be37" }, docker_image="multiversx/sdk-rust-contract-builder:v5.3.0" + ), + PreviousBuild( + name="a.3", + project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/v2.4.1-pair-safe-price-v2-reproducible.zip", + project_relative_path_in_archive="mx-exchange-sc-2.4.1-pair-safe-price-v2-reproducible", + packaged_src_url=None, + contract_name=None, + expected_code_hashes={ + "pair": "e9f117971963cb3c24b14e2a7698d48c170335af2f5c8167774c48c3c1c654e3", + "pair-full": "f1af2b2bb42a9f035745777e4b2d4f72478569224f204d0d0103801faff9663a", + "safe-price-view": "b5a657445ae74423c60210c88a6fa89b0bd4bdd00d5f06e788e14495bccc34c9" + }, + docker_image="sdk-rust-contract-builder:next" ) ] From e0f089c7d90ed67e3ce5be7c06a1e846bd3c6acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 12:44:05 +0200 Subject: [PATCH 5/8] Adjusts integration tests. --- ..._folder_and_packaged_src_are_equivalent.py | 108 +++++++++--------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/integration_tests/test_project_folder_and_packaged_src_are_equivalent.py b/integration_tests/test_project_folder_and_packaged_src_are_equivalent.py index 8d86748..2e5fc48 100644 --- a/integration_tests/test_project_folder_and_packaged_src_are_equivalent.py +++ b/integration_tests/test_project_folder_and_packaged_src_are_equivalent.py @@ -8,64 +8,70 @@ def main(cli_args: List[str]): - # TODO: when possible, also add multiversx/mx-exchange-sc (as of May 2023, it references mx-sdk-rs < v0.41.0, thus cannot be used for testing reproducible builds v5). - project_path = download_project_repository("https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.0.zip", "mx-exchange-sc-main") - parent_output_using_project = PARENT_OUTPUT_FOLDER / "using-project" - parent_output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src" + # We use "mx-exchange-sc" for the following integration tests, since it's a large and complex project. + repository_url = "https://github.com/multiversx/mx-exchange-sc" + branch_name = "reproducible-v2.4.1-pair-safe-price-v2" + archve_subfolder = f"mx-exchange-sc-{branch_name}" + project_path = download_project_repository(f"{repository_url}/archive/refs/heads/{branch_name}.zip", archve_subfolder) + project_path = project_path / archve_subfolder - shutil.rmtree(parent_output_using_project, ignore_errors=True) - shutil.rmtree(parent_output_using_packaged_src, ignore_errors=True) - - check_project_folder_and_packaged_src_are_equivalent(project_path, parent_output_using_project, parent_output_using_packaged_src, ["adder", "multisig"]) + check_project_folder_and_packaged_src_are_equivalent( + project_path=project_path, + package_whole_project_src=True, + parent_output_folder=PARENT_OUTPUT_FOLDER, + contracts=["pair", "farm"], + ) def check_project_folder_and_packaged_src_are_equivalent( project_path: Path, - parent_output_using_project: Path, - parent_output_using_packaged_src: Path, + package_whole_project_src: bool, + parent_output_folder: Path, contracts: List[str]): for contract in contracts: - for package_whole_project_src in [True, False]: - output_using_project = parent_output_using_project / contract / ("whole" if package_whole_project_src else "truncated") - output_using_packaged_src = parent_output_using_packaged_src / contract / ("whole" if package_whole_project_src else "truncated") - - output_using_packaged_src.mkdir(parents=True, exist_ok=True) - output_using_project.mkdir(parents=True, exist_ok=True) - - run_docker( - project_path=project_path, - package_whole_project_src=package_whole_project_src, - packaged_src_path=None, - contract_name=contract, - image="sdk-rust-contract-builder:next", - output_folder=output_using_project - ) - - packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json" - - run_docker( - project_path=None, - package_whole_project_src=package_whole_project_src, - packaged_src_path=packaged_src_path, - contract_name=contract, - image="sdk-rust-contract-builder:next", - output_folder=output_using_packaged_src - ) - - # Check that output folders are identical - using_project_output_files = sorted((output_using_project / contract).rglob("*")) - using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*")) - - assert len(using_project_output_files) == len(using_packaged_src_output_files) - - for index, file in enumerate(using_project_output_files): - if not file.is_file() or file.suffix == ".zip": - continue - using_project_file_content = file.read_bytes() - using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes() - - if using_project_file_content != using_packaged_src_file_content: - raise Exception(f"Files differ ({contract}): {file.name}") + output_using_project = parent_output_folder / "using-project" / contract / ("whole" if package_whole_project_src else "truncated") + output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract / ("whole" if package_whole_project_src else "truncated") + + shutil.rmtree(output_using_project, ignore_errors=True) + shutil.rmtree(output_using_packaged_src, ignore_errors=True) + + output_using_project.mkdir(parents=True, exist_ok=True) + output_using_packaged_src.mkdir(parents=True, exist_ok=True) + + run_docker( + project_path=project_path, + package_whole_project_src=package_whole_project_src, + packaged_src_path=None, + contract_name=contract, + image="sdk-rust-contract-builder:next", + output_folder=output_using_project + ) + + packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json" + + run_docker( + project_path=None, + package_whole_project_src=package_whole_project_src, + packaged_src_path=packaged_src_path, + contract_name=contract, + image="sdk-rust-contract-builder:next", + output_folder=output_using_packaged_src + ) + + # Check that output folders are identical + using_project_output_files = sorted((output_using_project / contract).rglob("*")) + using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*")) + + assert len(using_project_output_files) == len(using_packaged_src_output_files) + + for index, file in enumerate(using_project_output_files): + if not file.is_file() or file.suffix == ".zip": + continue + using_project_file_content = file.read_bytes() + using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes() + + if using_project_file_content != using_packaged_src_file_content: + raise Exception(f"Files differ ({contract}): {file.name}") if __name__ == "__main__": From 152ada665fc11840e11402d0a04225fcb38d042c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 13:11:56 +0200 Subject: [PATCH 6/8] Fix builder name. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6d6d03a..88cefb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 # Constants -ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v5.3.0" +ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.2.0-beta.1" ARG VERSION_RUST="nightly-2022-10-16" ARG VERSION_BINARYEN="version_105" ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz" From 42093db99d2be5194542c8bc8ba59853878c735f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 13:39:19 +0200 Subject: [PATCH 7/8] Adjust docker tag, adjust tests. --- .github/workflows/run_long_integration_tests.yml | 2 +- Dockerfile | 2 +- integration_tests/previous_builds.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_long_integration_tests.yml b/.github/workflows/run_long_integration_tests.yml index 987df35..a230dfc 100644 --- a/.github/workflows/run_long_integration_tests.yml +++ b/.github/workflows/run_long_integration_tests.yml @@ -22,7 +22,7 @@ jobs: - name: Build run: | export PYTHONPATH=. - python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" + python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3" - name: Save artifacts uses: actions/upload-artifact@v3 diff --git a/Dockerfile b/Dockerfile index 88cefb7..518a6ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 # Constants -ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.2.0-beta.1" +ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.2.0" ARG VERSION_RUST="nightly-2022-10-16" ARG VERSION_BINARYEN="version_105" ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz" diff --git a/integration_tests/previous_builds.py b/integration_tests/previous_builds.py index 0111cdd..da6b64e 100644 --- a/integration_tests/previous_builds.py +++ b/integration_tests/previous_builds.py @@ -51,7 +51,7 @@ def __init__(self, name: str, PreviousBuild( name="a.3", project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/v2.4.1-pair-safe-price-v2-reproducible.zip", - project_relative_path_in_archive="mx-exchange-sc-2.4.1-pair-safe-price-v2-reproducible", + project_relative_path_in_archive="reproducible-v2.4.1-pair-safe-price-v2", packaged_src_url=None, contract_name=None, expected_code_hashes={ From 8320fb1add593d7cf82c73b8bdfbad14766bee62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20B=C4=83ncioiu?= Date: Wed, 27 Dec 2023 13:52:59 +0200 Subject: [PATCH 8/8] Fix path for integration test. --- integration_tests/previous_builds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/previous_builds.py b/integration_tests/previous_builds.py index da6b64e..3f0d14c 100644 --- a/integration_tests/previous_builds.py +++ b/integration_tests/previous_builds.py @@ -50,8 +50,8 @@ def __init__(self, name: str, ), PreviousBuild( name="a.3", - project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/v2.4.1-pair-safe-price-v2-reproducible.zip", - project_relative_path_in_archive="reproducible-v2.4.1-pair-safe-price-v2", + project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.4.1-pair-safe-price-v2.zip", + project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.4.1-pair-safe-price-v2", packaged_src_url=None, contract_name=None, expected_code_hashes={