Fix v4, part 2: undo breaking changes of (v4) -> (v5), handle workspaces

.github/workflows/run_long_integration_tests.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Build
         run: |
           export PYTHONPATH=.
-          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2"
+          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3"
 
       - name: Save artifacts
         uses: actions/upload-artifact@v3

Dockerfile

@@ -1,12 +1,12 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v5.3.0"
-ARG VERSION_RUST="nightly-2023-05-26"
-ARG VERSION_BINARYEN="version_112"
+ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.2.0"
+ARG VERSION_RUST="nightly-2022-10-16"
+ARG VERSION_BINARYEN="version_105"
 ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz"
 ARG VERSION_WABT="1.0.27-1"
-ARG VERSION_SC_META="0.43.3"
+ARG VERSION_SC_META="0.39.5"
 ARG TARGETPLATFORM
 
 # Install system dependencies
@@ -39,7 +39,7 @@ RUN wget -O rustup.sh https://sh.rustup.rs && \
     rm -rf /rust/registry
 
 # Install sc-tool
-RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install multiversx-sc-meta --version ${VERSION_SC_META} && \
+RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install multiversx-sc-meta --locked --version ${VERSION_SC_META} && \
     rm -rf /rust/registry
 
 COPY "multiversx_sdk_rust_contract_builder" "/multiversx_sdk_rust_contract_builder"

build_with_docker.py

@@ -20,6 +20,7 @@ def main(cli_args: List[str]):
     parser.add_argument("--packaged-src", type=str, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str)
     parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output")
+    parser.add_argument("--package-whole-project-src", type=bool, default=False, help="include all project files in *.source.json (default: %(default)s)")
     parser.add_argument("--cargo-target-dir", help="deprecated parameter, not used anymore")
     parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)")
     parser.add_argument("--build-root", type=str, required=False, help="root path (within container) for the build (default: %(default)s)")
@@ -34,6 +35,7 @@ def main(cli_args: List[str]):
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     contract_path = parsed_args.contract
     output_path = Path(parsed_args.output).expanduser().resolve()
+    package_whole_project_src = parsed_args.package_whole_project_src
     no_wasm_opt = parsed_args.no_wasm_opt
     build_root = Path(parsed_args.build_root) if parsed_args.build_root else None
     cargo_verbose = parsed_args.cargo_verbose
@@ -95,6 +97,9 @@ def main(cli_args: List[str]):
     if build_root:
         entrypoint_args.extend(["--build-root", str(build_root)])
 
+    if package_whole_project_src:
+        entrypoint_args.append("--package-whole-project-src")
+
     # Run docker container
     args = docker_general_args + docker_mount_args + docker_env_args + [image] + entrypoint_args
     logger.info(f"Running docker: {args}")

integration_tests/previous_builds.py

@@ -47,5 +47,18 @@ def __init__(self, name: str,
             "multisig-view": "ebaf987b041fcda297da71291d76736e4e98a1e449e5ec37908cdc0198e8be37"
         },
         docker_image="multiversx/sdk-rust-contract-builder:v5.3.0"
+    ),
+    PreviousBuild(
+        name="a.3",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.4.1-pair-safe-price-v2.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.4.1-pair-safe-price-v2",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "pair": "e9f117971963cb3c24b14e2a7698d48c170335af2f5c8167774c48c3c1c654e3",
+            "pair-full": "f1af2b2bb42a9f035745777e4b2d4f72478569224f204d0d0103801faff9663a",
+            "safe-price-view": "b5a657445ae74423c60210c88a6fa89b0bd4bdd00d5f06e788e14495bccc34c9"
+        },
+        docker_image="sdk-rust-contract-builder:next"
     )
 ]

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -8,64 +8,70 @@
 
 
 def main(cli_args: List[str]):
-    # TODO: when possible, also add multiversx/mx-exchange-sc (as of May 2023, it references mx-sdk-rs < v0.41.0, thus cannot be used for testing reproducible builds v5).
-    project_path = download_project_repository("https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.0.zip", "mx-exchange-sc-main")
-    parent_output_using_project = PARENT_OUTPUT_FOLDER / "using-project"
-    parent_output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src"
+    # We use "mx-exchange-sc" for the following integration tests, since it's a large and complex project.
+    repository_url = "https://github.com/multiversx/mx-exchange-sc"
+    branch_name = "reproducible-v2.4.1-pair-safe-price-v2"
+    archve_subfolder = f"mx-exchange-sc-{branch_name}"
+    project_path = download_project_repository(f"{repository_url}/archive/refs/heads/{branch_name}.zip", archve_subfolder)
+    project_path = project_path / archve_subfolder
 
-    shutil.rmtree(parent_output_using_project, ignore_errors=True)
-    shutil.rmtree(parent_output_using_packaged_src, ignore_errors=True)
-
-    check_project_folder_and_packaged_src_are_equivalent(project_path, parent_output_using_project, parent_output_using_packaged_src, ["adder", "multisig"])
+    check_project_folder_and_packaged_src_are_equivalent(
+        project_path=project_path,
+        package_whole_project_src=True,
+        parent_output_folder=PARENT_OUTPUT_FOLDER,
+        contracts=["pair", "farm"],
+    )
 
 
 def check_project_folder_and_packaged_src_are_equivalent(
         project_path: Path,
-        parent_output_using_project: Path,
-        parent_output_using_packaged_src: Path,
+        package_whole_project_src: bool,
+        parent_output_folder: Path,
         contracts: List[str]):
     for contract in contracts:
-        for package_whole_project_src in [True, False]:
-            output_using_project = parent_output_using_project / contract / ("whole" if package_whole_project_src else "truncated")
-            output_using_packaged_src = parent_output_using_packaged_src / contract / ("whole" if package_whole_project_src else "truncated")
-
-            output_using_packaged_src.mkdir(parents=True, exist_ok=True)
-            output_using_project.mkdir(parents=True, exist_ok=True)
-
-            run_docker(
-                project_path=project_path,
-                package_whole_project_src=package_whole_project_src,
-                packaged_src_path=None,
-                contract_name=contract,
-                image="sdk-rust-contract-builder:next",
-                output_folder=output_using_project
-            )
-
-            packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
-
-            run_docker(
-                project_path=None,
-                package_whole_project_src=package_whole_project_src,
-                packaged_src_path=packaged_src_path,
-                contract_name=contract,
-                image="sdk-rust-contract-builder:next",
-                output_folder=output_using_packaged_src
-            )
-
-            # Check that output folders are identical
-            using_project_output_files = sorted((output_using_project / contract).rglob("*"))
-            using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
-
-            assert len(using_project_output_files) == len(using_packaged_src_output_files)
-
-            for index, file in enumerate(using_project_output_files):
-                if not file.is_file() or file.suffix == ".zip":
-                    continue
-                using_project_file_content = file.read_bytes()
-                using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
-
-                if using_project_file_content != using_packaged_src_file_content:
-                    raise Exception(f"Files differ ({contract}): {file.name}")
+        output_using_project = parent_output_folder / "using-project" / contract / ("whole" if package_whole_project_src else "truncated")
+        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract / ("whole" if package_whole_project_src else "truncated")
+
+        shutil.rmtree(output_using_project, ignore_errors=True)
+        shutil.rmtree(output_using_packaged_src, ignore_errors=True)
+
+        output_using_project.mkdir(parents=True, exist_ok=True)
+        output_using_packaged_src.mkdir(parents=True, exist_ok=True)
+
+        run_docker(
+            project_path=project_path,
+            package_whole_project_src=package_whole_project_src,
+            packaged_src_path=None,
+            contract_name=contract,
+            image="sdk-rust-contract-builder:next",
+            output_folder=output_using_project
+        )
+
+        packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
+
+        run_docker(
+            project_path=None,
+            package_whole_project_src=package_whole_project_src,
+            packaged_src_path=packaged_src_path,
+            contract_name=contract,
+            image="sdk-rust-contract-builder:next",
+            output_folder=output_using_packaged_src
+        )
+
+        # Check that output folders are identical
+        using_project_output_files = sorted((output_using_project / contract).rglob("*"))
+        using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
+
+        assert len(using_project_output_files) == len(using_packaged_src_output_files)
+
+        for index, file in enumerate(using_project_output_files):
+            if not file.is_file() or file.suffix == ".zip":
+                continue
+            using_project_file_content = file.read_bytes()
+            using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
+
+            if using_project_file_content != using_packaged_src_file_content:
+                raise Exception(f"Files differ ({contract}): {file.name}")
 
 
 if __name__ == "__main__":

multiversx_sdk_rust_contract_builder/constants.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 # The default value is the legacy one, to avoid breaking reproducibility of previous builds.
-DEFAULT_BUILD_ROOT = "/tmp/project"
+DEFAULT_BUILD_ROOT = "/tmp/elrond-contract-rust"
 HARDCODED_UNWRAP_FOLDER = Path("/tmp/unwrapped")
 ONE_KB_IN_BYTES: int = 1024
 MAX_PACKAGED_SOURCE_CODE_SIZE: int = 2 * ONE_KB_IN_BYTES * 1024

multiversx_sdk_rust_contract_builder/source_code.py

@@ -50,7 +50,7 @@ def get_source_code_files(
     files_related_to_contract = set(file.path for file in source_code_files)
 
     if include_unrelated_to_contract:
-        all_files = get_all_files(contract_folder, _is_source_code_file)
+        all_files = get_all_files(project_folder, _is_source_code_file)
         for file in all_files:
             if file not in files_related_to_contract:
                 source_code_files.append(SourceCodeFile(file, contract_folder, sys.maxsize))