Fix v4, part 1: bring "main" into release/v4

.github/workflows/publish.yml

@@ -38,7 +38,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: true

.github/workflows/run_long_integration_tests.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Build
         run: |
           export PYTHONPATH=.
-          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3" "b.1" "b.2" "b.3" "c.1" "c.2" "c.3" "c.4" "c.5" "d.1" "e.1" "f.1"
+          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2"
 
       - name: Save artifacts
         uses: actions/upload-artifact@v3

Dockerfile

@@ -1,25 +1,33 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.1.4"
-ARG VERSION_RUST="nightly-2022-10-16"
-ARG VERSION_BINARYEN="105-1"
+ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v5.3.0"
+ARG VERSION_RUST="nightly-2023-05-26"
+ARG VERSION_BINARYEN="version_112"
+ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz"
 ARG VERSION_WABT="1.0.27-1"
-ARG VERSION_SC_META="0.39.5"
+ARG VERSION_SC_META="0.43.3"
+ARG TARGETPLATFORM
 
-# Temporary workaround. Default Ubuntu archive mirrors are down.
-RUN sed -i 's|http://archive.ubuntu.com|http://de.archive.ubuntu.com|g' /etc/apt/sources.list
-
-# Install dependencies
+# Install system dependencies
 RUN apt-get update --fix-missing && apt-get install -y \
-    wget \ 
+    wget \
     build-essential \
     git \
     python3.11 python-is-python3 python3-pip \
-    binaryen=${VERSION_BINARYEN} \
-    wabt=${VERSION_WABT}
+    wabt=${VERSION_WABT} \
+    pkg-config \
+    libssl-dev
 
+# Install binaryen
+RUN wget -O binaryen.tar.gz ${DOWNLOAD_URL_BINARYEN} && \
+    tar -xf binaryen.tar.gz && \
+    mkdir -p /binaryen && \
+    cp binaryen-${VERSION_BINARYEN}/bin/wasm-opt /binaryen && \
+    rm -rf binaryen.tar.gz binaryen-${VERSION_BINARYEN} && \
+    chmod -R 777 /binaryen
 
+# Install Python dependencies
 RUN pip3 install toml==0.10.2 semver==3.0.0-dev.4
 
 # Install rust
@@ -36,7 +44,7 @@ RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install mu
 
 COPY "multiversx_sdk_rust_contract_builder" "/multiversx_sdk_rust_contract_builder"
 
-ENV PATH="/rust/bin:${PATH}"
+ENV PATH="/rust/bin:/binaryen:${PATH}"
 ENV CARGO_HOME="/rust"
 ENV RUSTUP_HOME="/rust"
 ENV PYTHONPATH=/
@@ -45,6 +53,7 @@ ENV BUILD_METADATA_VERSION_RUST=${VERSION_RUST}
 ENV BUILD_METADATA_VERSION_BINARYEN=${VERSION_BINARYEN}
 ENV BUILD_METADATA_VERSION_WABT=${VERSION_WABT}
 ENV BUILD_METADATA_VERSION_SC_META=${VERSION_SC_META}
+ENV BUILD_METADATA_TARGETPLATFORM=${TARGETPLATFORM}
 
 # Additional arguments (must be provided at "docker run"):
 # --project or --packaged-src

README.md

@@ -7,15 +7,15 @@ Docker image (and wrappers) for reproducible contract builds (Rust). See [docs.m
 We use `docker buildx` to build the image:
 
 ```
-docker buildx build --output type=docker . -t sdk-rust-contract-builder:next -f ./Dockerfile
+docker buildx build --network host  --output type=docker . -t sdk-rust-contract-builder:next -f ./Dockerfile
 ```
 
 Maintainers can publish the image as follows:
 
 ```
 docker buildx create --name multiarch --use
 
-docker buildx build --push --platform=linux/amd64 . -t multiversx/sdk-rust-contract-builder:next -f ./Dockerfile
+docker buildx build --network host --push --platform=linux/amd64 . -t multiversx/sdk-rust-contract-builder:next -f ./Dockerfile
 
 docker buildx rm multiarch
 ```

integration_tests/shared.py

@@ -54,6 +54,7 @@ def run_docker(
     docker_mount_args += ["--volume", f"{RUST_REGISTRY}:/rust/registry"]
     docker_mount_args += ["--volume", f"{RUST_GIT}:/rust/git"]
     docker_mount_args += ["--volume", f"{RUST_TMP}:/rust/tmp"]
+    docker_mount_args += ["--network", "host"]
 
     docker_args = ["docker", "run"]
     docker_args += docker_mount_args

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -1,21 +1,29 @@
 import shutil
 import sys
+from pathlib import Path
 from typing import List
 
 from integration_tests.config import PARENT_OUTPUT_FOLDER
 from integration_tests.shared import download_project_repository, run_docker
 
 
 def main(cli_args: List[str]):
-    project_path = download_project_repository("https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/main.zip", "mx-exchange-sc-main")
+    # TODO: when possible, also add multiversx/mx-exchange-sc (as of May 2023, it references mx-sdk-rs < v0.41.0, thus cannot be used for testing reproducible builds v5).
+    project_path = download_project_repository("https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.0.zip", "mx-exchange-sc-main")
     parent_output_using_project = PARENT_OUTPUT_FOLDER / "using-project"
     parent_output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src"
 
     shutil.rmtree(parent_output_using_project, ignore_errors=True)
     shutil.rmtree(parent_output_using_packaged_src, ignore_errors=True)
 
-    contracts = ['distribution', 'energy-factory', 'energy-update', 'factory', 'farm', 'farm-staking', 'farm-staking-proxy', 'farm-with-locked-rewards', 'fees-collector', 'governance', 'governance-v2', 'lkmex-transfer', 'locked-token-wrapper', 'metabonding-staking', 'pair', 'pause-all', 'price-discovery', 'proxy-deployer', 'proxy_dex', 'router', 'simple-lock', 'simple-lock-whitelist', 'token-unstake']
+    check_project_folder_and_packaged_src_are_equivalent(project_path, parent_output_using_project, parent_output_using_packaged_src, ["adder", "multisig"])
 
+
+def check_project_folder_and_packaged_src_are_equivalent(
+        project_path: Path,
+        parent_output_using_project: Path,
+        parent_output_using_packaged_src: Path,
+        contracts: List[str]):
     for contract in contracts:
         for package_whole_project_src in [True, False]:
             output_using_project = parent_output_using_project / contract / ("whole" if package_whole_project_src else "truncated")

multiversx_sdk_rust_contract_builder/build_metadata.py

@@ -10,12 +10,14 @@ def __init__(
         version_binaryen: str,
         version_wabt: str,
         version_sc_meta: str,
+        target_platform: str,
     ):
         self.builder_name = builder_name
         self.version_rust = version_rust
         self.version_binaryen = version_binaryen
         self.version_wabt = version_wabt
         self.version_sc_meta = version_sc_meta
+        self.target_platform = target_platform
 
     @classmethod
     def from_env(cls) -> 'BuildMetadata':
@@ -25,6 +27,7 @@ def from_env(cls) -> 'BuildMetadata':
             version_binaryen=os.environ["BUILD_METADATA_VERSION_BINARYEN"],
             version_wabt=os.environ["BUILD_METADATA_VERSION_WABT"],
             version_sc_meta=os.environ["BUILD_METADATA_VERSION_SC_META"],
+            target_platform=os.environ["BUILD_METADATA_TARGETPLATFORM"],
         )
 
     def to_dict(self) -> Dict[str, str]:
@@ -34,4 +37,5 @@ def to_dict(self) -> Dict[str, str]:
             "versionBinaryen": self.version_binaryen,
             "versionWabt": self.version_wabt,
             "versionScTool": self.version_sc_meta,
+            "targetPlatform": self.target_platform,
         }

multiversx_sdk_rust_contract_builder/build_outcome.py

@@ -5,7 +5,8 @@
 
 from multiversx_sdk_rust_contract_builder.cargo_toml import \
     get_contract_name_and_version
-from multiversx_sdk_rust_contract_builder.filesystem import find_file_in_folder
+from multiversx_sdk_rust_contract_builder.filesystem import (
+    find_file_in_folder, find_files_in_folder)
 
 
 class IWithToDict(Protocol):
@@ -18,8 +19,9 @@ def __init__(self, build_metadata: IWithToDict, build_options: IWithToDict):
         self.build_metadata = build_metadata
         self.build_options = build_options
 
-    def gather_artifacts(self, contract_name: str, build_folder: Path, output_subfolder: Path):
-        self.contracts[contract_name] = BuildOutcomeEntry.from_folders(build_folder, output_subfolder)
+    def gather_artifacts(self, build_folder: Path, output_subfolder: Path):
+        entries = BuildOutcomeEntry.many_from_folders(build_folder, output_subfolder)
+        self.contracts.update(entries)
 
     def get_entry(self, contract_name: str) -> 'BuildOutcomeEntry':
         return self.contracts[contract_name]
@@ -47,44 +49,42 @@ class BuildOutcomeEntry:
     def __init__(self) -> None:
         self.version = ""
         self.codehash = ""
-        self.artifacts = BunchOfBuildArtifacts()
+        self.bytecode_path = BuildArtifact(Path(""))
+        self.abi_path = BuildArtifact(Path(""))
+        self.src_package_path = BuildArtifact(Path(""))
 
     @classmethod
-    def from_folders(cls, build_folder: Path, output_folder: Path) -> 'BuildOutcomeEntry':
-        entry = BuildOutcomeEntry()
-        _, entry.version = get_contract_name_and_version(build_folder)
-        entry.codehash = find_file_in_folder(output_folder, "*.codehash.txt").read_text()
-        entry.artifacts = BunchOfBuildArtifacts.from_output_folder(output_folder)
-        return entry
+    def many_from_folders(cls, build_folder: Path, output_folder: Path) -> Dict[str, 'BuildOutcomeEntry']:
+        # Note: sub-contracts of multi-contracts share the same version.
+        _, version = get_contract_name_and_version(build_folder)
 
-    def to_dict(self) -> Dict[str, Any]:
-        return {
-            "version": self.version,
-            "codehash": self.codehash,
-            "artifacts": self.artifacts.to_dict()
-        }
+        # We consider all *.wasm files in the output folder to be standalone contracts or sub-contracts of multi-contracts.
+        wasm_files = find_files_in_folder(output_folder, "*.wasm")
 
+        result: Dict[str, BuildOutcomeEntry] = {}
 
-class BunchOfBuildArtifacts:
-    def __init__(self) -> None:
-        self.bytecode = BuildArtifact(Path(""))
-        self.abi = BuildArtifact(Path(""))
-        self.src_package = BuildArtifact(Path(""))
+        for wasm_file in wasm_files:
+            contract_name = wasm_file.stem
+            entry = BuildOutcomeEntry()
+            entry.version = version
+            entry.codehash = find_file_in_folder(output_folder, f"{contract_name}.codehash.txt").read_text()
+            entry.bytecode_path = BuildArtifact.find_in_output(f"{contract_name}.wasm", output_folder)
+            entry.abi_path = BuildArtifact.find_in_output(f"{contract_name}.abi.json", output_folder)
+            entry.src_package_path = BuildArtifact.find_in_output("*.source.json", output_folder)
 
-    @classmethod
-    def from_output_folder(cls, output_folder: Path) -> 'BunchOfBuildArtifacts':
-        artifacts = BunchOfBuildArtifacts()
-        artifacts.bytecode = BuildArtifact.find_in_output("*.wasm", output_folder)
-        artifacts.abi = BuildArtifact.find_in_output("*.abi.json", output_folder)
-        artifacts.src_package = BuildArtifact.find_in_output("*.source.json", output_folder)
+            result[contract_name] = entry
 
-        return artifacts
+        return result
 
-    def to_dict(self) -> Dict[str, str]:
+    def to_dict(self) -> Dict[str, Any]:
         return {
-            "bytecode": self.bytecode.path.name,
-            "abi": self.abi.path.name,
-            "srcPackage": self.src_package.path.name,
+            "version": self.version,
+            "codehash": self.codehash,
+            "artifacts": {
+                "bytecode": self.bytecode_path.path.name,
+                "abi": self.abi_path.path.name,
+                "srcPackage": self.src_package_path.path.name,
+            }
         }
 
 

multiversx_sdk_rust_contract_builder/builder.py

@@ -3,7 +3,7 @@
 import shutil
 import subprocess
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Set
 
 from multiversx_sdk_rust_contract_builder import cargo_toml, source_code
 from multiversx_sdk_rust_contract_builder.build_metadata import BuildMetadata
@@ -14,10 +14,10 @@
 from multiversx_sdk_rust_contract_builder.codehash import \
     generate_code_hash_artifact
 from multiversx_sdk_rust_contract_builder.constants import (
-    CONTRACT_CONFIG_FILENAME, MAX_PACKAGED_SOURCE_CODE_SIZE,
-    OLD_CONTRACT_CONFIG_FILENAME)
+    CONTRACT_CONFIG_FILENAME, MAX_PACKAGED_SOURCE_CODE_SIZE)
 from multiversx_sdk_rust_contract_builder.errors import ErrKnown
-from multiversx_sdk_rust_contract_builder.filesystem import find_file_in_folder
+from multiversx_sdk_rust_contract_builder.filesystem import \
+    find_files_in_folder
 from multiversx_sdk_rust_contract_builder.packaged_source_code import (
     PackagedSourceCode, PackagedSourceMetadata)
 
@@ -40,6 +40,7 @@ def build_project(
 
     outcome = BuildOutcome(metadata, options)
     contracts_folders = get_contracts_folders(project_folder)
+    ensure_distinct_contract_names(contracts_folders)
 
     # We copy the whole project folder to the build path, to ensure that all local dependencies are available.
     project_within_build_folder = copy_project_folder_to_build_folder(project_folder, build_root_folder)
@@ -81,7 +82,7 @@ def build_project(
             build_options=options.to_dict(),
         )
 
-        outcome.gather_artifacts(contract_name, contract_build_subfolder, output_subfolder)
+        outcome.gather_artifacts(contract_build_subfolder, output_subfolder)
 
     return outcome
 
@@ -93,11 +94,20 @@ def ensure_output_folder_is_empty(parent_output_folder: Path):
 
 
 def get_contracts_folders(project_path: Path) -> List[Path]:
-    old_markers = list(project_path.glob(f"**/{OLD_CONTRACT_CONFIG_FILENAME}"))
-    new_markers = list(project_path.glob(f"**/{CONTRACT_CONFIG_FILENAME}"))
-    marker_files = old_markers + new_markers
+    marker_files = list(project_path.glob(f"**/{CONTRACT_CONFIG_FILENAME}"))
     folders = [marker_file.parent for marker_file in marker_files]
-    return sorted(folders)
+    return sorted(set(folders))
+
+
+def ensure_distinct_contract_names(contracts_folders: List[Path]):
+    names: Set[str] = set()
+
+    for folder in sorted(contracts_folders):
+        name, _ = get_contract_name_and_version(folder)
+        if name in names:
+            raise Exception(f"""Name "{name}" already associated to another contract.""")
+
+        names.add(name)
 
 
 def copy_project_folder_to_build_folder(project_folder: Path, build_root_folder: Path):
@@ -129,7 +139,7 @@ def build_contract(build_folder: Path, output_folder: Path, cargo_target_dir: Pa
 
     # If the lock file is missing, or it needs to be updated, Cargo will exit with an error.
     # See: https://doc.rust-lang.org/cargo/commands/cargo-build.html
-    if cargo_toml.does_cargo_build_support_locked(build_folder) and cargo_lock.exists():
+    if cargo_lock.exists():
         args.append("--locked")
 
     custom_env = os.environ.copy()
@@ -144,8 +154,10 @@ def build_contract(build_folder: Path, output_folder: Path, cargo_target_dir: Pa
     if return_code != 0:
         raise ErrKnown(f"Failed to build contract {build_folder}. Return code: {return_code}.")
 
-    wasm_file = find_file_in_folder(cargo_output_folder, "*.wasm")
-    generate_code_hash_artifact(wasm_file)
+    # One or more WASM files should have been generated.
+    wasm_files = find_files_in_folder(cargo_output_folder, "*.wasm")
+    for wasm_file in wasm_files:
+        generate_code_hash_artifact(wasm_file)
 
     shutil.copytree(cargo_output_folder, output_folder, dirs_exist_ok=True)
 

multiversx_sdk_rust_contract_builder/cargo_toml.py

@@ -1,9 +1,8 @@
 import logging
 import shutil
 from pathlib import Path
-from typing import Any, Tuple
+from typing import Tuple
 
-import semver
 import toml
 
 from multiversx_sdk_rust_contract_builder.filesystem import get_all_files
@@ -38,29 +37,3 @@ def remove_dev_dependencies_sections(file: Path):
     if "dev-dependencies" in data:
         del data["dev-dependencies"]
         file.write_text(toml.dumps(data))
-
-
-def does_cargo_build_support_locked(contract_folder: Path) -> bool:
-    file = contract_folder / "Cargo.toml"
-    data: Any = toml.loads(file.read_text())
-
-    framework_version_old: str = str(data.get("dependencies", {}).get("elrond-wasm", {}).get("version", ""))
-    framework_version_new: str = str(data.get("dependencies", {}).get("multiversx-sc", {}).get("version", ""))
-    framework_version: str = framework_version_old or framework_version_new
-    framework_version = _normalize_rust_framework_version(framework_version)
-
-    # Before this version, --locked was ignored.
-    # On this version, using --locked resulted in an error.
-    # After this version, --locked is supported.
-    supports_locked: bool = semver.compare(framework_version, "0.39.2") > 0
-
-    logging.info(f"does_cargo_build_support_locked({contract_folder}), framework version = {framework_version}? {supports_locked}")
-    return supports_locked
-
-
-def _normalize_rust_framework_version(version: str) -> str:
-    version = version.strip('^').strip('=')
-    version_parts = version.split(".")
-    if len(version_parts) == 2:
-        version += ".0"
-    return version