Merge pull request #20 from multiversx/debug-19

Removing dev-dependencies, replacing test files

.github/workflows/run_long_integration_tests.yml

@@ -3,7 +3,6 @@ name: Long integration tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [ main ]
 
 jobs:
   build:

.github/workflows/run_regular_integration_tests.yml

@@ -3,7 +3,6 @@ name: Regular integration tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [ main ]
 
 jobs:
   build:

.gitignore

@@ -133,4 +133,5 @@ typings/**
 
 # Test data
 testdata/input/extracted
-testdata/**/output
+testdata/output
+testdata/rust

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG VERSION_RUST="nightly-2022-12-08"
+ARG VERSION_RUST="nightly-2022-10-16"
 ARG VERSION_BINARYEN="105-1"
 ARG VERSION_WABT="1.0.27-1"
 # Normally, this should be "multiversx/sdk-rust-contract-builder:{{pyproject.toml:project:version}}"
@@ -11,10 +11,12 @@ ARG CONTEXT="multiversx/sdk-rust-contract-builder:v4.1.0"
 RUN apt-get update && apt-get install -y \
     wget \ 
     build-essential \
-    python3.10 python-is-python3 \
+    python3.11 python-is-python3 python3-pip \
     binaryen=${VERSION_BINARYEN} \
     wabt=${VERSION_WABT}
 
+RUN pip3 install tomlkit==0.11.6
+
 # Install rust
 RUN wget -O rustup.sh https://sh.rustup.rs && \
     chmod +x rustup.sh && \

README.md

@@ -7,7 +7,7 @@ Docker image (and wrappers) for reproducible contract builds (Rust). See [docs.m
 We use `docker buildx` to build the image:
 
 ```
-docker buildx build --no-cache . -t sdk-rust-contract-builder:next -f ./Dockerfile
+docker buildx build --output type=docker --no-cache . -t sdk-rust-contract-builder:next -f ./Dockerfile
 ```
 
 Maintainers can publish the image as follows:

integration_tests/config.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 
-
 DOWNLOADS_FOLDER = Path("./testdata/downloads").resolve()
 EXTRACTED_FOLDER = Path("./testdata/input/extracted").resolve()
 PARENT_OUTPUT_FOLDER = Path("./testdata/output").resolve()
-CARGO_TARGET_DIR = Path("./testdata/output/cargo_target_dir").resolve()
+CARGO_TARGET_DIR = Path("./testdata/rust/cargo_target_dir").resolve()
+RUST_REGISTRY = Path("./testdata/rust/registry").resolve()
+RUST_GIT = Path("./testdata/rust/git").resolve()
+RUST_TMP = Path("./testdata/rust/tmp").resolve()

integration_tests/previous_builds.py

@@ -29,7 +29,7 @@ def __init__(self, name: str,
         expected_code_hashes={
             "adder": "58c6e78f40bd6ccc30d8a01f952b34a13ebfdad796a2526678be17c5d7820174"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.3"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="a.2",
@@ -40,7 +40,7 @@ def __init__(self, name: str,
         expected_code_hashes={
             "adder": "58c6e78f40bd6ccc30d8a01f952b34a13ebfdad796a2526678be17c5d7820174"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.3"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="a.3",
@@ -62,7 +62,7 @@ def __init__(self, name: str,
         expected_code_hashes={
             "metabonding-staking": "4a9b2afa13eca738b1804c48b82a961afd67adcbbf2aa518052fa124ac060bea"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="b.2",
@@ -73,7 +73,7 @@ def __init__(self, name: str,
         expected_code_hashes={
             "metabonding-staking": "4a9b2afa13eca738b1804c48b82a961afd67adcbbf2aa518052fa124ac060bea"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="b.3",
@@ -84,7 +84,7 @@ def __init__(self, name: str,
         expected_code_hashes={
             "metabonding-staking": "4a9b2afa13eca738b1804c48b82a961afd67adcbbf2aa518052fa124ac060bea"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="c.1",
@@ -119,7 +119,7 @@ def __init__(self, name: str,
             "simple-lock-whitelist": "c576c6106234e5f7978efb1885afe36c5d6da6a13c12b459fd7fe95967646d13",
             "token-unstake": "463e49892f64726450d0df5ab4ba26559ad882525ce5e93173a26fde8437266e",
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="c.2",
@@ -154,7 +154,7 @@ def __init__(self, name: str,
             "simple-lock-whitelist": "c576c6106234e5f7978efb1885afe36c5d6da6a13c12b459fd7fe95967646d13",
             "token-unstake": "463e49892f64726450d0df5ab4ba26559ad882525ce5e93173a26fde8437266e",
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="c.3",
@@ -189,7 +189,7 @@ def __init__(self, name: str,
             "simple-lock-whitelist": "3bc3cecbee78958e65efdaa077974d95d743d962254788d9280839362dc4da8b",
             "token-unstake": "2b0f59073bd697d75ec2009a3bf3c350b74ff9b10d6a7bfe1e13f653732ddb1a",
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="c.4",
@@ -224,7 +224,7 @@ def __init__(self, name: str,
             "simple-lock-whitelist": "b8f14e78635ad3894cb78f7d24302d17ee2059b353d1017e4b84f3beef2abc1a",
             "token-unstake": "3059ffec6f44259b8f2be56b9e1c67c72342f93c0c6f4b5cadc24364f2ee95e7",
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="c.5",
@@ -235,7 +235,7 @@ def __init__(self, name: str,
         expected_code_hashes={
             "energy-factory": "529fd987e7702b90044757073f36024d24cbe5cc8810d5abe93c6c5176a0ec53",
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="d.1",
@@ -258,6 +258,6 @@ def __init__(self, name: str,
         expected_code_hashes={
             "metabonding": "897b19e1990f7c487c99c12f50722febe1ee4468bcd3a7405641966dfff2791d"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v4.0.2"
+        docker_image="sdk-rust-contract-builder:next"
     )
 ]

integration_tests/shared.py

@@ -1,13 +1,13 @@
-import json
-import shutil
-import sys
-import urllib.request
 import os
+import shutil
 import subprocess
+import urllib.request
 from pathlib import Path
 from typing import List, Optional
 
-from integration_tests.config import DOWNLOADS_FOLDER, EXTRACTED_FOLDER, CARGO_TARGET_DIR
+from integration_tests.config import (CARGO_TARGET_DIR, DOWNLOADS_FOLDER,
+                                      EXTRACTED_FOLDER, RUST_GIT,
+                                      RUST_REGISTRY, RUST_TMP)
 
 
 def download_project_repository(zip_archive_url: str, name: str) -> Path:
@@ -26,6 +26,7 @@ def download_project_repository(zip_archive_url: str, name: str) -> Path:
 def download_packaged_src(json_url: str, name: str) -> Path:
     downloaded_packaged_src = DOWNLOADS_FOLDER / f"{name}.source.json"
     urllib.request.urlretrieve(json_url, downloaded_packaged_src)
+    return downloaded_packaged_src
 
 
 def run_docker(
@@ -36,6 +37,9 @@ def run_docker(
     output_folder: Path,
 ):
     CARGO_TARGET_DIR.mkdir(parents=True, exist_ok=True)
+    RUST_REGISTRY.mkdir(parents=True, exist_ok=True)
+    RUST_GIT.mkdir(parents=True, exist_ok=True)
+    RUST_TMP.mkdir(parents=True, exist_ok=True)
 
     docker_mount_args: List[str] = ["--volume", f"{output_folder}:/output"]
 
@@ -46,6 +50,9 @@ def run_docker(
         docker_mount_args.extend(["--volume", f"{packaged_src_path}:/packaged-src.json"])
 
     docker_mount_args += ["--volume", f"{CARGO_TARGET_DIR}:/rust/cargo-target-dir"]
+    docker_mount_args += ["--volume", f"{RUST_REGISTRY}:/rust/registry"]
+    docker_mount_args += ["--volume", f"{RUST_GIT}:/rust/git"]
+    docker_mount_args += ["--volume", f"{RUST_TMP}:/rust/tmp"]
 
     docker_args = ["docker", "run"]
     docker_args += docker_mount_args

integration_tests/test_previous_builds_are_reproducible.py

@@ -1,12 +1,15 @@
-from argparse import ArgumentParser
 import json
 import shutil
 import sys
+from argparse import ArgumentParser
 from pathlib import Path
 from typing import List, Optional, Tuple
-from integration_tests.config import DOWNLOADS_FOLDER, EXTRACTED_FOLDER, PARENT_OUTPUT_FOLDER, CARGO_TARGET_DIR
-from integration_tests.shared import download_packaged_src, download_project_repository, run_docker
+
+from integration_tests.config import (CARGO_TARGET_DIR, DOWNLOADS_FOLDER,
+                                      EXTRACTED_FOLDER, PARENT_OUTPUT_FOLDER)
 from integration_tests.previous_builds import PreviousBuild, previous_builds
+from integration_tests.shared import (download_packaged_src,
+                                      download_project_repository, run_docker)
 
 
 def main(cli_args: List[str]):

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -1,18 +1,21 @@
+import shutil
 import sys
 from typing import List
+
 from integration_tests.config import PARENT_OUTPUT_FOLDER
 from integration_tests.shared import download_project_repository, run_docker
 
-from multiversx_sdk_rust_contract_builder.main import main
-
 
 def main(cli_args: List[str]):
     project_path = download_project_repository("https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/main.zip", "mx-exchange-sc-main")
     output_using_project = PARENT_OUTPUT_FOLDER / "using-project"
     output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src"
 
-    output_using_project.mkdir(parents=True, exist_ok=True)
-    output_using_packaged_src.mkdir(parents=True, exist_ok=True)
+    shutil.rmtree(output_using_project, ignore_errors=True)
+    shutil.rmtree(output_using_packaged_src, ignore_errors=True)
+
+    output_using_project.mkdir(parents=True)
+    output_using_packaged_src.mkdir(parents=True)
 
     contracts = ['distribution', 'energy-factory', 'energy-update', 'factory', 'farm', 'farm-staking', 'farm-staking-proxy', 'farm-with-locked-rewards', 'fees-collector', 'governance', 'governance-v2', 'lkmex-transfer', 'locked-token-wrapper', 'metabonding-staking', 'pair', 'pause-all', 'price-discovery', 'proxy-deployer', 'proxy_dex', 'router', 'simple-lock', 'simple-lock-whitelist', 'token-unstake']
 
@@ -25,8 +28,6 @@ def main(cli_args: List[str]):
             output_folder=output_using_project
         )
 
-        code_hash_using_project = (output_using_project / f"{contract}/{contract}.codehash.txt").read_text().strip()
-
         packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
 
         run_docker(
@@ -37,9 +38,20 @@ def main(cli_args: List[str]):
             output_folder=output_using_packaged_src
         )
 
-        code_hash_using_packaged_src = (output_using_packaged_src / f"{contract}/{contract}.codehash.txt").read_text().strip()
+        # Check that output folders are identical
+        using_project_output_files = sorted((output_using_project / contract).rglob("*"))
+        using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
+
+        assert len(using_project_output_files) == len(using_packaged_src_output_files)
+
+        for index, file in enumerate(using_project_output_files):
+            if not file.is_file() or file.suffix == ".zip":
+                continue
+            using_project_file_content = file.read_bytes()
+            using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
 
-        assert code_hash_using_project == code_hash_using_packaged_src
+            if using_project_file_content != using_packaged_src_file_content:
+                raise Exception(f"Files differ ({contract}): {file.name}")
 
 
 if __name__ == "__main__":

multiversx_sdk_rust_contract_builder/builder.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import List, Optional
 
+from multiversx_sdk_rust_contract_builder import cargo_toml, source_code
 from multiversx_sdk_rust_contract_builder.build_outcome import BuildOutcome
 from multiversx_sdk_rust_contract_builder.cargo_toml import (
     get_contract_name_and_version, promote_cargo_lock_to_contract_folder)
@@ -38,6 +39,9 @@ def build_project(
     # We copy the whole project folder to the build path, to ensure that all local dependencies are available.
     project_within_build_folder = copy_project_folder_to_build_folder(project_folder)
 
+    cargo_toml.remove_dev_dependencies_sections_from_all(project_within_build_folder)
+    source_code.replace_all_test_content_with_noop(project_within_build_folder)
+
     for contract_folder in sorted(contracts_folders):
         contract_name, contract_version = get_contract_name_and_version(contract_folder)
         logging.info(f"Contract = {contract_name}, version = {contract_version}")
@@ -50,22 +54,22 @@ def build_project(
         output_subfolder.mkdir(parents=True, exist_ok=True)
 
         relative_contract_folder = contract_folder.relative_to(project_folder)
-        build_folder = project_within_build_folder / relative_contract_folder
+        contract_build_subfolder = project_within_build_folder / relative_contract_folder
 
         # Clean folder - it may contain externally-generated build artifacts
-        clean_contract(build_folder)
-        build_contract(build_folder, output_subfolder, cargo_target_dir, no_wasm_opt)
+        clean_contract(contract_build_subfolder)
+        build_contract(contract_build_subfolder, output_subfolder, cargo_target_dir, no_wasm_opt)
 
         # We do not clean the "output" folder, since it will be included in one of the generated archives.
-        clean_contract(build_folder, clean_output=False)
+        clean_contract(contract_build_subfolder, clean_output=False)
 
-        promote_cargo_lock_to_contract_folder(build_folder, contract_folder)
+        promote_cargo_lock_to_contract_folder(contract_build_subfolder, contract_folder)
 
         # The archives are created after build, so that Cargo.lock files are included (if previously missing).
-        create_archives(contract_name, contract_version, build_folder, output_subfolder)
-        create_packaged_source_code(project_within_build_folder, contract_name, contract_version, build_folder, output_subfolder)
+        create_archives(contract_name, contract_version, contract_build_subfolder, output_subfolder)
+        create_packaged_source_code(project_within_build_folder, contract_name, contract_version, contract_build_subfolder, output_subfolder)
 
-        outcome.gather_artifacts(contract_name, build_folder, output_subfolder)
+        outcome.gather_artifacts(contract_name, contract_build_subfolder, output_subfolder)
 
     return outcome
 
@@ -141,5 +145,5 @@ def create_packaged_source_code(parent_project_folder: Path, contract_name: str,
 
 
 def warn_file_too_large(path: Path, size: int, max_size: int):
-    logging.warning(f"""File is too large (this might cause issues with using downstream applications, such as the contract build verification services): 
+    logging.warning(f"""File is too large (this might cause issues with using downstream applications, such as the contract build verification services):
 file = {path}, size = {size}, maximum size = {max_size}""")