V4 (LTS): Package whole project source code

.github/workflows/run_long_integration_tests.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Build
         run: |
           export PYTHONPATH=.
-          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3" "a.4"
+          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3" "a.4" "a.5" "a.6" "a.7" "a.8" "a.9" "a.10" "a.11" "a.12" "a.13"
 
       - name: Save artifacts
         uses: actions/upload-artifact@v3

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.2.1"
+ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v4.3.0"
 ARG VERSION_RUST="nightly-2022-10-16"
 ARG VERSION_BINARYEN="version_105"
 ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz"

build_with_docker.py

@@ -20,7 +20,7 @@ def main(cli_args: List[str]):
     parser.add_argument("--packaged-src", type=str, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str)
     parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output")
-    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="deprecated parameter, not used anymore")
     parser.add_argument("--cargo-target-dir", help="deprecated parameter, not used anymore")
     parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)")
     parser.add_argument("--build-root", type=str, required=False, help="root path (within container) for the build (default: %(default)s)")
@@ -35,7 +35,6 @@ def main(cli_args: List[str]):
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     contract_path = parsed_args.contract
     output_path = Path(parsed_args.output).expanduser().resolve()
-    package_whole_project_src = parsed_args.package_whole_project_src
     no_wasm_opt = parsed_args.no_wasm_opt
     build_root = Path(parsed_args.build_root) if parsed_args.build_root else None
     cargo_verbose = parsed_args.cargo_verbose
@@ -97,9 +96,6 @@ def main(cli_args: List[str]):
     if build_root:
         entrypoint_args.extend(["--build-root", str(build_root)])
 
-    if package_whole_project_src:
-        entrypoint_args.append("--package-whole-project-src")
-
     # Run docker container
     args = docker_general_args + docker_mount_args + docker_env_args + [image] + entrypoint_args
     logger.info(f"Running docker: {args}")

integration_tests/previous_builds.py

@@ -22,57 +22,159 @@ def __init__(self, name: str,
 previous_builds: List[PreviousBuild] = [
     PreviousBuild(
         name="a.1",
-        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.0.zip",
-        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.4.0",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.4.1-pair-safe-price-v2.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.4.1-pair-safe-price-v2",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
-            "adder": "9fd12f88f9474ba115fb75e9d18a8fdbc4f42147de005445048442d49c3aa725",
-            "multisig": "2101bc2a7a31ea42e5ffaadd86c1640009690e93b1cb46c3566ba5eac2984e36",
-            "multisig-full": "ef468403354b6d3a728f86101354359fe6864187d216f674d99b31fc05313a39",
-            "multisig-view": "3690af76be10c0520e3c3545cde8d9ef6a15c2d0af74dbd8704b4909644049c9"
+            "pair": "e9f117971963cb3c24b14e2a7698d48c170335af2f5c8167774c48c3c1c654e3",
+            "pair-full": "f1af2b2bb42a9f035745777e4b2d4f72478569224f204d0d0103801faff9663a",
+            "safe-price-view": "b5a657445ae74423c60210c88a6fa89b0bd4bdd00d5f06e788e14495bccc34c9"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v5.1.0"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="a.2",
-        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.3.zip",
-        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.4.3",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.5.2-governance-merkle-tree.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.5.2-governance-merkle-tree",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
-            "adder": "9fd12f88f9474ba115fb75e9d18a8fdbc4f42147de005445048442d49c3aa725",
-            "multisig": "b73050629c11b1f1a20ca6232abcef07897624195691552e3f2e2fce47822166",
-            "multisig-full": "37c3b90bdaa7d8d203385c91b0b5cb4d3c444ab9ec5263351978046a545854e3",
-            "multisig-view": "ebaf987b041fcda297da71291d76736e4e98a1e449e5ec37908cdc0198e8be37"
+            "lkmex-transfer": "49809df9f07839f965f8197721083cf403c3db969ddc47b5940b4ee8b464af92",
+            "router": "3257d57945736298c96aa23f99ea0fba3b6da01f9d2103d81230d05cff62cb5a",
+            "pair": "e9f117971963cb3c24b14e2a7698d48c170335af2f5c8167774c48c3c1c654e3",
+            "locked-token-wrapper": "1d317cfa2bbe22ea1f878f8a32f90712d49f68a01665b86657d36910b430522f",
+            "safe-price-view": "b5a657445ae74423c60210c88a6fa89b0bd4bdd00d5f06e788e14495bccc34c9"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v5.3.0"
+        docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="a.3",
-        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.4.1-pair-safe-price-v2.zip",
-        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.4.1-pair-safe-price-v2",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.10.4-backwards-comp-proxy-dex.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.10.4-backwards-comp-proxy-dex",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
-            "pair": "e9f117971963cb3c24b14e2a7698d48c170335af2f5c8167774c48c3c1c654e3",
-            "pair-full": "f1af2b2bb42a9f035745777e4b2d4f72478569224f204d0d0103801faff9663a",
-            "safe-price-view": "b5a657445ae74423c60210c88a6fa89b0bd4bdd00d5f06e788e14495bccc34c9"
+            "farm-staking-proxy": "ee61abb8e639df4696900c5efdac6bec747679c149be26476ee7e384e35b1ff2",
+            "proxy_dex": "8bab3716a1a92bad1b5cb77b97a09cce1ffabd56b954d881b93e1b480984c3d2",
+            "factory": "b75f481df42c076f51cd7af04b914bc581e51784b31136ae27db368b37bd87b1",
+            "farm_with_lock": "6b9d9a0f6bba6004c7c1163890ea0794cc61ab16c967e1c5c2cd2b8a7c19ebbf",
         },
         docker_image="sdk-rust-contract-builder:next"
     ),
     PreviousBuild(
         name="a.4",
-        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.5.2-governance-merkle-tree.zip",
-        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.5.2-governance-merkle-tree",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.1.1-staking-upgrade.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.1.1-staking-upgrade",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
-            "lkmex-transfer": "49809df9f07839f965f8197721083cf403c3db969ddc47b5940b4ee8b464af92",
-            "router": "3257d57945736298c96aa23f99ea0fba3b6da01f9d2103d81230d05cff62cb5a",
-            "pair": "e9f117971963cb3c24b14e2a7698d48c170335af2f5c8167774c48c3c1c654e3",
-            "locked-token-wrapper": "1d317cfa2bbe22ea1f878f8a32f90712d49f68a01665b86657d36910b430522f",
-            "safe-price-view": "b5a657445ae74423c60210c88a6fa89b0bd4bdd00d5f06e788e14495bccc34c9"
+            "farm-staking": "6dc7c587b2cc4b177a192b709c092f3752b3dcf9ce1b484e69fe64dc333a9e0a",
+            "farm": "931ca233826ff9dacd889967365db1cde9ed8402eb553de2a3b9d58b6ff1098d",
+            "factory": "df06465b651594605466e817bfe9d8d7c68eef0f87df4a8d3266bcfb1bef6d83",
+            "pair": "f3f08ebd758fada871c113c18017d9761f157d00b19c4d3beaba530e6c53afc2",
+            "energy-factory": "241600c055df605cafd85b75d40b21316a6b35713485201b156d695b23c66a2f"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.5",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.0-rc6-reproducible.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.0-rc6-reproducible",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "pair": "23ce1e8910c105410b4a417153e4b38c550ab78b38b899ea786f0c78500caf21",
+            "simple-lock": "303290b7a08b091c29315dd6979c1f745fc05467467d7de64e252592074890a7",
+            "farm-staking-proxy": "56468a6ae726693a71edcf96cf44673466dd980412388e1e4b073a0b4ee592d7"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.6",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.10.2-legacy-farm-stripdown.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.10.2-legacy-farm-stripdown",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "farm": "bac43c58b865f55f303ae2d4100c5fe2d4492bc50cfb131d8206200039808242"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.7",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.10.3-backwards-comp-contracts.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.10.3-backwards-comp-contracts",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "factory": "70ef785d7574746cadfb7eec0e52cc68d6807ddbb54cb1365b3942cf903c12d2"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.8",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.1.3-price-discovery-comp-upgrade.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.1.3-price-discovery-comp-upgrade",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "price-discovery": "96b51ec9df3eb7a8e72f297aac2c8e4e609e39ac5a5f6d861c0819d010b87fde"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.9",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.9.1-governance-lkmex.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.9.1-governance-lkmex",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "governance-v2": "7c9c92d914542939c611aaabf21f17e9f71a29e196da319bcfdddd33c409f91d"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.10",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.5.4-metabonding-unbond.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.5.4-metabonding-unbond",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "metabonding-staking": "4a9b2afa13eca738b1804c48b82a961afd67adcbbf2aa518052fa124ac060bea"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.11",
+        project_archive_url="https://github.com/multiversx/mx-metabonding-sc/archive/refs/heads/reproducible-v1.1.1.zip",
+        project_relative_path_in_archive="mx-metabonding-sc-reproducible-v1.1.1",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "metabonding": "897b19e1990f7c487c99c12f50722febe1ee4468bcd3a7405641966dfff2791d"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.12",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.8.1-simple-lock-role.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.8.1-simple-lock-role",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "simple-lock": "90d7206ba3866d2e17433faead5b76d345d60c4b311f7c030ebddc0fd5677a00"
+        },
+        docker_image="sdk-rust-contract-builder:next"
+    ),
+    PreviousBuild(
+        name="a.13",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v1.8.2-simple-lock-whitelist-old-farms.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v1.8.2-simple-lock-whitelist-old-farms",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "simple-lock-whitelist": "c8868cf7599a94191ac327a81803eb9cee6e35ffddb007374a2bbbf6258d378f"
         },
         docker_image="sdk-rust-contract-builder:next"
     )

integration_tests/shared.py

@@ -31,7 +31,6 @@ def download_packaged_src(json_url: str, name: str) -> Path:
 
 def run_docker(
     project_path: Optional[Path],
-    package_whole_project_src: bool,
     packaged_src_path: Optional[Path],
     contract_name: Optional[str],
     image: str,
@@ -66,9 +65,6 @@ def run_docker(
     if project_path:
         entrypoint_args.extend(["--project", "project"])
 
-    if package_whole_project_src:
-        entrypoint_args.append("--package-whole-project-src")
-
     if packaged_src_path:
         entrypoint_args.extend(["--packaged-src", "packaged-src.json"])
 

integration_tests/test_previous_builds_are_reproducible.py

@@ -41,7 +41,6 @@ def main(cli_args: List[str]):
 
         run_docker(
             project_path=project_path,
-            package_whole_project_src=False,
             packaged_src_path=packaged_src_path,
             contract_name=build.contract_name,
             image=build.docker_image,

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -17,20 +17,18 @@ def main(cli_args: List[str]):
 
     check_project_folder_and_packaged_src_are_equivalent(
         project_path=project_path,
-        package_whole_project_src=True,
         parent_output_folder=PARENT_OUTPUT_FOLDER,
         contracts=["pair", "farm"],
     )
 
 
 def check_project_folder_and_packaged_src_are_equivalent(
         project_path: Path,
-        package_whole_project_src: bool,
         parent_output_folder: Path,
         contracts: List[str]):
     for contract in contracts:
-        output_using_project = parent_output_folder / "using-project" / contract / ("whole" if package_whole_project_src else "truncated")
-        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract / ("whole" if package_whole_project_src else "truncated")
+        output_using_project = parent_output_folder / "using-project" / contract
+        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract
 
         shutil.rmtree(output_using_project, ignore_errors=True)
         shutil.rmtree(output_using_packaged_src, ignore_errors=True)
@@ -40,18 +38,16 @@ def check_project_folder_and_packaged_src_are_equivalent(
 
         run_docker(
             project_path=project_path,
-            package_whole_project_src=package_whole_project_src,
             packaged_src_path=None,
             contract_name=contract,
             image="sdk-rust-contract-builder:next",
             output_folder=output_using_project
         )
 
-        packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
+        packaged_src_path = next((output_using_project / contract).glob("*.source.json"))
 
         run_docker(
             project_path=None,
-            package_whole_project_src=package_whole_project_src,
             packaged_src_path=packaged_src_path,
             contract_name=contract,
             image="sdk-rust-contract-builder:next",
@@ -64,14 +60,21 @@ def check_project_folder_and_packaged_src_are_equivalent(
 
         assert len(using_project_output_files) == len(using_packaged_src_output_files)
 
-        for index, file in enumerate(using_project_output_files):
-            if not file.is_file() or file.suffix == ".zip":
-                continue
-            using_project_file_content = file.read_bytes()
-            using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
+        for index, file_using_project in enumerate(using_project_output_files):
+            file_using_packaged_src = using_packaged_src_output_files[index]
 
-            if using_project_file_content != using_packaged_src_file_content:
-                raise Exception(f"Files differ ({contract}): {file.name}")
+            if not file_using_project.is_file() or file_using_project.suffix == ".zip":
+                continue
+            file_content_using_project = file_using_project.read_bytes()
+            file_content_using_packaged_src = file_using_packaged_src.read_bytes()
+
+            if file_content_using_project == file_content_using_packaged_src:
+                print(f"Files are identical ({contract}): {file_using_project.name}")
+            else:
+                print(f"Files differ ({contract}):")
+                print(f"  {file_using_project}")
+                print(f"  {file_using_packaged_src}")
+                raise Exception(f"Files differ ({contract}): {file_using_project.name}")
 
 
 if __name__ == "__main__":

multiversx_sdk_rust_contract_builder/build_options.py

@@ -6,21 +6,20 @@
 class BuildOptions:
     def __init__(
         self,
-        package_whole_project_src: bool,
         specific_contract: str,
         cargo_target_dir: Path,
         no_wasm_opt: bool,
         build_root_folder: Path,
     ) -> None:
-        self.package_whole_project_src = package_whole_project_src
         self.specific_contract = specific_contract
         self.cargo_target_dir = cargo_target_dir
         self.no_wasm_opt = no_wasm_opt
         self.build_root_folder = build_root_folder
 
     def to_dict(self) -> Dict[str, Any]:
         return {
-            "packageWholeProjectSrc": self.package_whole_project_src,
+            # "packageWholeProjectSrc" is kept due to compatibility reasons.
+            "packageWholeProjectSrc": True,
             "specificContract": self.specific_contract,
             "cargoTargetDir": str(self.cargo_target_dir),
             "noWasmOpt": self.no_wasm_opt,

multiversx_sdk_rust_contract_builder/build_outcome.py

@@ -70,6 +70,7 @@ def many_from_folders(cls, build_folder: Path, output_folder: Path) -> Dict[str,
             entry.codehash = find_file_in_folder(output_folder, f"{contract_name}.codehash.txt").read_text()
             entry.bytecode_path = BuildArtifact.find_in_output(f"{contract_name}.wasm", output_folder)
             entry.abi_path = BuildArtifact.find_in_output(f"{contract_name}.abi.json", output_folder)
+            # This is the whole project source code. The file *.partial-source.json is not listed here - so that it's advertised as little as possible.
             entry.src_package_path = BuildArtifact.find_in_output("*.source.json", output_folder)
 
             result[contract_name] = entry