Merge pull request #53 from multiversx/whole-src-11-jan

Package whole project source code

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v6.0.0"
+ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v6.1.0"
 ARG VERSION_RUST="nightly-2023-12-11"
 ARG VERSION_BINARYEN="version_112"
 ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz"

build_with_docker.py

@@ -20,7 +20,7 @@ def main(cli_args: List[str]):
     parser.add_argument("--packaged-src", type=str, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str)
     parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output")
-    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="deprecated parameter, not used anymore")
     parser.add_argument("--cargo-target-dir", help="deprecated parameter, not used anymore")
     parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)")
     parser.add_argument("--build-root", type=str, required=False, help="root path (within container) for the build (default: %(default)s)")
@@ -35,7 +35,6 @@ def main(cli_args: List[str]):
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     contract_path = parsed_args.contract
     output_path = Path(parsed_args.output).expanduser().resolve()
-    package_whole_project_src = parsed_args.package_whole_project_src
     no_wasm_opt = parsed_args.no_wasm_opt
     build_root = Path(parsed_args.build_root) if parsed_args.build_root else None
     cargo_verbose = parsed_args.cargo_verbose
@@ -97,9 +96,6 @@ def main(cli_args: List[str]):
     if build_root:
         entrypoint_args.extend(["--build-root", str(build_root)])
 
-    if package_whole_project_src:
-        entrypoint_args.append("--package-whole-project-src")
-
     # Run docker container
     args = docker_general_args + docker_mount_args + docker_env_args + [image] + entrypoint_args
     logger.info(f"Running docker: {args}")

integration_tests/previous_builds.py

@@ -48,32 +48,34 @@ def __init__(self, name: str,
     ),
     PreviousBuild(
         name="a.3",
-        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.7.zip",
-        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.4.7",
+        project_archive_url="https://github.com/multiversx/mx-contracts-rs/archive/refs/tags/v0.45.2.1-reproducible.zip",
+        project_relative_path_in_archive="mx-contracts-rs-0.45.2.1-reproducible",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
-            "adder": "9fd12f88f9474ba115fb75e9d18a8fdbc4f42147de005445048442d49c3aa725",
-            "multisig": "9600fc699c85fd5a24ecf28f0b8cf01dc281c81399fb018d5ad8405b7d401041",
-            "multisig-full": "9eed9c35113209fc69631cf29aac6e81f0e331132bf6e46198e679259075ad49",
-            "multisig-view": "3993cf3fb5cd18102e2b8946ea1997f6f1cc512537f453265ba1afd7378fc0c6",
-            "lottery-esdt": "e06b1a5c7fb71181a79e9be6b86d8ad154e5c2def4da6d2f0aa5266163823291"
+            "adder": "384b680df7a95ebceca02ffb3e760a2fc288dea1b802685ef15df22ae88ba15b",
+            "multisig": "b82f074c02e308b80cfb7144d7dc959bfac73e14dc3291837fdd8b042a7739cf",
+            "multisig-full": "44a0eafb3bedfd671d1df586313f716924e2e4ef00ae7bf26df2c11eb4291389",
+            "multisig-view": "d3e8328d525fcf196bb5bb4ce0741d9146dccb475461a693c407cdfa02334789",
+            "lottery-esdt": "e06b1a5c7fb71181a79e9be6b86d8ad154e5c2def4da6d2f0aa5266163823291",
+            "ping-pong-egld": "9283ca2f077edf2704053f0973fdd1eb90ee871ddcd672f962de4ba4422df84b"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v5.4.0"
+        docker_image="multiversx/sdk-rust-contract-builder:v5.4.1"
     ),
     PreviousBuild(
         name="a.4",
-        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.5.0-beta.0.zip",
-        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.5.0-beta.0",
+        project_archive_url="https://github.com/multiversx/mx-contracts-rs/archive/refs/tags/v0.45.2.1-reproducible.zip",
+        project_relative_path_in_archive="mx-contracts-rs-0.45.2.1-reproducible",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
             "adder": "384b680df7a95ebceca02ffb3e760a2fc288dea1b802685ef15df22ae88ba15b",
-            "multisig": "d1453017d1fcac43f3b54c390b112b37af38ae840a2464d8ff68e3981da9972d",
-            "multisig-full": "e7fd6d118639e4b4381b3a667435948cd70f1f06b6ef39e227bd88349f7e7979",
-            "multisig-view": "290c9b3e374dffa33649ed46bd0b626c66f933eff9437f11f3559372d7538f85",
-            "lottery-esdt": "d4d4b6d2d797749435a4127a12d5ea16b911d6783e00cbc9faf4bace7d655c7a"
+            "multisig": "87cb62542c9b2d0b5a791cb35f7e44e71bb6d768d6ddb93155be61ad76267475",
+            "multisig-full": "f6b5457682b39ea1bd52fd6fe293257a3d5a5bb931c9e404c9ba24617cd51438",
+            "multisig-view": "1904fe0bfd12cb90fda87e5cf2d2f211d9eed8b48c296e6d858547bfe39bec0c",
+            "lottery-esdt": "a54bd4278b12cc93fedd6ca0addf6aad4043528c33e54ce43cf92d4d2dd755ee",
+            "ping-pong-egld": "8b107da10aef0d9610a939c4ca07c666674c465d0266fb28d5f981861f084f62"
         },
         docker_image="sdk-rust-contract-builder:next"
-    )
+    ),
 ]

integration_tests/shared.py

@@ -31,7 +31,6 @@ def download_packaged_src(json_url: str, name: str) -> Path:
 
 def run_docker(
     project_path: Optional[Path],
-    package_whole_project_src: bool,
     packaged_src_path: Optional[Path],
     contract_name: Optional[str],
     image: str,
@@ -66,9 +65,6 @@ def run_docker(
     if project_path:
         entrypoint_args.extend(["--project", "project"])
 
-    if package_whole_project_src:
-        entrypoint_args.append("--package-whole-project-src")
-
     if packaged_src_path:
         entrypoint_args.extend(["--packaged-src", "packaged-src.json"])
 

integration_tests/test_previous_builds_are_reproducible.py

@@ -41,7 +41,6 @@ def main(cli_args: List[str]):
 
         run_docker(
             project_path=project_path,
-            package_whole_project_src=False,
             packaged_src_path=packaged_src_path,
             contract_name=build.contract_name,
             image=build.docker_image,

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -8,30 +8,26 @@
 
 
 def main(cli_args: List[str]):
-    repository_url = "https://github.com/multiversx/mx-reproducible-contract-build-example-sc"
-    tag = "0.5.0-beta.0"
-    archve_subfolder = f"mx-reproducible-contract-build-example-sc-{tag}"
+    repository_url = "https://github.com/multiversx/mx-contracts-rs"
+    tag = "0.45.2.1-reproducible"
+    archve_subfolder = f"mx-contracts-rs-{tag}"
     project_path = download_project_repository(f"{repository_url}/archive/refs/tags/v{tag}.zip", archve_subfolder)
     project_path = project_path / archve_subfolder
 
-    # Only package_whole_project_src = True works.
-    # package_whole_project_src = False does not work, since a missing Cargo.lock at the workspace level leads to build errors.
     check_project_folder_and_packaged_src_are_equivalent(
         project_path=project_path,
-        package_whole_project_src=True,
         parent_output_folder=PARENT_OUTPUT_FOLDER,
-        contracts=["adder", "multisig"],
+        contracts=["adder", "multisig", "lottery-esdt"],
     )
 
 
 def check_project_folder_and_packaged_src_are_equivalent(
         project_path: Path,
-        package_whole_project_src: bool,
         parent_output_folder: Path,
         contracts: List[str]):
     for contract in contracts:
-        output_using_project = parent_output_folder / "using-project" / contract / ("whole" if package_whole_project_src else "truncated")
-        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract / ("whole" if package_whole_project_src else "truncated")
+        output_using_project = parent_output_folder / "using-project" / contract
+        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract
 
         shutil.rmtree(output_using_project, ignore_errors=True)
         shutil.rmtree(output_using_packaged_src, ignore_errors=True)
@@ -41,18 +37,16 @@ def check_project_folder_and_packaged_src_are_equivalent(
 
         run_docker(
             project_path=project_path,
-            package_whole_project_src=package_whole_project_src,
             packaged_src_path=None,
             contract_name=contract,
             image="sdk-rust-contract-builder:next",
             output_folder=output_using_project
         )
 
-        packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
+        packaged_src_path = next((output_using_project / contract).glob("*.source.json"))
 
         run_docker(
             project_path=None,
-            package_whole_project_src=package_whole_project_src,
             packaged_src_path=packaged_src_path,
             contract_name=contract,
             image="sdk-rust-contract-builder:next",

multiversx_sdk_rust_contract_builder/build_options.py

@@ -6,21 +6,20 @@
 class BuildOptions:
     def __init__(
         self,
-        package_whole_project_src: bool,
         specific_contract: str,
         cargo_target_dir: Path,
         no_wasm_opt: bool,
         build_root_folder: Path,
     ) -> None:
-        self.package_whole_project_src = package_whole_project_src
         self.specific_contract = specific_contract
         self.cargo_target_dir = cargo_target_dir
         self.no_wasm_opt = no_wasm_opt
         self.build_root_folder = build_root_folder
 
     def to_dict(self) -> Dict[str, Any]:
         return {
-            "packageWholeProjectSrc": self.package_whole_project_src,
+            # "packageWholeProjectSrc" is kept due to compatibility reasons.
+            "packageWholeProjectSrc": True,
             "specificContract": self.specific_contract,
             "cargoTargetDir": str(self.cargo_target_dir),
             "noWasmOpt": self.no_wasm_opt,

multiversx_sdk_rust_contract_builder/build_outcome.py

@@ -70,6 +70,7 @@ def many_from_folders(cls, build_folder: Path, output_folder: Path) -> Dict[str,
             entry.codehash = find_file_in_folder(output_folder, f"{contract_name}.codehash.txt").read_text()
             entry.bytecode_path = BuildArtifact.find_in_output(f"{contract_name}.wasm", output_folder)
             entry.abi_path = BuildArtifact.find_in_output(f"{contract_name}.abi.json", output_folder)
+            # This is the whole project source code. The file *.partial-source.json is not listed here - so that it's advertised as little as possible.
             entry.src_package_path = BuildArtifact.find_in_output("*.source.json", output_folder)
 
             result[contract_name] = entry

multiversx_sdk_rust_contract_builder/builder.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Set
 
-from multiversx_sdk_rust_contract_builder import cargo_toml, source_code
+from multiversx_sdk_rust_contract_builder import source_code
 from multiversx_sdk_rust_contract_builder.build_metadata import BuildMetadata
 from multiversx_sdk_rust_contract_builder.build_options import BuildOptions
 from multiversx_sdk_rust_contract_builder.build_outcome import BuildOutcome
@@ -31,7 +31,6 @@ def build_project(
     project_folder = project_folder.expanduser().resolve()
     parent_output_folder = parent_output_folder.expanduser().resolve()
     cargo_target_dir = options.cargo_target_dir.expanduser().resolve()
-    package_whole_project_src = options.package_whole_project_src
     no_wasm_opt = options.no_wasm_opt
     specific_contract = options.specific_contract
     build_root_folder = options.build_root_folder
@@ -45,9 +44,6 @@ def build_project(
     # We copy the whole project folder to the build path, to ensure that all local dependencies are available.
     project_within_build_folder = copy_project_folder_to_build_folder(project_folder, build_root_folder)
 
-    if not package_whole_project_src:
-        cargo_toml.remove_dev_dependencies_sections_from_all(project_within_build_folder)
-
     for contract_folder in sorted(contracts_folders):
         contract_name, contract_version = get_contract_name_and_version(contract_folder)
         logging.info(f"Contract = {contract_name}, version = {contract_version}")
@@ -75,13 +71,23 @@ def build_project(
         # The bundle (packaged source code) is created after build, so that Cargo.lock files are included (if previously missing).
         create_packaged_source_code(
             parent_project_folder=project_within_build_folder,
-            package_whole_project_src=package_whole_project_src,
+            package_whole_project_src=True,
             contract_folder=contract_build_subfolder,
             output_folder=output_subfolder,
             build_metadata=metadata.to_dict(),
             build_options=options.to_dict(),
+            package_filename=f"{contract_name}-{contract_version}.source.json"
         )
 
+        create_packaged_source_code(
+            parent_project_folder=project_within_build_folder,
+            package_whole_project_src=False,
+            contract_folder=contract_build_subfolder,
+            output_folder=output_subfolder,
+            build_metadata=metadata.to_dict(),
+            build_options=options.to_dict(),
+            package_filename=f"{contract_name}-{contract_version}.partial-source.json"
+        )
         outcome.gather_artifacts(contract_build_subfolder, output_subfolder)
 
     return outcome
@@ -168,7 +174,8 @@ def create_packaged_source_code(
         contract_folder: Path,
         output_folder: Path,
         build_metadata: Dict[str, Any],
-        build_options: Dict[str, Any]
+        build_options: Dict[str, Any],
+        package_filename: str
 ):
     source_code_files = source_code.get_source_code_files(
         project_folder=parent_project_folder,
@@ -185,7 +192,7 @@ def create_packaged_source_code(
     )
 
     package = PackagedSourceCode.from_filesystem(metadata, parent_project_folder, source_code_files)
-    package_path = output_folder / f"{contract_name}-{contract_version}.source.json"
+    package_path = output_folder / package_filename
     package.save_to_file(package_path)
 
     size_of_file = package_path.stat().st_size

multiversx_sdk_rust_contract_builder/cargo_toml.py

@@ -1,12 +1,9 @@
-import logging
 import shutil
 from pathlib import Path
 from typing import Tuple
 
 import toml
 
-from multiversx_sdk_rust_contract_builder.filesystem import get_all_files
-
 
 def get_contract_name_and_version(contract_folder: Path) -> Tuple[str, str]:
     file = contract_folder / "Cargo.toml"
@@ -21,19 +18,3 @@ def promote_cargo_lock_to_contract_folder(build_folder: Path, contract_folder: P
     from_path = build_folder / "wasm" / "Cargo.lock"
     to_path = contract_folder / "wasm" / "Cargo.lock"
     shutil.copy(from_path, to_path)
-
-
-def remove_dev_dependencies_sections_from_all(folder: Path):
-    logging.info(f"remove_dev_dependencies_sections_from_all({folder})")
-
-    all_files = get_all_files(folder, lambda file: file.name == "Cargo.toml")
-    for file in all_files:
-        remove_dev_dependencies_sections(file)
-
-
-def remove_dev_dependencies_sections(file: Path):
-    data = toml.loads(file.read_text())
-
-    if "dev-dependencies" in data:
-        del data["dev-dependencies"]
-        file.write_text(toml.dumps(data))

multiversx_sdk_rust_contract_builder/main.py

@@ -24,7 +24,7 @@ def main(cli_args: List[str]):
 
     parser = ArgumentParser()
     parser.add_argument("--project", type=str, required=False, help="source code folder (project)")
-    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="deprecated parameter, not used anymore")
     parser.add_argument("--packaged-src", type=str, required=False, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str, required=False, help="contract to build from within the source code folder; should be relative to the project path")
     parser.add_argument("--output", type=str, required=True)
@@ -34,7 +34,6 @@ def main(cli_args: List[str]):
 
     parsed_args = parser.parse_args(cli_args)
     project_path = Path(parsed_args.project).expanduser().resolve() if parsed_args.project else None
-    package_whole_project_src = parsed_args.package_whole_project_src
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     parent_output_folder = Path(parsed_args.output)
     specific_contract = parsed_args.contract
@@ -55,7 +54,6 @@ def main(cli_args: List[str]):
     metadata = BuildMetadata.from_env()
 
     options = BuildOptions(
-        package_whole_project_src=package_whole_project_src,
         specific_contract=specific_contract,
         cargo_target_dir=cargo_target_dir,
         no_wasm_opt=no_wasm_opt,

multiversx_sdk_rust_contract_builder/source_code.py

@@ -61,8 +61,6 @@ def get_source_code_files(
 def _is_source_code_file(path: Path) -> bool:
     if path.suffix == ".rs":
         return True
-    if path.parent.name == "meta" and path.name == "Cargo.lock":
-        return False
     if path.name in ["Cargo.toml", "Cargo.lock", "multicontract.toml", "sc-config.toml", CONTRACT_CONFIG_FILENAME]:
         return True
     return False