Fix source code packaging

.github/workflows/publish.yml

@@ -1,7 +1,9 @@
-name: Build & publish
+name: Build & publish image
 
 on:
   workflow_dispatch:
+  release:
+    types: [published]
 
 jobs:
   push_to_registry:

.github/workflows/run_long_integration_tests.yml

@@ -0,0 +1,25 @@
+name: Long integration tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build image
+        run: |
+          docker buildx build --output type=docker --no-cache . -t sdk-rust-contract-builder:next -f ./Dockerfile
+
+      - name: Build
+        run: |
+          export PYTHONPATH=.
+          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3" "b.1" "b.2" "b.3" "c.1" "c.2" "c.3" "c.4" "c.5" "d.1" "e.1"

.github/workflows/run_regular_integration_tests.yml

@@ -0,0 +1,25 @@
+name: Regular integration tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build image
+        run: |
+          docker buildx build --output type=docker --no-cache . -t sdk-rust-contract-builder:next -f ./Dockerfile
+
+      - name: Build
+        run: |
+          export PYTHONPATH=.
+          python ./integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

.gitignore

@@ -132,4 +132,6 @@ dmypy.json
 typings/**
 
 # Test data
-testdata/**/output
+testdata/input/extracted
+testdata/output
+testdata/rust

Dockerfile

@@ -1,18 +1,22 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG VERSION_RUST="nightly-2022-12-08"
+ARG VERSION_RUST="nightly-2022-10-16"
 ARG VERSION_BINARYEN="105-1"
 ARG VERSION_WABT="1.0.27-1"
+# Normally, this should be "multiversx/sdk-rust-contract-builder:{{pyproject.toml:project:version}}"
+ARG CONTEXT="multiversx/sdk-rust-contract-builder:v4.1.0"
 
 # Install dependencies (including binaryen and wabt)
 RUN apt-get update && apt-get install -y \
     wget \ 
     build-essential \
-    python3.10 python-is-python3 \
+    python3.11 python-is-python3 python3-pip \
     binaryen=${VERSION_BINARYEN} \
     wabt=${VERSION_WABT}
 
+RUN pip3 install tomlkit==0.11.6
+
 # Install rust
 RUN wget -O rustup.sh https://sh.rustup.rs && \
     chmod +x rustup.sh && \
@@ -25,6 +29,7 @@ COPY "multiversx_sdk_rust_contract_builder" "/multiversx_sdk_rust_contract_build
 ENV PATH="/rust/bin:${PATH}"
 ENV CARGO_HOME="/rust"
 ENV RUSTUP_HOME="/rust"
+ENV CONTEXT=${CONTEXT}
 ENV PYTHONPATH=/
 
 # Additional arguments (must be provided at "docker run"):

README.md

@@ -1,56 +1,66 @@
-# mx-sdk-build-contract
+# mx-sdk-rust-contract-builder
 
 Docker image (and wrappers) for reproducible contract builds (Rust). See [docs.multiversx.com](https://docs.multiversx.com/developers/reproducible-contract-builds/).
 
 ## Build the Docker image
 
+We use `docker buildx` to build the image:
+
+```
+docker buildx build --output type=docker --no-cache . -t sdk-rust-contract-builder:next -f ./Dockerfile
 ```
-docker buildx build --no-cache . -t sdk-rust-contract-builder:experimental -f ./Dockerfile
+
+Maintainers can publish the image as follows:
+
+```
+docker buildx create --name multiarch --use
+
+docker buildx build --no-cache --push --platform=linux/amd64,linux/arm64 . -t multiversx/sdk-rust-contract-builder:next -f ./Dockerfile
+
+docker buildx rm multiarch
 ```
 
+For the above to work properly, make sure to install `tonistiigi/binfmt` beforehand. Please follow the official Docker documentation [here](https://docs.docker.com/build/building/multi-platform/).
+
+Though, note that currently (January 2023) we recommend against using the `linux/arm64` image for performing reproducible contract builds. This is because, in some (possibly rare) circumstances, a WASM binary generated on the `linux/amd64` image _might_ differ (at the bytecode level) from one generated on the `linux/arm64` image - probably due to distinct (unfortunate) bytecode-emitting logic in the Rust compiler.
+
 ## Build contract using the wrapper
 
-Without providing `cargo-target-dir`:
+If you are using a Mac with ARM64, we _recommend_ setting the following variable beforehand (contract builds will be slower, but this eliminates the risk of not being able to reproduce the build on Linux):
 
 ```
-python3 ./build_with_docker.py --image=sdk-rust-contract-builder:experimental \
-    --project=~/contracts/reproducible-contract-build-example \
-    --output=~/contracts/output-from-docker
+export DOCKER_DEFAULT_PLATFORM=linux/amd64
 ```
 
-With providing `cargo-target-dir`:
+Building from a project folder:
 
 ```
-python3 ./build_with_docker.py --image=sdk-rust-contract-builder:experimental \
-    --project=~/contracts/reproducible-contract-build-example \
-    --output=~/contracts/output-from-docker \
-    --cargo-target-dir=~/cargo-target-dir-docker
+python3 ./build_with_docker.py --image=sdk-rust-contract-builder:next \
+    --project=~/contracts/example \
+    --output=~/contracts/output-from-docker
 ```
 
 Building from a packaged source code:
 
 ```
-python3 ./build_with_docker.py --image=sdk-rust-contract-builder:experimental \
+python3 ./build_with_docker.py --image=sdk-rust-contract-builder:next \
     --packaged-src=~/contracts/example-0.0.0.source.json \
     --output=~/contracts/output-from-docker
 ```
 
-## Build contract using the Docker inner script
-
-This is useful for useful for testing, debugging and reviewing the script.
+## Run unit tests (without Docker)
 
 ```
-export PROJECT=${HOME}/contracts/reproducible-contract-build-example
-export OUTPUT=${HOME}/contracts/output
-export CARGO_TARGET_DIR=${HOME}/cargo-target-dir
 export PATH=${HOME}/multiversx-sdk/vendor-rust/bin:${HOME}/multiversx-sdk/wabt/latest/bin:${PATH}
 export RUSTUP_HOME=${HOME}/multiversx-sdk/vendor-rust
 export CARGO_HOME=${HOME}/multiversx-sdk/vendor-rust
+
+pytest .
 ```
 
-Build a project:
+## Run integration tests (with Docker)
 
 ```
-python3 ./build_within_docker.py --project=${PROJECT} --output=${OUTPUT} \
-    --cargo-target-dir=${CARGO_TARGET_DIR}
+python3 ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" [...]
+python3 ./integration_tests/test_project_folder_and_packaged_src_are_equivalent.py
 ```

build_with_docker.py

@@ -75,7 +75,7 @@ def run_docker(
         docker_mount_args.extend(["--volume", f"{packaged_src_path}:/packaged-src.json"])
 
     if cargo_target_dir:
-        docker_mount_args += ["--volume", f"{cargo_target_dir}:/cargo-target-dir"]
+        docker_mount_args += ["--volume", f"{cargo_target_dir}:/rust/cargo-target-dir"]
 
     docker_args = ["docker", "run"]
 

integration_tests/__init__.py

@@ -0,0 +1 @@
+__all__ = []

integration_tests/config.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+DOWNLOADS_FOLDER = Path("./testdata/downloads").resolve()
+EXTRACTED_FOLDER = Path("./testdata/input/extracted").resolve()
+PARENT_OUTPUT_FOLDER = Path("./testdata/output").resolve()
+CARGO_TARGET_DIR = Path("./testdata/rust/cargo_target_dir").resolve()
+RUST_REGISTRY = Path("./testdata/rust/registry").resolve()
+RUST_GIT = Path("./testdata/rust/git").resolve()
+RUST_TMP = Path("./testdata/rust/tmp").resolve()