Skip to content

Commit

Permalink
feat(acir)!: Program and witness stack structure (#5149)
Browse files Browse the repository at this point in the history
  • Loading branch information
vezenovm committed Mar 19, 2024
1 parent b47abcf commit ccc5016
Show file tree
Hide file tree
Showing 64 changed files with 2,905 additions and 2,297 deletions.
229 changes: 117 additions & 112 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,103 +92,6 @@ jobs:
- continuation/continue:
configuration_path: .circleci/generated_config.yml

# Noir
noir-x86_64:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir 32
aztec_manifest_key: noir

noir-arm64:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir 32 arm64
aztec_manifest_key: noir

noir-ecr-manifest:
machine:
image: default
resource_class: medium
steps:
- *checkout
- *setup_env
- run:
name: "Create ECR manifest"
command: create_ecr_manifest noir x86_64,arm64
aztec_manifest_key: noir

noir-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-tests 32
aztec_manifest_key: noir-tests

noir-packages:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-packages 32
aztec_manifest_key: noir-packages

noir-packages-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-packages-tests 32
aztec_manifest_key: noir-packages-tests

noir-compile-acir-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-compile-acir-tests 32
aztec_manifest_key: noir-compile-acir-tests

avm-transpiler:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build avm-transpiler 32
aztec_manifest_key: avm-transpiler

# Barretenberg
barretenberg-wasm-linux-clang:
docker:
Expand Down Expand Up @@ -356,7 +259,19 @@ jobs:
command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 stdlib_plonk_recursion_tests --gtest_filter=-*turbo*
aztec_manifest_key: barretenberg-x86_64-linux-clang-assert

barretenberg-acir-tests-bb:
bb-js:
machine:
image: default
resource_class: large
steps:
- *checkout
- *setup_env
- run:
name: "Build and test"
command: build bb.js
aztec_manifest_key: bb.js

bb-js-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
Expand All @@ -365,34 +280,119 @@ jobs:
- *setup_env
- run:
name: "Build and test"
command: cond_spot_run_build barretenberg-acir-tests-bb 32
aztec_manifest_key: barretenberg-acir-tests-bb
command: cond_spot_run_test bb.js 32 ./scripts/run_tests
aztec_manifest_key: bb.js

# Noir
noir-x86_64:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir 32
aztec_manifest_key: noir

barretenberg-acir-tests-bb-sol:
noir-arm64:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build and test"
command: cond_spot_run_build barretenberg-acir-tests-bb-sol 32
aztec_manifest_key: barretenberg-acir-tests-bb-sol
name: "Build"
command: cond_spot_run_build noir 32 arm64
aztec_manifest_key: noir

bb-js:
noir-ecr-manifest:
machine:
image: default
resource_class: large
resource_class: medium
steps:
- *checkout
- *setup_env
- run:
name: "Create ECR manifest"
command: create_ecr_manifest noir x86_64,arm64
aztec_manifest_key: noir

noir-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-tests 32
aztec_manifest_key: noir-tests

noir-packages:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-packages 32
aztec_manifest_key: noir-packages

noir-packages-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-packages-tests 32
aztec_manifest_key: noir-packages-tests

noir-compile-acir-tests:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build noir-compile-acir-tests 32
aztec_manifest_key: noir-compile-acir-tests

avm-transpiler:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build"
command: cond_spot_run_build avm-transpiler 32
aztec_manifest_key: avm-transpiler

barretenberg-acir-tests-bb:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
steps:
- *checkout
- *setup_env
- run:
name: "Build and test"
command: build bb.js
aztec_manifest_key: bb.js
command: cond_spot_run_build barretenberg-acir-tests-bb 32
aztec_manifest_key: barretenberg-acir-tests-bb

bb-js-tests:
barretenberg-acir-tests-bb-sol:
docker:
- image: aztecprotocol/alpine-build-image
resource_class: small
Expand All @@ -401,8 +401,8 @@ jobs:
- *setup_env
- run:
name: "Build and test"
command: cond_spot_run_test bb.js 32 ./scripts/run_tests
aztec_manifest_key: bb.js
command: cond_spot_run_build barretenberg-acir-tests-bb-sol 32
aztec_manifest_key: barretenberg-acir-tests-bb-sol

bb-js-acir-tests:
docker:
Expand Down Expand Up @@ -1272,10 +1272,14 @@ workflows:
- noir-arm64
<<: *defaults
- noir-tests: *defaults
- noir-packages: *defaults
- noir-packages:
requires:
- bb-js
<<: *defaults
- noir-packages-tests:
requires:
- noir-ecr-manifest
- noir-packages
<<: *defaults
- noir-compile-acir-tests:
requires:
Expand Down Expand Up @@ -1305,6 +1309,7 @@ workflows:
- barretenberg-acir-tests-bb-sol:
requires:
- barretenberg-x86_64-linux-clang-sol
- noir-compile-acir-tests
<<: *bb_acir_tests
- barretenberg-docs: *defaults
- bb-js:
Expand Down
1 change: 1 addition & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@
"C_Cpp.default.includePath": ["barretenberg/cpp/src"],
"rust-analyzer.linkedProjects": [
"noir/noir-repo/Cargo.toml",
"noir/noir-repo/acvm-repo/acvm_js/Cargo.toml",
"avm-transpiler/Cargo.toml"
]
}
12 changes: 6 additions & 6 deletions avm-transpiler/src/transpile_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use log::info;
use regex::Regex;
use serde::{Deserialize, Serialize};

use acvm::acir::circuit::Circuit;
use acvm::acir::circuit::Program;

use crate::transpile::brillig_to_avm;
use crate::utils::extract_brillig_from_acir;
Expand Down Expand Up @@ -55,10 +55,10 @@ pub struct AcirContractFunction {
pub custom_attributes: Vec<String>,
pub abi: serde_json::Value,
#[serde(
serialize_with = "Circuit::serialize_circuit_base64",
deserialize_with = "Circuit::deserialize_circuit_base64"
serialize_with = "Program::serialize_program_base64",
deserialize_with = "Program::deserialize_program_base64"
)]
pub bytecode: Circuit,
pub bytecode: Program,
pub debug_symbols: serde_json::Value,
}

Expand Down Expand Up @@ -88,8 +88,8 @@ impl From<CompiledAcirContract> for TranspiledContract {
function.name, contract.name
);
// Extract Brillig Opcodes from acir
let acir_circuit = function.bytecode.clone();
let brillig = extract_brillig_from_acir(&acir_circuit.opcodes);
let acir_program = function.bytecode;
let brillig = extract_brillig_from_acir(&acir_program.functions[0].opcodes);

// Transpile to AVM
let avm_bytecode = brillig_to_avm(brillig);
Expand Down
7 changes: 7 additions & 0 deletions barretenberg/acir_tests/flows/write_contract.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu

export TEST_NAME=$(basename $(pwd))

$BIN write_vk -o vk
$BIN contract -k vk -c $CRS_PATH -b ./target/acir.gz -o $TEST_NAME.sol
34 changes: 34 additions & 0 deletions barretenberg/cpp/src/barretenberg/dsl/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Domain Specific Language

This package adds support to use [ACIR](https://github.com/noir-lang/noir/tree/master/acvm-repo/acir) with barretenberg.

## Serialization Changes

There are two types of breaking serialization changes. One that alters that alters the internal ACIR structure passed to barretenberg, and one that changes how we serialize the buffer passed to barretenberg.

1. Internal Structure Change

Go to the ACVM `acir` crate and re-run the [serde reflection test](../../../../../noir/noir-repo/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file.

You will have to update a couple things in the new `acir.hpp`:
- Replace all `throw serde::deserialization_error` with `throw_or_abort`
- The top-level struct (such as `Program`) will still use its own namespace for its internal fields. This extra `Program::` can be removed from the top-level `struct Program { .. }` object.

The same can then be done for any breaking changes introduced to `witness_stack.hpp`.

2. Full Breaking Change

This type of breaking change is rarely expected to happen, however, due to its nature there are multiple consumers of the pre-existing serialization you should be aware of if you ever need to make this change.

A full change is when you attempt to change the object whose buffer we are actually deserializing in barretenberg. To give more detail, when [deserializing the constraint buffer](./acir_format/acir_to_constraint_buf.hpp#366) if the object for which we call `bincodeDeserialize` on changes, anything that has pre-existing ACIR using the previous object's `bincodeDeserialize` method will now fail. The serialization is once again determined by the top-level object in the [acir crate](../../../../../noir/noir-repo/acvm-repo/acir/src/circuit/mod.rs). After performing the steps outlined for an internal structure breaking change as listed in (1) above, we need to update all consumers of barretenberg.

Even if you correctly update all serialization in [acvm_js](../../../../../noir/noir-repo/acvm-repo/acvm_js/README.md) such as during [execution](../../../../../noir/noir-repo/acvm-repo/acvm_js/src/execute.rs#57), there is multiple places the `yarn-project` uses the ACIR top-level serialization. The `yarn-project` sequencer also uses the native `acvm_cli tool` that has an execute method that [expects raw byte code](../../../../../noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs#63).

In the context of Aztec we need to regenerate all the artifacts in [noir-projects](../../../../../noir-projects/bootstrap.sh). This regeneration assumes that we have rebuilt the compilers (Noir compiler and AVM transpiler) to use the new serialization. After regenerating these artifacts we can bootstrap the yarn-project. There are multiple packages in the yarn-project that rely on pre-computed artifacts such as `yarn-project/circuits.js` and `yarn-project/protocol-contracts`.

The Aztec artifacts can be individually regenerated as well using `yarn test -u`.
You can also run the command on the relevant workspaces, which at the time are the following:
```
yarn workspace @aztec/circuits.js test -u
yarn workspace @aztec/protocol-contracts test -u
```
Loading

0 comments on commit ccc5016

Please sign in to comment.