From 29559bd3ef28d7f208ebd7052fd85a8a4cd23436 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 16 May 2024 19:24:01 +0100 Subject: [PATCH] feat(avm): gzip avm bytecode (#6475) We can use this to reduce bytecode size (just as Brillig does) until we do bytecode validation. ``` AvmAcvmInteropTest::constant_field_avm: compressed 139 to 95 bytes AvmAcvmInteropTest::call_acvm_from_avm: compressed 6343 to 1486 bytes AvmAcvmInteropTest::avm_to_acvm_call: compressed 2145 to 552 bytes AvmAcvmInteropTest::new_nullifier: compressed 91 to 74 bytes AvmAcvmInteropTest::test_authwit_send_money: compressed 9090 to 1977 bytes AvmInitializerTest::read_storage_immutable: compressed 2078 to 560 bytes AvmInitializerTest::constructor: compressed 73769 to 6638 bytes AvmNestedCallsTest::set_storage_single: compressed 719 to 201 bytes AvmNestedCallsTest::nested_call_to_add_with_gas: compressed 48796 to 3976 bytes AvmNestedCallsTest::assert_same: compressed 1758 to 452 bytes AvmNestedCallsTest::new_nullifier: compressed 91 to 74 bytes AvmNestedCallsTest::nested_static_call_to_add: compressed 48643 to 3871 bytes AvmNestedCallsTest::nested_static_call_to_set_storage: compressed 47167 to 3573 bytes AvmNestedCallsTest::add_args_return: compressed 140 to 92 bytes AvmNestedCallsTest::nested_call_to_add: compressed 48258 to 3843 bytes AvmNestedCallsTest::create_same_nullifier_in_nested_call: compressed 46760 to 3527 bytes AvmNestedCallsTest::create_different_nullifier_in_nested_call: compressed 46809 to 3540 bytes AvmTest::get_chain_id: compressed 111 to 81 bytes AvmTest::to_radix_le: compressed 41037 to 2164 bytes AvmTest::read_storage_list: compressed 1186 to 345 bytes AvmTest::set_storage_map: compressed 5335 to 930 bytes AvmTest::new_nullifier: compressed 91 to 74 bytes AvmTest::set_opcode_u32: compressed 116 to 83 bytes AvmTest::assertion_failure: compressed 2107 to 495 bytes AvmTest::set_opcode_small_field: compressed 139 to 106 bytes AvmTest::read_storage_map: compressed 4987 to 890 bytes AvmTest::get_address: compressed 111 to 81 bytes AvmTest::pedersen_hash_with_index: compressed 199 to 114 bytes AvmTest::get_sender: compressed 111 to 81 bytes AvmTest::add_storage_map: compressed 10417 to 1742 bytes AvmTest::set_storage_list: compressed 864 to 219 bytes AvmTest::get_fee_per_da_gas: compressed 111 to 81 bytes AvmTest::u128_addition_overflow: compressed 46973 to 2808 bytes AvmTest::read_storage_single: compressed 728 to 227 bytes AvmTest::test_get_contract_instance_raw: compressed 1576 to 442 bytes AvmTest::test_get_contract_instance: compressed 44797 to 2953 bytes AvmTest::get_version: compressed 111 to 80 bytes AvmTest::check_selector: compressed 44567 to 3039 bytes AvmTest::get_transaction_fee: compressed 111 to 81 bytes AvmTest::add_u128: compressed 2390 to 593 bytes AvmTest::nullifier_collision: compressed 97 to 76 bytes AvmTest::pedersen_hash: compressed 199 to 113 bytes AvmTest::modulo2: compressed 177 to 104 bytes AvmTest::debug_logging: compressed 8916 to 1718 bytes AvmTest::assert_nullifier_exists: compressed 1964 to 502 bytes AvmTest::l1_to_l2_msg_exists: compressed 162 to 101 bytes AvmTest::note_hash_exists: compressed 162 to 102 bytes AvmTest::emit_unencrypted_log: compressed 47663 to 3297 bytes AvmTest::send_l2_to_l1_msg: compressed 105 to 79 bytes AvmTest::get_args_hash: compressed 176 to 105 bytes AvmTest::get_block_number: compressed 111 to 80 bytes AvmTest::set_read_storage_single: compressed 1372 to 350 bytes AvmTest::get_fee_per_l2_gas: compressed 111 to 81 bytes AvmTest::keccak_hash: compressed 559 to 220 bytes AvmTest::sha256_hash: compressed 537 to 207 bytes AvmTest::nullifier_exists: compressed 158 to 99 bytes AvmTest::new_note_hash: compressed 91 to 74 bytes AvmTest::set_opcode_u8: compressed 113 to 82 bytes AvmTest::add_args_return: compressed 140 to 92 bytes AvmTest::get_storage_address: compressed 111 to 81 bytes AvmTest::emit_nullifier_and_check: compressed 4100 to 682 bytes AvmTest::u128_from_integer_overflow: compressed 43568 to 2048 bytes AvmTest::poseidon2_hash: compressed 55082 to 4986 bytes AvmTest::set_opcode_u64: compressed 120 to 86 bytes AvmTest::get_timestamp: compressed 111 to 80 bytes AvmTest::set_storage_single: compressed 719 to 201 bytes AvmTest::set_opcode_big_field: compressed 263 to 137 bytes ``` --------- Co-authored-by: dbanks12 --- avm-transpiler/Cargo.lock | 1 + avm-transpiler/Cargo.toml | 3 ++- avm-transpiler/src/transpile_contract.rs | 20 +++++++++++++++++-- .../noir-contracts/scripts/transpile.sh | 3 +-- .../simulator/src/avm/avm_simulator.test.ts | 14 ++++++------- .../simulator/src/avm/avm_simulator.ts | 8 +++++--- yarn-project/simulator/src/public/executor.ts | 15 +++++++++++--- .../src/public/transitional_adaptors.ts | 19 ++++++++++++++++-- 8 files changed, 63 insertions(+), 20 deletions(-) diff --git a/avm-transpiler/Cargo.lock b/avm-transpiler/Cargo.lock index 14beeb02d67..af42b7a046a 100644 --- a/avm-transpiler/Cargo.lock +++ b/avm-transpiler/Cargo.lock @@ -311,6 +311,7 @@ dependencies = [ "acvm", "base64 0.21.7", "env_logger", + "flate2", "log", "noirc_driver", "noirc_errors", diff --git a/avm-transpiler/Cargo.toml b/avm-transpiler/Cargo.toml index 0e868d476bb..275af5ad673 100644 --- a/avm-transpiler/Cargo.toml +++ b/avm-transpiler/Cargo.toml @@ -19,4 +19,5 @@ regex = "1.10" env_logger = "0.11" log = "0.4" serde_json = "1.0" -serde = { version = "1.0.136", features = ["derive"]} +serde = { version = "1.0.136", features = ["derive"] } +flate2 = "1.0" diff --git a/avm-transpiler/src/transpile_contract.rs b/avm-transpiler/src/transpile_contract.rs index 1b2987403b0..1b449495baa 100644 --- a/avm-transpiler/src/transpile_contract.rs +++ b/avm-transpiler/src/transpile_contract.rs @@ -1,9 +1,10 @@ +use std::io::Read; + use base64::Engine; use log::info; use serde::{Deserialize, Serialize}; use acvm::acir::circuit::Program; -use noirc_errors::debug_info::DebugInfo; use noirc_errors::debug_info::ProgramDebugInfo; use crate::transpile::{brillig_to_avm, map_brillig_pcs_to_avm_pcs, patch_debug_info_pcs}; @@ -104,6 +105,21 @@ impl From for TranspiledContractArtifact { // Transpile to AVM let avm_bytecode = brillig_to_avm(brillig_bytecode, &brillig_pcs_to_avm_pcs); + // Gzip AVM bytecode. This has to be removed once we need to do bytecode verification. + let mut compressed_avm_bytecode = Vec::new(); + let mut encoder = + flate2::read::GzEncoder::new(&avm_bytecode[..], flate2::Compression::best()); + let _ = encoder.read_to_end(&mut compressed_avm_bytecode); + + log::info!( + "{}::{}: compressed {} to {} bytes ({}% reduction)", + contract.name, + function.name, + avm_bytecode.len(), + compressed_avm_bytecode.len(), + 100 - (compressed_avm_bytecode.len() * 100 / avm_bytecode.len()) + ); + // Patch the debug infos with updated PCs let debug_infos = patch_debug_info_pcs( &function.debug_symbols.debug_infos, @@ -117,7 +133,7 @@ impl From for TranspiledContractArtifact { is_unconstrained: function.is_unconstrained, custom_attributes: function.custom_attributes, abi: function.abi, - bytecode: base64::prelude::BASE64_STANDARD.encode(avm_bytecode), + bytecode: base64::prelude::BASE64_STANDARD.encode(compressed_avm_bytecode), debug_symbols: ProgramDebugInfo { debug_infos }, }, )); diff --git a/noir-projects/noir-contracts/scripts/transpile.sh b/noir-projects/noir-contracts/scripts/transpile.sh index 934f8982d55..528c95dd634 100755 --- a/noir-projects/noir-contracts/scripts/transpile.sh +++ b/noir-projects/noir-contracts/scripts/transpile.sh @@ -2,5 +2,4 @@ set -eu TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpiler} -ls target/*.json | parallel "$TRANSPILER {} {}" - +ls target/*.json | parallel "$TRANSPILER {} {}" \ No newline at end of file diff --git a/yarn-project/simulator/src/avm/avm_simulator.test.ts b/yarn-project/simulator/src/avm/avm_simulator.test.ts index 4190e3eaeae..c692284d1c4 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.test.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.test.ts @@ -10,7 +10,7 @@ import { AvmNestedCallsTestContractArtifact, AvmTestContractArtifact } from '@az import { jest } from '@jest/globals'; import { strict as assert } from 'assert'; -import { isAvmBytecode } from '../public/transitional_adaptors.js'; +import { isAvmBytecode, markBytecodeAsAvm } from '../public/transitional_adaptors.js'; import { AvmMachineState } from './avm_machine_state.js'; import { type MemoryValue, TypeTag, type Uint8 } from './avm_memory_types.js'; import { AvmSimulator } from './avm_simulator.js'; @@ -39,14 +39,14 @@ describe('AVM simulator: injected bytecode', () => { ]); }); - it('Should not be recognized as AVM bytecode (magic missing)', () => { - expect(!isAvmBytecode(bytecode)); + it('Should not be recognized as AVM bytecode (magic missing)', async () => { + expect(!(await isAvmBytecode(bytecode))); }); it('Should execute bytecode that performs basic addition', async () => { const context = initContext({ env: initExecutionEnvironment({ calldata }) }); const { l2GasLeft: initialL2GasLeft } = AvmMachineState.fromState(context.machineState); - const results = await new AvmSimulator(context).executeBytecode(bytecode); + const results = await new AvmSimulator(context).executeBytecode(markBytecodeAsAvm(bytecode)); expect(results.reverted).toBe(false); expect(results.output).toEqual([new Fr(3)]); @@ -59,7 +59,7 @@ describe('AVM simulator: injected bytecode', () => { machineState: initMachineState({ l2GasLeft: 5 }), }); - const results = await new AvmSimulator(context).executeBytecode(bytecode); + const results = await new AvmSimulator(context).executeBytecode(markBytecodeAsAvm(bytecode)); expect(results.reverted).toBe(true); expect(results.output).toEqual([]); expect(results.revertReason?.message).toEqual('Not enough L2GAS gas left'); @@ -91,9 +91,9 @@ describe('AVM simulator: transpiled Noir contracts', () => { expect(results.output).toEqual([new Fr(0)]); }); - it('Should be recognized as AVM bytecode (magic present)', () => { + it('Should be recognized as AVM bytecode (magic present)', async () => { const bytecode = getAvmTestContractBytecode('add_args_return'); - expect(isAvmBytecode(bytecode)); + expect(await isAvmBytecode(bytecode)); }); describe('U128 addition and overflows', () => { diff --git a/yarn-project/simulator/src/avm/avm_simulator.ts b/yarn-project/simulator/src/avm/avm_simulator.ts index 428dcb624d2..65ea246b10e 100644 --- a/yarn-project/simulator/src/avm/avm_simulator.ts +++ b/yarn-project/simulator/src/avm/avm_simulator.ts @@ -2,7 +2,7 @@ import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { strict as assert } from 'assert'; -import { isAvmBytecode } from '../public/transitional_adaptors.js'; +import { decompressBytecodeIfCompressed, isAvmBytecode } from '../public/transitional_adaptors.js'; import type { AvmContext } from './avm_context.js'; import { AvmContractCallResults } from './avm_message_call_result.js'; import { @@ -39,7 +39,6 @@ export class AvmSimulator { if (!bytecode) { throw new NoBytecodeForContractError(this.context.environment.address); } - assert(isAvmBytecode(bytecode), "AVM simulator can't execute non-AVM bytecode"); return await this.executeBytecode(bytecode); } @@ -49,7 +48,10 @@ export class AvmSimulator { * This method is useful for testing and debugging. */ public async executeBytecode(bytecode: Buffer): Promise { - return await this.executeInstructions(decodeFromBytecode(bytecode)); + const decompressedBytecode = await decompressBytecodeIfCompressed(bytecode); + assert(isAvmBytecode(decompressedBytecode), "AVM simulator can't execute non-AVM bytecode"); + + return await this.executeInstructions(decodeFromBytecode(decompressedBytecode)); } /** diff --git a/yarn-project/simulator/src/public/executor.ts b/yarn-project/simulator/src/public/executor.ts index c8ee7056014..9157508a9fb 100644 --- a/yarn-project/simulator/src/public/executor.ts +++ b/yarn-project/simulator/src/public/executor.ts @@ -11,6 +11,7 @@ import { import { createDebugLogger } from '@aztec/foundation/log'; import { spawn } from 'child_process'; +import { assert } from 'console'; import fs from 'fs/promises'; import path from 'path'; @@ -27,7 +28,12 @@ import { PackedValuesCache } from '../common/packed_values_cache.js'; import { type CommitmentsDB, type PublicContractsDB, type PublicStateDB } from './db.js'; import { type PublicExecution, type PublicExecutionResult, checkValidStaticCall } from './execution.js'; import { PublicExecutionContext } from './public_execution_context.js'; -import { convertAvmResultsToPxResult, createAvmExecutionEnvironment, isAvmBytecode } from './transitional_adaptors.js'; +import { + convertAvmResultsToPxResult, + createAvmExecutionEnvironment, + decompressBytecodeIfCompressed, + isAvmBytecode, +} from './transitional_adaptors.js'; /** * Execute a public function and return the execution result. @@ -46,7 +52,7 @@ export async function executePublicFunction( ); } - if (isAvmBytecode(bytecode)) { + if (await isAvmBytecode(bytecode)) { return await executeTopLevelPublicFunctionAvm(context, bytecode); } else { return await executePublicFunctionAcvm(context, bytecode, nested); @@ -355,7 +361,10 @@ export class PublicExecutor { const proofPath = path.join(artifactsPath, 'proof'); const { args, functionData, contractAddress } = avmExecution; - const bytecode = await this.contractsDb.getBytecode(contractAddress, functionData.selector); + let bytecode = await this.contractsDb.getBytecode(contractAddress, functionData.selector); + assert(!!bytecode, `Bytecode not found for ${contractAddress}:${functionData.selector}`); + // This should be removed once we do bytecode validation. + bytecode = await decompressBytecodeIfCompressed(bytecode!); // Write call data and bytecode to files. await fs.writeFile( calldataPath, diff --git a/yarn-project/simulator/src/public/transitional_adaptors.ts b/yarn-project/simulator/src/public/transitional_adaptors.ts index 161b10091c8..74e6d004788 100644 --- a/yarn-project/simulator/src/public/transitional_adaptors.ts +++ b/yarn-project/simulator/src/public/transitional_adaptors.ts @@ -10,6 +10,9 @@ import { } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; +import { promisify } from 'util'; +import { gunzip } from 'zlib'; + import { type AvmContext } from '../avm/avm_context.js'; import { AvmExecutionEnvironment } from '../avm/avm_execution_environment.js'; import { type AvmContractCallResults } from '../avm/avm_message_call_result.js'; @@ -111,7 +114,19 @@ export function markBytecodeAsAvm(bytecode: Buffer): Buffer { return Buffer.concat([bytecode, AVM_MAGIC_SUFFIX]); } -export function isAvmBytecode(bytecode: Buffer): boolean { +// This is just a helper function for the AVM circuit. +export async function decompressBytecodeIfCompressed(bytecode: Buffer): Promise { + try { + return await promisify(gunzip)(bytecode); + } catch { + // If the bytecode is not compressed, the gunzip call will throw an error + // In this case, we assume the bytecode is not compressed and continue. + return Promise.resolve(bytecode); + } +} + +export async function isAvmBytecode(bytecode: Buffer): Promise { + const decompressedBytecode = await decompressBytecodeIfCompressed(bytecode); const magicSize = AVM_MAGIC_SUFFIX.length; - return bytecode.subarray(-magicSize).equals(AVM_MAGIC_SUFFIX); + return decompressedBytecode.subarray(-magicSize).equals(AVM_MAGIC_SUFFIX); }