Skip to content

Commit

Permalink
Merge branch 'ulan/run-721' into 'master'
Browse files Browse the repository at this point in the history 
RUN-721: Implement reservation mechanism in Wasm execution

The reservation mechanism is disabled by default, so this MR has no impact
on production.

This MR adds a new field in `SystemStateChanges` that keeps track of changes
in the `reserved_balance`. The `allocate_execution_memory()` function is
modified to update the new field based on the allocated bytes.

At the of execution, the replica code applies changes to the main and reserved
balances of the canister based on the newly added field in `SystemStateChanges`. 

See merge request dfinity-lab/public/ic!14324
  • Loading branch information
@ulan
ulan committed Sep 7, 2023
2 parents d81acd4 + 48bde3e commit c2adb4a
Show file tree
Hide file tree
Showing 3 changed files with 315 additions and 55 deletions.
163 changes: 163 additions & 0 deletions rs/execution_environment/src/hypervisor/tests.rs
View file
@@ -1,6 +1,7 @@
use assert_matches::assert_matches;
use candid::{Decode, Encode};
use ic_base_types::{NumSeconds, PrincipalId};
use ic_cycles_account_manager::ResourceSaturation;
use ic_error_types::{ErrorCode, RejectCode};
use ic_ic00_types::{CanisterChange, CanisterHttpResponsePayload, SkipPreUpgrade};
use ic_interfaces::execution_environment::{HypervisorError, SubnetAvailableMemory};
Expand Down Expand Up @@ -5978,6 +5979,168 @@ fn memory_grow_succeeds_in_post_upgrade_if_the_same_amount_is_dropped_afer_pre_u
test.upgrade_canister(canister_id, wasm).unwrap();
}

#[test]
fn stable_memory_grow_reserves_cycles() {
const CYCLES: Cycles = Cycles::new(20_000_000_000_000);
const CAPACITY: u64 = 1_000_000_000;
const THRESHOLD: u64 = 500_000_000;
const WASM_PAGE_SIZE: u64 = 65_536;
// 7500 of stable memory pages is close to 500MB, but still leaves some room
// for Wasm memory of the universal canister.
const NUM_PAGES: u64 = 7_500;

let mut test = ExecutionTestBuilder::new()
.with_subnet_execution_memory(CAPACITY as i64)
.with_subnet_memory_threshold(THRESHOLD as i64)
.with_subnet_memory_reservation(0)
.build();

let canister_id = test
.canister_from_cycles_and_binary(CYCLES, UNIVERSAL_CANISTER_WASM.into())
.unwrap();

test.update_freezing_threshold(canister_id, NumSeconds::new(0))
.unwrap();

let balance_before = test.canister_state(canister_id).system_state.balance();
let result = test
.ingress(
canister_id,
"update",
wasm()
.stable64_grow(NUM_PAGES)
// Access the last byte to make sure that growing succeeded.
.stable64_read(NUM_PAGES * WASM_PAGE_SIZE - 1, 1)
.push_bytes(&[])
.append_and_reply()
.build(),
)
.unwrap();
assert_eq!(result, WasmResult::Reply(vec![]));
let balance_after = test.canister_state(canister_id).system_state.balance();

assert_eq!(
test.canister_state(canister_id)
.system_state
.reserved_balance(),
Cycles::zero()
);
// Message execution fee is an order of a few million cycles.
assert!(balance_before - balance_after < Cycles::new(1_000_000_000));

let subnet_memory_usage =
CAPACITY - test.subnet_available_memory().get_execution_memory() as u64;
let memory_usage_before = test.canister_state(canister_id).execution_memory_usage();
let balance_before = test.canister_state(canister_id).system_state.balance();
let result = test
.ingress(
canister_id,
"update",
wasm()
.stable64_grow(NUM_PAGES)
// Access the last byte to make sure that growing succeeded.
.stable64_read(2 * NUM_PAGES * WASM_PAGE_SIZE - 1, 1)
.push_bytes(&[])
.append_and_reply()
.build(),
)
.unwrap();
assert_eq!(result, WasmResult::Reply(vec![]));
let balance_after = test.canister_state(canister_id).system_state.balance();
let memory_usage_after = test.canister_state(canister_id).execution_memory_usage();

let reserved_cycles = test
.canister_state(canister_id)
.system_state
.reserved_balance();

assert_eq!(
reserved_cycles,
test.cycles_account_manager().storage_reservation_cycles(
memory_usage_after - memory_usage_before,
&ResourceSaturation::new(subnet_memory_usage, THRESHOLD, CAPACITY),
test.subnet_size(),
)
);

assert!(balance_before - balance_after > reserved_cycles);
}

#[test]
fn wasm_memory_grow_reserves_cycles() {
const CYCLES: Cycles = Cycles::new(20_000_000_000_000);
const CAPACITY: u64 = 1_000_000_000;
const THRESHOLD: u64 = 500_000_000;

let mut test = ExecutionTestBuilder::new()
.with_subnet_execution_memory(CAPACITY as i64)
.with_subnet_memory_threshold(THRESHOLD as i64)
.with_subnet_memory_reservation(0)
.build();

let wat = r#"
(module
(import "ic0" "msg_reply" (func $msg_reply))
(import "ic0" "msg_reply_data_append"
(func $msg_reply_data_append (param i32 i32)))
(func $update
;; 7500 Wasm pages is close to 500MB.
(if (i32.eq (memory.grow (i32.const 7500)) (i32.const -1))
(then (unreachable))
)
(call $msg_reply)
)
(memory $memory 1)
(export "canister_update update" (func $update))
)"#;

let wasm = wat::parse_str(wat).unwrap();

let canister_id = test.canister_from_cycles_and_binary(CYCLES, wasm).unwrap();

test.update_freezing_threshold(canister_id, NumSeconds::new(0))
.unwrap();

let balance_before = test.canister_state(canister_id).system_state.balance();
let result = test.ingress(canister_id, "update", vec![]).unwrap();
assert_eq!(result, WasmResult::Reply(vec![]));
let balance_after = test.canister_state(canister_id).system_state.balance();

assert_eq!(
test.canister_state(canister_id)
.system_state
.reserved_balance(),
Cycles::zero()
);
// Message execution fee is an order of a few million cycles.
assert!(balance_before - balance_after < Cycles::new(1_000_000_000));

let subnet_memory_usage =
CAPACITY - test.subnet_available_memory().get_execution_memory() as u64;
let memory_usage_before = test.canister_state(canister_id).execution_memory_usage();
let balance_before = test.canister_state(canister_id).system_state.balance();
let result = test.ingress(canister_id, "update", vec![]).unwrap();
assert_eq!(result, WasmResult::Reply(vec![]));
let balance_after = test.canister_state(canister_id).system_state.balance();
let memory_usage_after = test.canister_state(canister_id).execution_memory_usage();

let reserved_cycles = test
.canister_state(canister_id)
.system_state
.reserved_balance();

assert_eq!(
reserved_cycles,
test.cycles_account_manager().storage_reservation_cycles(
memory_usage_after - memory_usage_before,
&ResourceSaturation::new(subnet_memory_usage, THRESHOLD, CAPACITY),
test.subnet_size(),
)
);

assert!(balance_before - balance_after > reserved_cycles);
}

#[test]
fn upgrade_with_skip_pre_upgrade_preserves_stable_memory() {
let mut test: ExecutionTest = ExecutionTestBuilder::new().build();
Expand Down
42 changes: 17 additions & 25 deletions rs/system_api/src/lib.rs
View file
Expand Up @@ -625,26 +625,6 @@ impl MemoryUsage {
}
}

/// Tries to allocate the requested number of Wasm pages.
///
/// Returns `Err(HypervisorError::OutOfMemory)` and leaves `self` unchanged
/// if either the canister memory limit or the subnet memory limit would be
/// exceeded.
///
/// Returns `Err(HypervisorError::InsufficientCyclesInMemoryGrow)` and
/// leaves `self` unchanged if freezing threshold check is needed for the
/// given API type and canister would be frozen after the allocation.
fn allocate_pages(
&mut self,
pages: usize,
api_type: &ApiType,
sandbox_safe_system_state: &SandboxSafeSystemState,
) -> HypervisorResult<()> {
let bytes = ic_replicated_state::num_bytes_try_from(NumWasmPages::from(pages))
.map_err(|_| HypervisorError::OutOfMemory)?;
self.allocate_execution_memory(bytes, api_type, sandbox_safe_system_state)
}

/// Tries to allocate the requested amount of the Wasm or stable memory.
///
/// If the canister has memory allocation, then this function doesn't allocate
Expand All @@ -661,7 +641,8 @@ impl MemoryUsage {
&mut self,
execution_bytes: NumBytes,
api_type: &ApiType,
sandbox_safe_system_state: &SandboxSafeSystemState,
sandbox_safe_system_state: &mut SandboxSafeSystemState,
subnet_memory_saturation: &ResourceSaturation,
) -> HypervisorResult<()> {
let (new_usage, overflow) = self
.current_usage
Expand All @@ -677,6 +658,12 @@ impl MemoryUsage {
NumBytes::new(new_usage),
)?;

sandbox_safe_system_state.reserve_storage_cycles(
execution_bytes,
&subnet_memory_saturation.add(self.allocated_execution_memory.get()),
api_type,
)?;

// The canister can increase its memory usage up to the reserved bytes without
// decrementing the subnet available memory because it was already decremented
// at the time of reservation.
Expand Down Expand Up @@ -2306,7 +2293,8 @@ impl SystemApi for SystemApiImpl {
match self.memory_usage.allocate_execution_memory(
bytes,
&self.api_type,
&self.sandbox_safe_system_state,
&mut self.sandbox_safe_system_state,
&self.execution_parameters.subnet_memory_saturation,
) {
Ok(()) => Ok(()),
Err(err @ HypervisorError::InsufficientCyclesInMemoryGrow { .. }) => {
Expand Down Expand Up @@ -2347,10 +2335,14 @@ impl SystemApi for SystemApiImpl {
if resulting_size > MAX_STABLE_MEMORY_IN_BYTES / WASM_PAGE_SIZE_IN_BYTES as u64 {
return Ok(StableGrowOutcome::Failure);
}
match self.memory_usage.allocate_pages(
additional_pages as usize,
match self.memory_usage.allocate_execution_memory(
// From the checks above we know that converting `additional_pages`
// to bytes will not overflow, so the `unwrap()` will succeed.
ic_replicated_state::num_bytes_try_from(NumWasmPages::new(additional_pages as usize))
.unwrap(),
&self.api_type,
&self.sandbox_safe_system_state,
&mut self.sandbox_safe_system_state,
&self.execution_parameters.subnet_memory_saturation,
) {
Ok(()) => Ok(StableGrowOutcome::Success),
Err(err @ HypervisorError::InsufficientCyclesInMemoryGrow { .. }) => {
Expand Down

0 comments on commit c2adb4a

Please sign in to comment.