-
Notifications
You must be signed in to change notification settings - Fork 295
/
execution_environment.rs
353 lines (287 loc) · 16.7 KB
/
execution_environment.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
use crate::embedders::Config as EmbeddersConfig;
use crate::flag_status::FlagStatus;
use ic_base_types::{CanisterId, NumSeconds};
use ic_types::{
Cycles, NumBytes, NumInstructions, MAX_STABLE_MEMORY_IN_BYTES, MAX_WASM_MEMORY_IN_BYTES,
};
use serde::{Deserialize, Serialize};
use std::{str::FromStr, time::Duration};
const MIB: u64 = 1024 * 1024;
const GIB: u64 = MIB * 1024;
/// This specifies the threshold in bytes at which the subnet memory usage is
/// considered to be high. If this value is greater or equal to the subnet
/// capacity, then the subnet is never considered to have high usage.
const SUBNET_MEMORY_THRESHOLD: NumBytes = NumBytes::new(450 * GIB);
/// This is the upper limit on how much logical storage canisters can request to
/// be store on a given subnet.
///
/// Logical storage is the amount of storage being used from the point of view
/// of the canister. The actual storage used by the nodes can be higher as the
/// IC protocol requires storing copies of the canister state.
const SUBNET_MEMORY_CAPACITY: NumBytes = NumBytes::new(700 * GIB);
/// This is the upper limit on how much memory can be used by all canister
/// messages on a given subnet.
///
/// Message memory usage is calculated as the total size of enqueued canister
/// responses; plus the maximum allowed response size per queue reservation.
const SUBNET_MESSAGE_MEMORY_CAPACITY: NumBytes = NumBytes::new(25 * GIB);
/// This is the upper limit on how much memory can be used by the ingress
/// history on a given subnet. It is lower than the subnet message memory
/// capacity because here we count actual memory consumption as opposed to
/// memory plus reservations.
const INGRESS_HISTORY_MEMORY_CAPACITY: NumBytes = NumBytes::new(4 * GIB);
/// This is the upper limit on how much memory can be used by wasm custom
/// sections on a given subnet.
const SUBNET_WASM_CUSTOM_SECTIONS_MEMORY_CAPACITY: NumBytes = NumBytes::new(2 * GIB);
/// The number of bytes reserved for response callback executions.
const SUBNET_MEMORY_RESERVATION: NumBytes = NumBytes::new(10 * GIB);
/// The duration a stop_canister has to stop the canister before timing out.
pub const STOP_CANISTER_TIMEOUT_DURATION: Duration = Duration::from_secs(5 * 60); // 5 minutes
/// This is the upper limit on how big heap deltas all the canisters together
/// can produce on a subnet in between checkpoints. Once, the total delta size
/// is above this limit, no more canisters will be executed till the next
/// checkpoint is taken. This is a soft limit in the sense that the actual delta
/// size can grow above this limit but no new execution will be done if the the
/// current size is above this limit.
///
/// Currently heap delta pages are stored in memory and not backed by a file.
/// The gen 1 machines in production have 500GiB of RAM available to replica.
/// Set the upper limit to 140GiB to reserve memory for other components and
/// potential fragmentation. This limit should be larger than the maximum
/// canister memory size to guarantee that a message that overwrites the whole
/// memory can succeed.
pub(crate) const SUBNET_HEAP_DELTA_CAPACITY: NumBytes = NumBytes::new(140 * GIB);
/// The maximum number of instructions for inspect_message calls.
const MAX_INSTRUCTIONS_FOR_MESSAGE_ACCEPTANCE_CALLS: NumInstructions =
NumInstructions::new(200_000_000);
/// The maximum depth of call graphs allowed for composite query calls
pub(crate) const MAX_QUERY_CALL_DEPTH: usize = 6;
/// Equivalent to MAX_INSTRUCTIONS_PER_MESSAGE_WITHOUT_DTS for now
pub(crate) const MAX_INSTRUCTIONS_PER_COMPOSITE_QUERY_CALL: u64 = 5_000_000_000;
/// The maximum time in seconds a query call is allowed to run.
pub(crate) const MAX_TIME_PER_COMPOSITE_QUERY_CALL: Duration = Duration::from_secs(10);
/// This would allow 100 calls with the current MAX_INSTRUCTIONS_PER_COMPOSITE_QUERY_CALL
pub const INSTRUCTION_OVERHEAD_PER_QUERY_CALL: u64 = 50_000_000;
/// The number of query execution threads overall for all canisters.
/// See also `QUERY_EXECUTION_THREADS_PER_CANISTER`.
pub(crate) const QUERY_EXECUTION_THREADS_TOTAL: usize = 4;
/// When a canister is scheduled for query execution, it is allowed to run for
/// this amount of time. This limit controls how many queries the canister
/// executes when it is scheduled. The limit does not control the duration of an
/// individual query. In the worst case, a single query may exceed this limit if
/// it executes the maximum number of instructions. In that case, the canister
/// would execute only that one query. Generally, we expect queries to finish in
/// a few milliseconds, so multiple queries will run in a slice.
///
/// The current value of 20ms is chosen arbitrarily. Any value between 10ms and
/// 100ms would work reasonably well. Reducing the value increases the overhead
/// of synchronization to fetch queries and also increases "unfairness" in the
/// presence of canisters that execute long-running queries. Increasing the
/// value increases the user-visible latency of the queries.
const QUERY_SCHEDULING_TIME_SLICE_PER_CANISTER: Duration = Duration::from_millis(20);
/// The upper limit on how much memory query cache can occupy.
///
/// The limit includes both cache keys and values, for successful query
/// executions and user errors.
const QUERY_CACHE_CAPACITY: NumBytes = NumBytes::new(200 * MIB);
/// The upper limit on how long the cache entry stays valid in the query cache.
const QUERY_CACHE_MAX_EXPIRY_TIME: Duration = Duration::from_secs(600);
/// The upper limit on how long the data certificate stays valid in the query cache.
///
/// The [HTTP Gateway Protocol Specification](https://internetcomputer.org/docs/current/references/http-gateway-protocol-spec#certificate-validation)
/// states that the certified timestamp must be recent, e.g. 5 minutes.
/// So queries using the `ic0.data_certificate_copy()` System API call
/// should not be cached for more than 5 minutes.
const QUERY_CACHE_DATA_CERTIFICATE_EXPIRY_TIME: Duration = Duration::from_secs(60);
/// Length of an epoch of query statistics in blocks
pub const QUERY_STATS_EPOCH_LENGTH: u64 = 600;
// The ID of the Bitcoin testnet canister.
pub const BITCOIN_TESTNET_CANISTER_ID: &str = "g4xu7-jiaaa-aaaan-aaaaq-cai";
// The ID of the Bitcoin mainnet canister.
pub const BITCOIN_MAINNET_CANISTER_ID: &str = "ghsi2-tqaaa-aaaan-aaaca-cai";
// The ID of the "soft launch" Bitcoin mainnet canister.
// This is a canister that will be used to run the bitcoin mainnet state pre-launch
// for final validation. Once the validation is complete, this canister will be uninstalled
// in favour of the "real" Bitcoin mainnet canister defined above.
// TODO(EXC-1298): Uninstall this canister once the bitcoin mainnet canister is live.
const BITCOIN_MAINNET_SOFT_LAUNCH_CANISTER_ID: &str = "gsvzx-syaaa-aaaan-aaabq-cai";
/// The capacity of the Wasm compilation cache.
pub const MAX_COMPILATION_CACHE_SIZE: NumBytes = NumBytes::new(10 * GIB);
/// Maximum number of controllers allowed in a request (specified in the interface spec).
pub const MAX_ALLOWED_CONTROLLERS_COUNT: usize = 10;
#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
#[serde(default)]
pub struct Config {
pub embedders_config: EmbeddersConfig,
/// This is no longer used in the code. It is not removed yet as removing
/// this option will be a breaking change.
pub create_funds_whitelist: String,
/// The maximum number of instructions that the methods that are invoked to
/// check message acceptance can run for.
pub max_instructions_for_message_acceptance_calls: NumInstructions,
/// This specifies the threshold in bytes at which the subnet memory usage is
/// considered to be high. If this value is greater or equal to the subnet
/// capacity, then the subnet is never considered to have high usage.
pub subnet_memory_threshold: NumBytes,
/// The maximum amount of logical storage available to all the canisters on
/// the subnet.
pub subnet_memory_capacity: NumBytes,
/// The maximum amount of logical storage available to canister messages
/// across the whole subnet.
pub subnet_message_memory_capacity: NumBytes,
/// The maximum amount of logical storage available to the ingress history
/// across the whole subnet.
pub ingress_history_memory_capacity: NumBytes,
/// The maximum amount of logical storage available to wasm custom sections
/// across the whole subnet.
pub subnet_wasm_custom_sections_memory_capacity: NumBytes,
/// The number of bytes reserved for response callback execution.
pub subnet_memory_reservation: NumBytes,
/// The maximum amount of memory that can be utilized by a single canister.
pub max_canister_memory_size: NumBytes,
/// The default value used when provisioning a canister
/// if amount of cycles was not specified.
pub default_provisional_cycles_balance: Cycles,
/// The default number of seconds after which a canister will freeze.
pub default_freeze_threshold: NumSeconds,
/// Maximum number of controllers a canister can have.
pub max_controllers: usize,
/// Indicates whether canisters sandboxing is enabled or not.
pub canister_sandboxing_flag: FlagStatus,
/// The number of threads to use for query execution overall.
pub query_execution_threads_total: usize,
/// When a canister is scheduled for query execution, it is allowed to run for
/// this amount of time.
pub query_scheduling_time_slice_per_canister: Duration,
/// The maximum depth of a query call graph.
pub max_query_call_graph_depth: usize,
/// The maximum number of instructions allowed for a query call graph.
pub max_query_call_graph_instructions: NumInstructions,
/// The maximum time a query call in non-replicated mode is allowed to run.
/// In replicated code we cannot rely on the walltime, since that is not
/// deterministic.
pub max_query_call_walltime: Duration,
/// Instructions to charge for each composite query call in addition to the
/// instructions in the actual query call. This is meant to protect from
/// cases where we have many calls into canister that execute little work.
/// This cost is meant to cover for some of the overhead associated with
/// the actual call.
pub instruction_overhead_per_query_call: NumInstructions,
/// If this flag is enabled, then message execution of canisters will be
/// rate limited based on the amount of modified memory.
pub rate_limiting_of_heap_delta: FlagStatus,
/// If this flag is enabled, then message execution of canisters will be
/// rate limited based on the number of executed instructions per round.
pub rate_limiting_of_instructions: FlagStatus,
/// Specifies the percentage of subnet compute capacity that is allocatable
/// by canisters.
pub allocatable_compute_capacity_in_percent: usize,
/// Indicates whether deterministic time slicing is enabled or not.
pub deterministic_time_slicing: FlagStatus,
/// Bitcoin configuration.
pub bitcoin: BitcoinConfig,
/// Indicates whether composite queries are available or not.
pub composite_queries: FlagStatus,
/// Indicates whether replica side query caching is enabled.
pub query_caching: FlagStatus,
/// Query cache capacity in bytes
pub query_cache_capacity: NumBytes,
/// The upper limit on how long the cache entry stays valid in the query cache.
pub query_cache_max_expiry_time: Duration,
/// The upper limit on how long the data certificate stays valid in the query cache.
pub query_cache_data_certificate_expiry_time: Duration,
/// The capacity of the Wasm compilation cache.
pub max_compilation_cache_size: NumBytes,
/// Indicate whether query stats should be collected or not.
pub query_stats_aggregation: FlagStatus,
/// Length of an epoch for query stats collection.
pub query_stats_epoch_length: u64,
/// Indicates whether the Wasm chunk store feature has been enabled or not.
pub wasm_chunk_store: FlagStatus,
/// The duration a stop_canister has to stop the canister before timing out.
pub stop_canister_timeout_duration: Duration,
/// Indicates whether canister backup and restore feature is enabled or not.
pub canister_snapshots: FlagStatus,
/// Indicates whether dirty page logging is enabled or not.
pub dirty_page_logging: FlagStatus,
}
impl Default for Config {
fn default() -> Self {
let bitcoin_testnet_canister_id = CanisterId::from_str(BITCOIN_TESTNET_CANISTER_ID)
.expect("bitcoin testnet canister id must be a valid principal");
let bitcoin_mainnet_canister_id = CanisterId::from_str(BITCOIN_MAINNET_CANISTER_ID)
.expect("bitcoin mainnet canister id must be a valid principal");
let bitcoin_mainnet_soft_launch_canister_id =
CanisterId::from_str(BITCOIN_MAINNET_SOFT_LAUNCH_CANISTER_ID)
.expect("bitcoin mainnet soft-launch canister id must be a valid principal");
Self {
embedders_config: EmbeddersConfig::default(),
create_funds_whitelist: String::default(),
max_instructions_for_message_acceptance_calls:
MAX_INSTRUCTIONS_FOR_MESSAGE_ACCEPTANCE_CALLS,
subnet_memory_threshold: SUBNET_MEMORY_THRESHOLD,
subnet_memory_capacity: SUBNET_MEMORY_CAPACITY,
subnet_message_memory_capacity: SUBNET_MESSAGE_MEMORY_CAPACITY,
ingress_history_memory_capacity: INGRESS_HISTORY_MEMORY_CAPACITY,
subnet_wasm_custom_sections_memory_capacity:
SUBNET_WASM_CUSTOM_SECTIONS_MEMORY_CAPACITY,
subnet_memory_reservation: SUBNET_MEMORY_RESERVATION,
max_canister_memory_size: NumBytes::new(
MAX_STABLE_MEMORY_IN_BYTES + MAX_WASM_MEMORY_IN_BYTES,
),
default_provisional_cycles_balance: Cycles::new(100_000_000_000_000),
// The default freeze threshold is 30 days.
default_freeze_threshold: NumSeconds::from(30 * 24 * 60 * 60),
max_controllers: MAX_ALLOWED_CONTROLLERS_COUNT,
canister_sandboxing_flag: FlagStatus::Enabled,
query_execution_threads_total: QUERY_EXECUTION_THREADS_TOTAL,
query_scheduling_time_slice_per_canister: QUERY_SCHEDULING_TIME_SLICE_PER_CANISTER,
max_query_call_graph_depth: MAX_QUERY_CALL_DEPTH,
max_query_call_graph_instructions: NumInstructions::from(
MAX_INSTRUCTIONS_PER_COMPOSITE_QUERY_CALL,
),
max_query_call_walltime: MAX_TIME_PER_COMPOSITE_QUERY_CALL,
instruction_overhead_per_query_call: NumInstructions::from(
INSTRUCTION_OVERHEAD_PER_QUERY_CALL,
),
rate_limiting_of_heap_delta: FlagStatus::Enabled,
rate_limiting_of_instructions: FlagStatus::Enabled,
// The allocatable compute capacity is capped at 50% to ensure that
// best-effort canisters have sufficient compute to make progress.
allocatable_compute_capacity_in_percent: 50,
deterministic_time_slicing: FlagStatus::Enabled,
bitcoin: BitcoinConfig {
privileged_access: vec![
bitcoin_testnet_canister_id,
bitcoin_mainnet_canister_id,
bitcoin_mainnet_soft_launch_canister_id,
],
testnet_canister_id: Some(bitcoin_testnet_canister_id),
mainnet_canister_id: Some(bitcoin_mainnet_canister_id),
},
composite_queries: FlagStatus::Enabled,
query_caching: FlagStatus::Enabled,
query_cache_capacity: QUERY_CACHE_CAPACITY,
query_cache_max_expiry_time: QUERY_CACHE_MAX_EXPIRY_TIME,
query_cache_data_certificate_expiry_time: QUERY_CACHE_DATA_CERTIFICATE_EXPIRY_TIME,
max_compilation_cache_size: MAX_COMPILATION_CACHE_SIZE,
query_stats_aggregation: FlagStatus::Enabled,
query_stats_epoch_length: QUERY_STATS_EPOCH_LENGTH,
wasm_chunk_store: FlagStatus::Enabled,
stop_canister_timeout_duration: STOP_CANISTER_TIMEOUT_DURATION,
canister_snapshots: FlagStatus::Disabled,
dirty_page_logging: FlagStatus::Disabled,
}
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize, Default)]
pub struct BitcoinConfig {
/// Canisters that have access to privileged bitcoin API (e.g. `bitcoin_get_successors`)
/// This list is intentionally separate from the bitcoin canister IDs below because it
/// allows us to spin up new bitcoin canisters without necessarily routing requests to them.
pub privileged_access: Vec<CanisterId>,
/// The bitcoin testnet canister to forward requests to.
pub testnet_canister_id: Option<CanisterId>,
/// The bitcoin mainnet canister to forward requests to.
pub mainnet_canister_id: Option<CanisterId>,
}