feat(fuzzing): Fuzzer for ic_http_endpoints CallService

Author: eduarddfinity

.gitlab/CODEOWNERS

@@ -124,6 +124,7 @@ go_deps.bzl               @dfinity-lab/teams/idx
 /rs/gitlab-ci-config.yml                                @dfinity-lab/teams/idx
 /rs/ic_os/                                              @dfinity-lab/teams/node-team
 /rs/http_endpoints/                                     @dfinity-lab/teams/networking-team
+/rs/http_endpoints/fuzz/                                @dfinity-lab/teams/prodsec
 /rs/http_utils/                                         @dfinity-lab/teams/consensus-owners
 /rs/https_outcalls/                                     @dfinity-lab/teams/networking-team
 /rs/https_outcalls/consensus/                           @dfinity-lab/teams/consensus-owners

Cargo.lock

@@ -0,0 +1,58 @@
+load("//bazel:fuzz_testing.bzl", "rust_fuzz_test_binary")
+
+package(default_visibility = ["//visibility:private"])
+
+MACRO_DEPENDENCIES = []
+
+CALLSERVICE_FUZZER_DEPENDENCIES = [
+    "//rs/config",
+    "//rs/http_endpoints/public",
+    "//rs/interfaces",
+    "//rs/interfaces/registry",
+    "//rs/interfaces/registry/mocks",
+    "//rs/monitoring/logger",
+    "//rs/monitoring/metrics",
+    "//rs/protobuf",
+    "//rs/registry/keys",
+    "//rs/registry/provisional_whitelist",
+    "//rs/test_utilities",
+    "//rs/types/error_types",
+    "//rs/types/types",
+    "@crate_index//:arbitrary",
+    "@crate_index//:crossbeam",
+    "@crate_index//:bytes",
+    "@crate_index//:hyper",
+    "@crate_index//:libfuzzer-sys",
+    "@crate_index//:mockall",
+    "@crate_index//:prost",
+    "@crate_index//:tokio",
+    "@crate_index//:tower",
+    "@crate_index//:tower-test",
+]
+
+# required to compile tests/common
+DEV_DEPENDENCIES = [
+    "//rs/crypto/tree_hash",
+    "//rs/interfaces/state_manager",
+    "//rs/registry/subnet_type",
+    "//rs/replicated_state",
+    "//rs/monitoring/pprof",
+    "//rs/certification/test-utils",
+    "//rs/crypto/tls_interfaces/mocks",
+    "//rs/interfaces/mocks",
+    "//rs/interfaces/state_manager/mocks",
+    "//rs/registry/routing_table",
+    "@crate_index//:ic-agent",
+]
+
+rust_fuzz_test_binary(
+    name = "execute_call_service_libfuzzer",
+    testonly = True,
+    srcs = [
+        "fuzz_targets/execute_call_service.rs",
+        "//rs/http_endpoints/public:tests/common/mod.rs",
+    ],
+    crate_root = "fuzz_targets/execute_call_service.rs",
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    deps = CALLSERVICE_FUZZER_DEPENDENCIES + DEV_DEPENDENCIES,
+)

rs/http_endpoints/fuzz/BUILD.bazel

@@ -0,0 +1,143 @@
+#![no_main]
+use bytes::Bytes;
+use hyper::{Body, Method, Request, Response};
+use ic_config::http_handler::Config;
+use ic_error_types::UserError;
+use ic_http_endpoints_public::call::CallService;
+use ic_http_endpoints_public::metrics::HttpHandlerMetrics;
+use ic_http_endpoints_public::validator_executor::ValidatorExecutor;
+use ic_interfaces::{
+    execution_environment::QueryExecutionResponse, ingress_pool::IngressPoolThrottler,
+};
+use ic_interfaces_registry::RegistryClient;
+use ic_logger::replica_logger::no_op_logger;
+use ic_metrics::MetricsRegistry;
+use ic_registry_provisional_whitelist::ProvisionalWhitelist;
+use ic_test_utilities::{
+    crypto::temp_crypto_component_with_fake_registry,
+    types::ids::{node_test_id, subnet_test_id},
+};
+use ic_types::{
+    malicious_flags::MaliciousFlags,
+    messages::{CertificateDelegation, SignedIngressContent, UserQuery},
+    PrincipalId,
+};
+use libfuzzer_sys::fuzz_target;
+use mockall::mock;
+use std::{
+    convert::Infallible,
+    net::SocketAddr,
+    str::FromStr,
+    sync::{Arc, RwLock},
+};
+use tokio::runtime::Runtime;
+use tower::{util::BoxCloneService, Service, ServiceExt};
+use tower_test::mock::Handle;
+
+#[path = "../../public/tests/common/mod.rs"]
+pub mod common;
+use common::{basic_registry_client, get_free_localhost_socket_addr, setup_ingress_filter_mock};
+
+pub type IngressFilterHandle =
+    Handle<(ProvisionalWhitelist, SignedIngressContent), Result<(), UserError>>;
+pub type QueryExecutionHandle =
+    Handle<(UserQuery, Option<CertificateDelegation>), QueryExecutionResponse>;
+
+mock! {
+    pub IngressPoolThrottler {}
+
+    impl IngressPoolThrottler for IngressPoolThrottler {
+        fn exceeds_threshold(&self) -> bool;
+    }
+}
+
+// This fuzzer attempts to execute the CallService call method. The input to the call method
+// is an HTTP request. Currently only the HTTP request body is fuzzed. The HTTP requests
+// headers are fixed.
+//
+// The fuzz test is only compiled but not executed by CI.
+//
+// To execute the fuzzer run
+// bazel run --config=fuzzing //rs/http_endpoints/fuzz:execute_call_service_libfuzzer -- corpus/
+//
+// TODO (PSEC-1654): Implement Arbitrary for the request body. Details:
+// This initial version of the fuzzer is currently likely ineffective. This is because as soon as the data
+// can't be CBOR decoded, is incorrectly signed, or contains a mismatching effective canister id, `call`
+// will fail, and such a failure will happen for most mutations of `data`.
+// To address this, the next MR (PSEC-1654) will implement Arbitrary so that mutations of the data more
+// effectively explore interesting inputs.
+fuzz_target!(|data: &[u8]| {
+    let rt = Runtime::new().unwrap();
+    let addr = get_free_localhost_socket_addr();
+    let effective_canister_id = "223xb-saaaa-aaaaf-arlqa-cai";
+
+    let mut call_service = new_call_service(addr);
+    let mut req = Request::builder()
+        .method(Method::POST)
+        .uri(format!(
+            "http://{}/api/v2/canister/{}/call",
+            addr, effective_canister_id,
+        ))
+        .header("Content-Type", "application/cbor")
+        .body(Bytes::from(data.to_vec()))
+        .expect("Failed to build the request");
+
+    // The effective_canister_id is added to the request during routing
+    // and then removed from the request parts (see `remove_effective_principal_id`
+    // in http_endponts/public/src/common.rs).
+    // We simulate that behaviour in this line.
+    req.extensions_mut()
+        .insert(PrincipalId::from_str(effective_canister_id).unwrap());
+
+    rt.block_on(async move {
+        call_service
+            .ready()
+            .await
+            .expect("could not create call service")
+            .call(req)
+            .await
+            .unwrap()
+    });
+});
+
+fn new_call_service(
+    addr: SocketAddr,
+) -> BoxCloneService<Request<Bytes>, Response<Body>, Infallible> {
+    let config = Config {
+        listen_addr: addr,
+        ..Default::default()
+    };
+    let log = no_op_logger();
+    let metrics_registry = MetricsRegistry::new();
+    let mock_registry_client: Arc<dyn RegistryClient> = Arc::new(basic_registry_client());
+
+    let (ingress_filter, _ingress_filter_handle) = setup_ingress_filter_mock();
+    let mut ingress_pool_throttler = MockIngressPoolThrottler::new();
+    ingress_pool_throttler
+        .expect_exceeds_threshold()
+        .returning(|| false);
+
+    let ingress_throttler = Arc::new(RwLock::new(ingress_pool_throttler));
+
+    let (ingress_tx, _ingress_rx) = crossbeam::channel::unbounded();
+
+    let sig_verifier = Arc::new(temp_crypto_component_with_fake_registry(node_test_id(1)));
+
+    CallService::new_service(
+        config,
+        log.clone(),
+        HttpHandlerMetrics::new(&metrics_registry),
+        node_test_id(1),
+        subnet_test_id(1),
+        Arc::clone(&mock_registry_client),
+        ValidatorExecutor::new(
+            Arc::clone(&mock_registry_client),
+            sig_verifier,
+            &MaliciousFlags::default(),
+            log,
+        ),
+        ingress_filter,
+        ingress_throttler,
+        ingress_tx,
+    )
+}

rs/http_endpoints/fuzz/fuzz_targets/execute_call_service.rs

@@ -1,8 +1,12 @@
 load("@rules_rust//cargo:cargo_build_script.bzl", "cargo_build_script")
 load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
 load("//bazel:defs.bzl", "rust_test_suite_with_extra_srcs")
+load("//bazel:fuzz_testing.bzl", "DEFAULT_RUSTC_FLAGS_FOR_FUZZING")
 
-package(default_visibility = ["//rs/replica:__subpackages__"])
+package(default_visibility = [
+    "//rs/http_endpoints:__subpackages__",
+    "//rs/replica:__subpackages__",
+])
 
 DEPENDENCIES = [
     "//rs/async_utils",
@@ -29,6 +33,7 @@ DEPENDENCIES = [
     "@crate_index//:askama",
     "@crate_index//:bytes",
     "@crate_index//:byte-unit",
+    "@crate_index//:cfg-if",
     "@crate_index//:crossbeam",
     "@crate_index//:futures",
     "@crate_index//:futures-util",
@@ -101,10 +106,15 @@ rust_library(
     aliases = ALIASES,
     crate_features = select({
         "//bazel:malicious_code_enabled": ["malicious_code"],
+        "//bazel:fuzzing_code_enabled": ["fuzzing_code"],
         "//conditions:default": [],
     }),
     crate_name = "ic_http_endpoints_public",
     proc_macro_deps = MACRO_DEPENDENCIES,
+    rustc_flags = select({
+        "//bazel:fuzzing_code_enabled": DEFAULT_RUSTC_FLAGS_FOR_FUZZING,
+        "//conditions:default": [],
+    }),
     version = "0.9.0",
     deps = DEPENDENCIES + [":build_script"],
 )

rs/http_endpoints/public/BUILD.bazel

@@ -11,6 +11,7 @@ askama = { workspace = true }
 async-trait = "0.1.68"
 bytes = { workspace = true }
 byte-unit = "4.0.14"
+cfg-if = "1.0.0"
 crossbeam = "0.8.2"
 hex = "0.4.2"
 http = "0.2.5"

rs/http_endpoints/public/Cargo.toml

@@ -38,7 +38,7 @@ use tower::{
 };
 
 #[derive(Clone)]
-pub(crate) struct CallService {
+pub struct CallService {
     log: ReplicaLogger,
     metrics: HttpHandlerMetrics,
     node_id: NodeId,
@@ -52,7 +52,7 @@ pub(crate) struct CallService {
 
 impl CallService {
     #[allow(clippy::too_many_arguments)]
-    pub(crate) fn new_service(
+    pub fn new_service(
         config: Config,
         log: ReplicaLogger,
         metrics: HttpHandlerMetrics,

rs/http_endpoints/public/src/call.rs

@@ -6,20 +6,29 @@
 //! naming used in the [Interface
 //! Specification](https://sdk.dfinity.org/docs/interface-spec/index.html)
 mod body;
-mod call;
 mod catch_up_package;
 mod common;
 mod dashboard;
 mod health_status_refresher;
-mod metrics;
 mod pprof;
 mod query;
 mod read_state;
 mod state_reader_executor;
 mod status;
 mod threads;
 mod types;
-mod validator_executor;
+
+cfg_if::cfg_if! {
+    if #[cfg(feature = "fuzzing_code")] {
+        pub mod validator_executor;
+        pub mod metrics;
+        pub mod call;
+    } else {
+        mod validator_executor;
+        mod metrics;
+        mod call;
+    }
+}
 
 use crate::{
     body::BodyReceiverLayer,
@@ -109,7 +118,7 @@ const HTTP_DASHBOARD_URL_PATH: &str = "/_/dashboard";
 const CONTENT_TYPE_CBOR: &str = "application/cbor";
 
 #[derive(Debug, Clone, PartialEq)]
-pub(crate) struct HttpError {
+pub struct HttpError {
     pub status: StatusCode,
     pub message: String,
 }

rs/http_endpoints/public/src/lib.rs

@@ -22,7 +22,7 @@ pub const REQUESTS_LABEL_NAMES: [&str; REQUESTS_NUM_LABELS] = [LABEL_REQUEST_TYP
 // Struct holding only Prometheus metric objects. Hence, it is thread-safe iff
 // the data members are thread-safe.
 #[derive(Clone)]
-pub(crate) struct HttpHandlerMetrics {
+pub struct HttpHandlerMetrics {
     pub requests: HistogramVec,
     pub request_body_size_bytes: HistogramVec,
     pub response_body_size_bytes: HistogramVec,

rs/http_endpoints/public/src/metrics.rs

@@ -20,7 +20,7 @@ use tokio::sync::oneshot;
 const VALIDATOR_EXECUTOR_THREADS: usize = 1;
 
 #[derive(Clone)]
-pub(crate) struct ValidatorExecutor<C> {
+pub struct ValidatorExecutor<C> {
     registry_client: Arc<dyn RegistryClient>,
     validator: Arc<dyn HttpRequestVerifier<C, RegistryRootOfTrustProvider>>,
     threadpool: ThreadPool,

rs/http_endpoints/public/src/validator_executor.rs

@@ -104,7 +104,7 @@ fn setup_query_execution_mock() -> (QueryExecutionService, QueryExecutionHandle)
 }
 
 #[allow(clippy::type_complexity)]
-fn setup_ingress_filter_mock() -> (IngressFilterService, IngressFilterHandle) {
+pub fn setup_ingress_filter_mock() -> (IngressFilterService, IngressFilterHandle) {
     let (service, handle) = tower_test::mock::pair::<
         (ProvisionalWhitelist, SignedIngressContent),
         Result<(), UserError>,