diff --git a/scheds/rust/scx_chaos/src/lib.rs b/scheds/rust/scx_chaos/src/lib.rs index 655e91abe4..4fdde0a9e1 100644 --- a/scheds/rust/scx_chaos/src/lib.rs +++ b/scheds/rust/scx_chaos/src/lib.rs @@ -7,7 +7,6 @@ mod bpf_skel; use bpf_skel::BpfSkel; -use scx_p2dq::P2dqArenaProgs; use scx_p2dq::SchedulerOpts as P2dqOpts; use scx_userspace_arena::alloc::Allocator; use scx_userspace_arena::alloc::HeapAllocator; @@ -17,6 +16,9 @@ use scx_utils::scx_ops_load; use scx_utils::scx_ops_open; use scx_utils::uei_exited; use scx_utils::uei_report; +use scx_utils::Core; +use scx_utils::Llc; +use scx_utils::Topology; use anyhow::bail; use anyhow::Context; @@ -24,7 +26,6 @@ use anyhow::Result; use clap::Parser; use libbpf_rs::OpenObject; use libbpf_rs::ProgramInput; -use libbpf_rs::ProgramOutput; use log::debug; use log::info; use nix::unistd::Pid; @@ -74,24 +75,6 @@ unsafe impl Allocator for ArenaAllocator { } } -impl P2dqArenaProgs for BpfSkel<'_> { - fn run_arena_init<'b>(&self, input: ProgramInput<'b>) -> Result> { - Ok(self.progs.p2dq_arena_init.test_run(input)?) - } - - fn run_alloc_mask<'b>(&self, input: ProgramInput<'b>) -> Result> { - Ok(self.progs.p2dq_alloc_mask.test_run(input)?) - } - - fn run_topology_node_init<'b>(&self, input: ProgramInput<'b>) -> Result> { - Ok(self.progs.p2dq_topology_node_init.test_run(input)?) - } - - fn setup_ptr(&self) -> u64 { - self.maps.bss_data.setup_ptr - } -} - #[derive(Debug)] pub enum Trait { RandomDelays { @@ -194,6 +177,96 @@ impl Scheduler { } impl Builder<'_> { + fn setup_arenas(&self, skel: &mut BpfSkel) -> Result<()> { + // Allocate the arena memory from the BPF side so userspace initializes it before starting + // the scheduler. Despite the function call's name this is neither a test nor a test run, + // it's the recommended way of executing SEC("syscall") probes. + let input = ProgramInput { + ..Default::default() + }; + + let output = skel.progs.p2dq_arena_init.test_run(input)?; + if output.return_value != 0 { + bail!( + "Could not initialize arenas, p2dq_setup returned {}", + output.return_value as i32 + ); + } + + Ok(()) + } + + fn setup_topology_node(&self, skel: &mut BpfSkel, mask: &[u64]) -> Result<()> { + // Copy the address of ptr to the kernel to populate it from BPF with the arena pointer. + let input = ProgramInput { + ..Default::default() + }; + + let output = skel.progs.p2dq_alloc_mask.test_run(input)?; + if output.return_value != 0 { + bail!( + "Could not initialize arenas, setup_topology_node returned {}", + output.return_value as i32 + ); + } + + let ptr = + unsafe { std::mem::transmute::(skel.maps.bss_data.setup_ptr) }; + + let (valid_mask, _) = ptr.split_at_mut(mask.len()); + valid_mask.clone_from_slice(mask); + + let input = ProgramInput { + ..Default::default() + }; + let output = skel.progs.p2dq_topology_node_init.test_run(input)?; + if output.return_value != 0 { + bail!( + "p2dq_topology_node_init returned {}", + output.return_value as i32 + ); + } + + Ok(()) + } + + fn setup_topology(&self, skel: &mut BpfSkel) -> Result<()> { + let topo = Topology::new().expect("Failed to build host topology"); + + self.setup_topology_node(skel, topo.span.as_raw_slice())?; + + for (_, node) in topo.nodes { + self.setup_topology_node(skel, node.span.as_raw_slice())?; + } + + for (_, llc) in topo.all_llcs { + self.setup_topology_node( + skel, + Arc::::into_inner(llc) + .expect("missing llc") + .span + .as_raw_slice(), + )?; + } + + for (_, core) in topo.all_cores { + self.setup_topology_node( + skel, + Arc::::into_inner(core) + .expect("missing core") + .span + .as_raw_slice(), + )?; + } + for (_, cpu) in topo.all_cpus { + let mut mask = [0; 9]; + mask[cpu.id.checked_shr(64).unwrap_or(0)] |= 1 << (cpu.id % 64); + self.setup_topology_node(skel, &mask)?; + } + + Ok(()) + } + fn load_skel(&self) -> Result>> { let mut out: Rc> = Rc::new_uninit(); let uninit_skel = Rc::get_mut(&mut out).expect("brand new rc should be unique"); @@ -306,6 +379,9 @@ impl Builder<'_> { let mut skel = scx_ops_load!(open_skel, chaos, uei)?; scx_p2dq::init_skel!(&mut skel); + self.setup_arenas(&mut skel)?; + self.setup_topology(&mut skel)?; + let out = unsafe { // SAFETY: initialising field by field. open_object is already "initialised" (it's // permanently MaybeUninit so any state is fine), hence the structure will be diff --git a/scheds/rust/scx_p2dq/src/lib.rs b/scheds/rust/scx_p2dq/src/lib.rs index dc020fbeae..2834b6b0d9 100644 --- a/scheds/rust/scx_p2dq/src/lib.rs +++ b/scheds/rust/scx_p2dq/src/lib.rs @@ -5,14 +5,8 @@ pub use scx_utils::CoreType; use scx_utils::Topology; pub use scx_utils::NR_CPU_IDS; -use scx_utils::{Core, Llc}; -use anyhow::{bail, Result}; use clap::Parser; -use libbpf_rs::ProgramInput; -use libbpf_rs::ProgramOutput; - -use std::sync::Arc; lazy_static::lazy_static! { pub static ref TOPO: Topology = Topology::new().unwrap(); @@ -145,110 +139,6 @@ pub fn dsq_slice_ns(dsq_index: u64, min_slice_us: u64, dsq_shift: u64) -> u64 { result } -/// Trait for interfacing with BPF arena programs -pub trait P2dqArenaProgs { - /// Run the arena initialization program and return the result - fn run_arena_init<'a>(&self, input: ProgramInput<'a>) -> Result>; - - /// Run the allocation mask program and return the result - fn run_alloc_mask<'a>(&self, input: ProgramInput<'a>) -> Result>; - - /// Run the topology node initialization program and return the result - fn run_topology_node_init<'a>(&self, input: ProgramInput<'a>) -> Result>; - - /// Access to the setup pointer in BSS data - fn setup_ptr(&self) -> u64; -} - -pub fn setup_arenas(skel: &T) -> Result<()> { - // Allocate the arena memory from the BPF side so userspace initializes it before starting - // the scheduler. Despite the function call's name this is neither a test nor a test run, - // it's the recommended way of executing SEC("syscall") probes. - let input = ProgramInput { - ..Default::default() - }; - - let output = skel.run_arena_init(input)?; - if output.return_value != 0 { - bail!( - "Could not initialize arenas, p2dq_setup returned {}", - output.return_value as i32 - ); - } - - Ok(()) -} - -fn setup_topology_node(skel: &T, mask: &[u64]) -> Result<()> { - // Copy the address of ptr to the kernel to populate it from BPF with the arena pointer. - let input = ProgramInput { - ..Default::default() - }; - - let output = skel.run_alloc_mask(input)?; - if output.return_value != 0 { - bail!( - "Could not initialize arenas, setup_topology_node returned {}", - output.return_value as i32 - ); - } - - let ptr = unsafe { std::mem::transmute::(skel.setup_ptr()) }; - - let (valid_mask, _) = ptr.split_at_mut(mask.len()); - valid_mask.clone_from_slice(mask); - - let input = ProgramInput { - ..Default::default() - }; - let output = skel.run_topology_node_init(input)?; - if output.return_value != 0 { - bail!( - "p2dq_topology_node_init returned {}", - output.return_value as i32 - ); - } - - Ok(()) -} - -pub fn setup_topology(skel: &T) -> Result<()> { - let topo = Topology::new().expect("Failed to build host topology"); - - setup_topology_node(skel, topo.span.as_raw_slice())?; - - for (_, node) in topo.nodes { - setup_topology_node(skel, node.span.as_raw_slice())?; - } - - for (_, llc) in topo.all_llcs { - setup_topology_node( - skel, - Arc::::into_inner(llc) - .expect("missing llc") - .span - .as_raw_slice(), - )?; - } - - for (_, core) in topo.all_cores { - setup_topology_node( - skel, - Arc::::into_inner(core) - .expect("missing core") - .span - .as_raw_slice(), - )?; - } - for (_, cpu) in topo.all_cpus { - let mut mask = [0; 9]; - mask[cpu.id.checked_shr(64).unwrap_or(0)] |= 1 << (cpu.id % 64); - setup_topology_node(skel, &mask)?; - } - - Ok(()) -} - #[macro_export] macro_rules! init_open_skel { ($skel: expr, $opts: expr, $verbose: expr) => { @@ -348,9 +238,6 @@ macro_rules! init_skel { $skel.maps.bss_data.cpu_llc_ids[cpu.id] = cpu.llc_id as u64; $skel.maps.bss_data.cpu_node_ids[cpu.id] = cpu.node_id as u64; } - - $crate::setup_arenas($skel)?; - $crate::setup_topology($skel)?; }; } diff --git a/scheds/rust/scx_p2dq/src/main.rs b/scheds/rust/scx_p2dq/src/main.rs index 123eeb1fe6..69cb6c8924 100644 --- a/scheds/rust/scx_p2dq/src/main.rs +++ b/scheds/rust/scx_p2dq/src/main.rs @@ -34,8 +34,10 @@ use scx_utils::scx_ops_load; use scx_utils::scx_ops_open; use scx_utils::uei_exited; use scx_utils::uei_report; +use scx_utils::Topology; use scx_utils::UserExitInfo; use scx_utils::NR_CPU_IDS; +use scx_utils::{Core, Llc}; use bpf_intf::stat_idx_P2DQ_NR_STATS; use bpf_intf::stat_idx_P2DQ_STAT_DIRECT; @@ -50,31 +52,9 @@ use bpf_intf::stat_idx_P2DQ_STAT_SELECT_PICK2; use bpf_intf::stat_idx_P2DQ_STAT_WAKE_LLC; use bpf_intf::stat_idx_P2DQ_STAT_WAKE_MIG; use bpf_intf::stat_idx_P2DQ_STAT_WAKE_PREV; -use scx_p2dq::P2dqArenaProgs; use scx_p2dq::SchedulerOpts; use scx_p2dq::TOPO; -impl P2dqArenaProgs for BpfSkel<'_> { - fn run_arena_init<'b>(&self, input: ProgramInput<'b>) -> Result> { - Ok(self.progs.p2dq_arena_init.test_run(input)?) - } - - fn run_alloc_mask<'b>(&self, input: ProgramInput<'b>) -> Result> { - Ok(self.progs.p2dq_alloc_mask.test_run(input)?) - } - - fn run_topology_node_init<'b>( - &self, - input: ProgramInput<'b>, - ) -> Result> { - Ok(self.progs.p2dq_topology_node_init.test_run(input)?) - } - - fn setup_ptr(&self) -> u64 { - self.maps.bss_data.setup_ptr - } -} - /// scx_p2dq: A pick 2 dumb queuing load balancing scheduler. /// /// The BPF part does simple vtime or round robin scheduling in each domain @@ -180,6 +160,95 @@ impl<'a> Scheduler<'a> { } } + fn setup_arenas(&mut self) -> Result<()> { + // Allocate the arena memory from the BPF side so userspace initializes it before starting + // the scheduler. Despite the function call's name this is neither a test nor a test run, + // it's the recommended way of executing SEC("syscall") probes. + let input = ProgramInput { + ..Default::default() + }; + + let output = self.skel.progs.p2dq_arena_init.test_run(input)?; + if output.return_value != 0 { + bail!( + "Could not initialize arenas, p2dq_setup returned {}", + output.return_value as i32 + ); + } + + Ok(()) + } + + fn setup_topology_node(&mut self, mask: &[u64]) -> Result<()> { + // Copy the address of ptr to the kernel to populate it from BPF with the arena pointer. + let input = ProgramInput { + ..Default::default() + }; + + let output = self.skel.progs.p2dq_alloc_mask.test_run(input)?; + if output.return_value != 0 { + bail!( + "Could not initialize arenas, setup_topology_node returned {}", + output.return_value as i32 + ); + } + + let ptr = unsafe { + std::mem::transmute::(self.skel.maps.bss_data.setup_ptr) + }; + + let (valid_mask, _) = ptr.split_at_mut(mask.len()); + valid_mask.clone_from_slice(mask); + + let input = ProgramInput { + ..Default::default() + }; + let output = self.skel.progs.p2dq_topology_node_init.test_run(input)?; + if output.return_value != 0 { + bail!( + "p2dq_topology_node_init returned {}", + output.return_value as i32 + ); + } + + Ok(()) + } + + fn setup_topology(&mut self) -> Result<()> { + let topo = Topology::new().expect("Failed to build host topology"); + + self.setup_topology_node(topo.span.as_raw_slice())?; + + for (_, node) in topo.nodes { + self.setup_topology_node(node.span.as_raw_slice())?; + } + + for (_, llc) in topo.all_llcs { + self.setup_topology_node( + Arc::::into_inner(llc) + .expect("missing llc") + .span + .as_raw_slice(), + )?; + } + + for (_, core) in topo.all_cores { + self.setup_topology_node( + Arc::::into_inner(core) + .expect("missing core") + .span + .as_raw_slice(), + )?; + } + for (_, cpu) in topo.all_cpus { + let mut mask = [0; 9]; + mask[cpu.id.checked_shr(64).unwrap_or(0)] |= 1 << (cpu.id % 64); + self.setup_topology_node(&mask)?; + } + + Ok(()) + } + fn run(&mut self, shutdown: Arc) -> Result { let (res_ch, req_ch) = self.stats_server.channels(); @@ -212,6 +281,9 @@ impl<'a> Scheduler<'a> { } fn start(&mut self) -> Result<()> { + self.setup_arenas()?; + self.setup_topology()?; + self.struct_ops = Some(scx_ops_attach!(self.skel, p2dq)?); if self.verbose > 1 {