-
Notifications
You must be signed in to change notification settings - Fork 367
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(pagebench): add aux file bench (#7746)
part of #7462 ## Summary of changes This pull request adds two APIs to the pageserver management API: list_aux_files and ingest_aux_files. The aux file pagebench is intended to be used on an empty timeline because the data do not go through the safekeeper. LSNs are advanced by 8 for each ingestion, to avoid invariant checks inside the pageserver. For now, I only care about space amplification / read amplification, so the bench is designed in a very simple way: ingest 10000 files, and I will manually dump the layer map to analyze. --------- Signed-off-by: Alex Chi Z <chi@neon.tech>
- Loading branch information
1 parent
21ef21b
commit 052ca59
Showing
5 changed files
with
243 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
use pageserver_api::models::{AuxFilePolicy, TenantConfig, TenantConfigRequest}; | ||
use pageserver_api::shard::TenantShardId; | ||
use utils::id::TenantTimelineId; | ||
use utils::lsn::Lsn; | ||
|
||
use std::collections::HashMap; | ||
use std::sync::Arc; | ||
|
||
/// Ingest aux files into the pageserver. | ||
#[derive(clap::Parser)] | ||
pub(crate) struct Args { | ||
#[clap(long, default_value = "http://localhost:9898")] | ||
mgmt_api_endpoint: String, | ||
#[clap(long, default_value = "postgres://postgres@localhost:64000")] | ||
page_service_connstring: String, | ||
#[clap(long)] | ||
pageserver_jwt: Option<String>, | ||
|
||
targets: Option<Vec<TenantTimelineId>>, | ||
} | ||
|
||
pub(crate) fn main(args: Args) -> anyhow::Result<()> { | ||
let rt = tokio::runtime::Builder::new_multi_thread() | ||
.enable_all() | ||
.build() | ||
.unwrap(); | ||
|
||
let main_task = rt.spawn(main_impl(args)); | ||
rt.block_on(main_task).unwrap() | ||
} | ||
|
||
async fn main_impl(args: Args) -> anyhow::Result<()> { | ||
let args: &'static Args = Box::leak(Box::new(args)); | ||
|
||
let mgmt_api_client = Arc::new(pageserver_client::mgmt_api::Client::new( | ||
args.mgmt_api_endpoint.clone(), | ||
args.pageserver_jwt.as_deref(), | ||
)); | ||
|
||
// discover targets | ||
let timelines: Vec<TenantTimelineId> = crate::util::cli::targets::discover( | ||
&mgmt_api_client, | ||
crate::util::cli::targets::Spec { | ||
limit_to_first_n_targets: None, | ||
targets: { | ||
if let Some(targets) = &args.targets { | ||
if targets.len() != 1 { | ||
anyhow::bail!("must specify exactly one target"); | ||
} | ||
Some(targets.clone()) | ||
} else { | ||
None | ||
} | ||
}, | ||
}, | ||
) | ||
.await?; | ||
|
||
let timeline = timelines[0]; | ||
let tenant_shard_id = TenantShardId::unsharded(timeline.tenant_id); | ||
let timeline_id = timeline.timeline_id; | ||
|
||
println!("operating on timeline {}", timeline); | ||
|
||
mgmt_api_client | ||
.tenant_config(&TenantConfigRequest { | ||
tenant_id: timeline.tenant_id, | ||
config: TenantConfig { | ||
switch_aux_file_policy: Some(AuxFilePolicy::V2), | ||
..Default::default() | ||
}, | ||
}) | ||
.await?; | ||
|
||
for batch in 0..100 { | ||
let items = (0..100) | ||
.map(|id| { | ||
( | ||
format!("pg_logical/mappings/{:03}.{:03}", batch, id), | ||
format!("{:08}", id), | ||
) | ||
}) | ||
.collect::<HashMap<_, _>>(); | ||
let file_cnt = items.len(); | ||
mgmt_api_client | ||
.ingest_aux_files(tenant_shard_id, timeline_id, items) | ||
.await?; | ||
println!("ingested {file_cnt} files"); | ||
} | ||
|
||
let files = mgmt_api_client | ||
.list_aux_files(tenant_shard_id, timeline_id, Lsn(Lsn::MAX.0 - 1)) | ||
.await?; | ||
|
||
println!("{} files found", files.len()); | ||
|
||
anyhow::Ok(()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters