Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replace sleep calls in tests with a direct cache refresh function #1648

Merged
merged 4 commits into from
Jul 27, 2023
Merged

Replace sleep calls in tests with a direct cache refresh function #1648

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a92d82e
Replace `sleep` calls in tests with a direct cache refresh function
inahga Jul 26, 2023
de52002
Refactor refresh code into single function, add more tracing
inahga Jul 27, 2023
2fdd57c
Use field directly in tracing event
inahga Jul 27, 2023
67d96fc
Use debug implementation for recording duration
inahga Jul 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions aggregator/src/aggregator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use crate::{
query_type::{CollectableQueryType, UploadableQueryType},
report_writer::{ReportWriteBatcher, WritableReport},
},
cache::GlobalHpkeKeypairCache,
cache::{GlobalHpkeKeypairCache, DEFAULT_REFRESH_INTERVAL},
config::TaskprovConfig,
Operation,
};
Expand Down Expand Up @@ -163,7 +163,7 @@ pub struct Aggregator<C: Clock> {
aggregate_step_failure_counter: Counter<u64>,

/// Cache of global HPKE keypairs and configs.
global_hpke_keypairs: GlobalHpkeKeypairCache,
global_hpke_keypairs: GlobalHpkeKeypairCache<C>,
}

/// Config represents a configuration for an Aggregator.
Expand Down Expand Up @@ -196,7 +196,7 @@ impl Default for Config {
max_upload_batch_size: 1,
max_upload_batch_write_delay: StdDuration::ZERO,
batch_aggregation_shard_count: 1,
global_hpke_configs_refresh_interval: GlobalHpkeKeypairCache::DEFAULT_REFRESH_INTERVAL,
global_hpke_configs_refresh_interval: DEFAULT_REFRESH_INTERVAL,
taskprov_config: TaskprovConfig::default(),
}
}
Expand Down Expand Up @@ -496,6 +496,11 @@ impl<C: Clock> Aggregator<C> {
Ok(Arc::clone(task_aggs.entry(*task_id).or_insert(task_agg)))
}
}

#[cfg(feature = "test-util")]
pub async fn refresh_caches(&self) -> Result<(), Error> {
self.global_hpke_keypairs.refresh().await
}
}

/// TaskAggregator provides aggregation functionality for a single task.
Expand Down Expand Up @@ -620,7 +625,7 @@ impl<C: Clock> TaskAggregator<C> {
async fn handle_upload(
&self,
clock: &C,
global_hpke_keypairs: &GlobalHpkeKeypairCache,
global_hpke_keypairs: &GlobalHpkeKeypairCache<C>,
upload_decrypt_failure_counter: &Counter<u64>,
upload_decode_failure_counter: &Counter<u64>,
report: Report,
Expand All @@ -641,7 +646,7 @@ impl<C: Clock> TaskAggregator<C> {
async fn handle_aggregate_init(
&self,
datastore: &Datastore<C>,
global_hpke_keypairs: &GlobalHpkeKeypairCache,
global_hpke_keypairs: &GlobalHpkeKeypairCache<C>,
aggregate_step_failure_counter: &Counter<u64>,
aggregation_job_id: &AggregationJobId,
req_bytes: &[u8],
Expand Down Expand Up @@ -876,7 +881,7 @@ impl VdafOps {
async fn handle_upload<C: Clock>(
&self,
clock: &C,
global_hpke_keypairs: &GlobalHpkeKeypairCache,
global_hpke_keypairs: &GlobalHpkeKeypairCache<C>,
upload_decrypt_failure_counter: &Counter<u64>,
upload_decode_failure_counter: &Counter<u64>,
task: &Task,
Expand Down Expand Up @@ -927,7 +932,7 @@ impl VdafOps {
async fn handle_aggregate_init<C: Clock>(
&self,
datastore: &Datastore<C>,
global_hpke_keypairs: &GlobalHpkeKeypairCache,
global_hpke_keypairs: &GlobalHpkeKeypairCache<C>,
aggregate_step_failure_counter: &Counter<u64>,
task: Arc<Task>,
aggregation_job_id: &AggregationJobId,
Expand Down Expand Up @@ -1019,7 +1024,7 @@ impl VdafOps {
async fn handle_upload_generic<const SEED_SIZE: usize, Q, A, C>(
vdaf: Arc<A>,
clock: &C,
global_hpke_keypairs: &GlobalHpkeKeypairCache,
global_hpke_keypairs: &GlobalHpkeKeypairCache<C>,
upload_decrypt_failure_counter: &Counter<u64>,
upload_decode_failure_counter: &Counter<u64>,
task: &Task,
Expand Down Expand Up @@ -1307,7 +1312,7 @@ impl VdafOps {
/// helper, described in §4.4.4.1 of draft-gpew-priv-ppm.
async fn handle_aggregate_init_generic<const SEED_SIZE: usize, Q, A, C>(
datastore: &Datastore<C>,
global_hpke_keypairs: &GlobalHpkeKeypairCache,
global_hpke_keypairs: &GlobalHpkeKeypairCache<C>,
vdaf: &A,
aggregate_step_failure_counter: &Counter<u64>,
task: Arc<Task>,
Expand Down Expand Up @@ -2823,7 +2828,6 @@ mod tests {
};
use rand::random;
use std::{collections::HashSet, iter, sync::Arc, time::Duration as StdDuration};
use tokio::time::sleep;

pub(crate) const BATCH_AGGREGATION_SHARD_COUNT: u64 = 32;

Expand Down Expand Up @@ -3217,9 +3221,7 @@ mod tests {
})
.await
.unwrap();

// Let keypair cache refresh.
sleep(StdDuration::from_millis(750)).await;
aggregator.refresh_caches().await.unwrap();

for report in [
create_report(&task, clock.now()),
Expand Down
29 changes: 22 additions & 7 deletions aggregator/src/aggregator/http_handlers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,13 @@ pub async fn aggregator_handler<C: Clock>(
cfg: Config,
) -> Result<impl Handler, Error> {
let aggregator = Arc::new(Aggregator::new(datastore, clock, meter, cfg).await?);
aggregator_handler_with_aggregator(aggregator, meter).await
}

async fn aggregator_handler_with_aggregator<C: Clock>(
aggregator: Arc<Aggregator<C>>,
meter: &Meter,
) -> Result<impl Handler, Error> {
Ok((
State(aggregator),
metrics("janus_aggregator").with_route(|conn| conn.route().map(ToString::to_string)),
Expand Down Expand Up @@ -554,7 +560,7 @@ mod tests {
},
collection_job_tests::setup_collection_job_test_case,
empty_batch_aggregations,
http_handlers::aggregator_handler,
http_handlers::{aggregator_handler, aggregator_handler_with_aggregator},
tests::{
create_report, create_report_with_id, default_aggregator_config,
generate_helper_report_share, generate_helper_report_share_for_plaintext,
Expand Down Expand Up @@ -613,7 +619,6 @@ mod tests {
use std::{
borrow::Cow, collections::HashMap, io::Cursor, sync::Arc, time::Duration as StdDuration,
};
use tokio::time::sleep;
use trillium::{KnownHeaderName, Status};
use trillium_testing::{
assert_headers,
Expand Down Expand Up @@ -727,7 +732,17 @@ mod tests {
..Default::default()
};

let handler = aggregator_handler(datastore.clone(), clock.clone(), &noop_meter(), cfg)
let aggregator = Arc::new(
crate::aggregator::Aggregator::new(
datastore.clone(),
clock.clone(),
&noop_meter(),
cfg,
)
.await
.unwrap(),
);
let handler = aggregator_handler_with_aggregator(aggregator.clone(), &noop_meter())
.await
.unwrap();

Expand Down Expand Up @@ -756,7 +771,7 @@ mod tests {
})
.await
.unwrap();
sleep(StdDuration::from_millis(750)).await;
aggregator.refresh_caches().await.unwrap();
let mut test_conn = get("/hpke_config").run_async(&handler).await;
assert_eq!(test_conn.status(), Some(Status::Ok));
let bytes = take_response_body(&mut test_conn).await;
Expand All @@ -777,7 +792,7 @@ mod tests {
})
.await
.unwrap();
sleep(StdDuration::from_millis(750)).await;
aggregator.refresh_caches().await.unwrap();
let mut test_conn = get("/hpke_config").run_async(&handler).await;
assert_eq!(test_conn.status(), Some(Status::Ok));
let bytes = take_response_body(&mut test_conn).await;
Expand Down Expand Up @@ -813,7 +828,7 @@ mod tests {
})
.await
.unwrap();
sleep(StdDuration::from_millis(750)).await;
aggregator.refresh_caches().await.unwrap();
let mut test_conn = get("/hpke_config").run_async(&handler).await;
assert_eq!(test_conn.status(), Some(Status::Ok));
let bytes = take_response_body(&mut test_conn).await;
Expand All @@ -831,7 +846,7 @@ mod tests {
})
.await
.unwrap();
sleep(StdDuration::from_millis(750)).await;
aggregator.refresh_caches().await.unwrap();
let test_conn = get("/hpke_config").run_async(&handler).await;
assert_eq!(test_conn.status(), Some(Status::BadRequest));
}
Expand Down
4 changes: 2 additions & 2 deletions aggregator/src/bin/aggregator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use janus_aggregator::{
binary_utils::{
janus_main, setup_server, setup_signal_handler, BinaryOptions, CommonBinaryOptions,
},
cache::GlobalHpkeKeypairCache,
cache::DEFAULT_REFRESH_INTERVAL,
config::{BinaryConfig, CommonConfig, TaskprovConfig},
};
use janus_aggregator_api::{self, aggregator_api_handler};
Expand Down Expand Up @@ -329,7 +329,7 @@ impl Config {
taskprov_config: self.taskprov_config.clone(),
global_hpke_configs_refresh_interval: match self.global_hpke_configs_refresh_interval {
Some(duration) => Duration::from_millis(duration),
None => GlobalHpkeKeypairCache::DEFAULT_REFRESH_INTERVAL,
None => DEFAULT_REFRESH_INTERVAL,
},
}
}
Expand Down
37 changes: 28 additions & 9 deletions aggregator/src/cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,15 @@ use std::{
use tokio::{spawn, task::JoinHandle, time::sleep};
use tracing::error;

pub const DEFAULT_REFRESH_INTERVAL: StdDuration =
StdDuration::from_secs(60 * 30 /* 30 minutes */);

inahga marked this conversation as resolved.
Show resolved Hide resolved
type HpkeConfigs = Arc<Vec<HpkeConfig>>;
type HpkeKeypairs = HashMap<HpkeConfigId, Arc<HpkeKeypair>>;

#[derive(Debug)]
pub struct GlobalHpkeKeypairCache {
pub struct GlobalHpkeKeypairCache<C: Clock> {
datastore: Arc<Datastore<C>>,
// We use a std::sync::Mutex in this cache because we won't hold locks across
// `.await` boundaries. StdMutex is lighter weight than `tokio::sync::Mutex`.
/// Cache of HPKE configs for advertisement.
Expand All @@ -33,11 +37,8 @@ pub struct GlobalHpkeKeypairCache {
refresh_handle: JoinHandle<()>,
}

impl GlobalHpkeKeypairCache {
pub const DEFAULT_REFRESH_INTERVAL: StdDuration =
StdDuration::from_secs(60 * 30 /* 30 minutes */);

pub async fn new<C: Clock>(
impl<C: Clock> GlobalHpkeKeypairCache<C> {
pub async fn new(
datastore: Arc<Datastore<C>>,
refresh_interval: StdDuration,
) -> Result<Self, Error> {
Expand All @@ -49,11 +50,12 @@ impl GlobalHpkeKeypairCache {
// Start refresh task.
let refresh_configs = configs.clone();
let refresh_keypairs = keypairs.clone();
let refresh_datastore = datastore.clone();
let refresh_handle = spawn(async move {
loop {
sleep(refresh_interval).await;

match Self::get_global_keypairs(&datastore).await {
match Self::get_global_keypairs(&refresh_datastore).await {
Ok(global_keypairs) => {
let new_configs = Self::filter_active_configs(&global_keypairs);
let new_keypairs = Self::map_keypairs(&global_keypairs);
Expand All @@ -74,6 +76,7 @@ impl GlobalHpkeKeypairCache {
});

Ok(Self {
datastore,
configs,
keypairs,
refresh_handle,
Expand Down Expand Up @@ -102,7 +105,7 @@ impl GlobalHpkeKeypairCache {
.collect()
}

async fn get_global_keypairs<C: Clock>(
async fn get_global_keypairs(
datastore: &Datastore<C>,
) -> Result<Vec<GlobalHpkeKeypair>, Error> {
Ok(datastore
Expand All @@ -125,9 +128,25 @@ impl GlobalHpkeKeypairCache {
let keypairs = self.keypairs.lock().unwrap();
keypairs.get(id).cloned()
}

#[cfg(feature = "test-util")]
pub async fn refresh(&self) -> Result<(), Error> {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should extract the common body of this and the async block in new() to a new private method.

let global_keypairs = Self::get_global_keypairs(&self.datastore).await?;
let new_configs = Self::filter_active_configs(&global_keypairs);
let new_keypairs = Self::map_keypairs(&global_keypairs);
{
let mut configs = self.configs.lock().unwrap();
*configs = new_configs;
}
{
let mut keypairs = self.keypairs.lock().unwrap();
*keypairs = new_keypairs;
}
Ok(())
}
}

impl Drop for GlobalHpkeKeypairCache {
impl<C: Clock> Drop for GlobalHpkeKeypairCache<C> {
fn drop(&mut self) {
self.refresh_handle.abort()
}
Expand Down
Loading