-
Notifications
You must be signed in to change notification settings - Fork 70
/
gc.rs
243 lines (205 loc) · 7.34 KB
/
gc.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
//! Garbage collection.
use std::sync::Arc;
use std::time::Duration;
use anyhow::{anyhow, Result};
use chrono::{Duration as ChronoDuration, Utc};
use futures::future::join_all;
use sea_orm::entity::prelude::*;
use sea_orm::query::QuerySelect;
use sea_orm::sea_query::{LockBehavior, LockType, Query};
use sea_orm::{ConnectionTrait, FromQueryResult};
use tokio::sync::Semaphore;
use tokio::time;
use tracing::instrument;
use super::{State, StateInner};
use crate::config::Config;
use crate::database::entity::cache::{self, Entity as Cache};
use crate::database::entity::chunk::{self, ChunkState, Entity as Chunk};
use crate::database::entity::chunkref::{self, Entity as ChunkRef};
use crate::database::entity::nar::{self, Entity as Nar, NarState};
use crate::database::entity::object::{self, Entity as Object};
#[derive(Debug, FromQueryResult)]
struct CacheIdAndRetentionPeriod {
id: i64,
name: String,
retention_period: i32,
}
/// Runs garbage collection periodically.
pub async fn run_garbage_collection(config: Config) {
let interval = config.garbage_collection.interval;
if interval == Duration::ZERO {
// disabled
return;
}
loop {
// We don't stop even if it errors
if let Err(e) = run_garbage_collection_once(config.clone()).await {
tracing::warn!("Garbage collection failed: {}", e);
}
time::sleep(interval).await;
}
}
/// Runs garbage collection once.
#[instrument(skip_all)]
pub async fn run_garbage_collection_once(config: Config) -> Result<()> {
tracing::info!("Running garbage collection...");
let state = StateInner::new(config).await;
run_time_based_garbage_collection(&state).await?;
run_reap_orphan_nars(&state).await?;
run_reap_orphan_chunks(&state).await?;
Ok(())
}
#[instrument(skip_all)]
async fn run_time_based_garbage_collection(state: &State) -> Result<()> {
let db = state.database().await?;
let now = Utc::now();
let default_retention_period = state.config.garbage_collection.default_retention_period;
let retention_period =
cache::Column::RetentionPeriod.if_null(default_retention_period.as_secs() as i32);
// Find caches with retention periods set
let caches = Cache::find()
.select_only()
.column(cache::Column::Id)
.column(cache::Column::Name)
.column_as(retention_period.clone(), "retention_period")
.filter(retention_period.ne(0))
.into_model::<CacheIdAndRetentionPeriod>()
.all(db)
.await?;
tracing::info!(
"Found {} caches subject to time-based garbage collection",
caches.len()
);
let mut objects_deleted = 0;
for cache in caches {
let period = ChronoDuration::seconds(cache.retention_period.into());
let cutoff = now.checked_sub_signed(period).ok_or_else(|| {
anyhow!(
"Somehow subtracting retention period for cache {} underflowed",
cache.name
)
})?;
let deletion = Object::delete_many()
.filter(object::Column::CacheId.eq(cache.id))
.filter(object::Column::CreatedAt.lt(cutoff))
.filter(
object::Column::LastAccessedAt
.is_null()
.or(object::Column::LastAccessedAt.lt(cutoff)),
)
.exec(db)
.await?;
tracing::info!(
"Deleted {} objects from {} (ID {})",
deletion.rows_affected,
cache.name,
cache.id
);
objects_deleted += deletion.rows_affected;
}
tracing::info!("Deleted {} objects in total", objects_deleted);
Ok(())
}
#[instrument(skip_all)]
async fn run_reap_orphan_nars(state: &State) -> Result<()> {
let db = state.database().await?;
// find all orphan NARs...
let orphan_nar_ids = Query::select()
.from(Nar)
.expr(nar::Column::Id.into_expr())
.left_join(
Object,
object::Column::NarId
.into_expr()
.eq(nar::Column::Id.into_expr()),
)
.and_where(object::Column::Id.is_null())
.and_where(nar::Column::State.eq(NarState::Valid))
.and_where(nar::Column::HoldersCount.eq(0))
.lock_with_tables_behavior(LockType::Update, [Nar], LockBehavior::SkipLocked)
.to_owned();
// ... and simply delete them
let deletion = Nar::delete_many()
.filter(nar::Column::Id.in_subquery(orphan_nar_ids))
.exec(db)
.await?;
tracing::info!("Deleted {} orphan NARs", deletion.rows_affected,);
Ok(())
}
#[instrument(skip_all)]
async fn run_reap_orphan_chunks(state: &State) -> Result<()> {
let db = state.database().await?;
let storage = state.storage().await?;
// find all orphan chunks...
let orphan_chunk_ids = Query::select()
.from(Chunk)
.expr(chunk::Column::Id.into_expr())
.left_join(
ChunkRef,
chunkref::Column::ChunkId
.into_expr()
.eq(chunk::Column::Id.into_expr()),
)
.and_where(chunkref::Column::Id.is_null())
.and_where(chunk::Column::State.eq(ChunkState::Valid))
.and_where(chunk::Column::HoldersCount.eq(0))
.lock_with_tables_behavior(LockType::Update, [Chunk], LockBehavior::SkipLocked)
.to_owned();
// ... and transition their state to Deleted
//
// Deleted chunks are essentially invisible from our normal queries
let transition_statement = {
let change_state = Query::update()
.table(Chunk)
.value(chunk::Column::State, ChunkState::Deleted)
.and_where(chunk::Column::Id.in_subquery(orphan_chunk_ids))
.to_owned();
db.get_database_backend().build(&change_state)
};
db.execute(transition_statement).await?;
let orphan_chunks: Vec<chunk::Model> = Chunk::find()
.filter(chunk::Column::State.eq(ChunkState::Deleted))
.all(db)
.await?;
if orphan_chunks.is_empty() {
return Ok(());
}
// Delete the chunks from remote storage
let delete_limit = Arc::new(Semaphore::new(20)); // TODO: Make this configurable
let futures: Vec<_> = orphan_chunks
.into_iter()
.map(|chunk| {
let delete_limit = delete_limit.clone();
async move {
let permit = delete_limit.acquire().await?;
storage.delete_file_db(&chunk.remote_file.0).await?;
drop(permit);
Result::<_, anyhow::Error>::Ok(chunk.id)
}
})
.collect();
// Deletions can result in spurious failures, tolerate them
//
// Chunks that failed to be deleted from the remote storage will
// just be stuck in Deleted state.
//
// TODO: Maybe have an interactive command to retry deletions?
let deleted_chunk_ids: Vec<_> = join_all(futures)
.await
.into_iter()
.filter(|r| {
if let Err(e) = r {
tracing::warn!("Deletion failed: {}", e);
}
r.is_ok()
})
.map(|r| r.unwrap())
.collect();
// Finally, delete them from the database
let deletion = Chunk::delete_many()
.filter(chunk::Column::Id.is_in(deleted_chunk_ids))
.exec(db)
.await?;
tracing::info!("Deleted {} orphan chunks", deletion.rows_affected);
Ok(())
}