Skip to content

fix: check for draining before installing/creating dns #773

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 15, 2024
Merged

fix: check for draining before installing/creating dns #773

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 17 additions & 15 deletions svc/pkg/cluster/worker/src/workers/server_dns_create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::util::CloudflareError;
struct Server {
datacenter_id: Uuid,
public_ip: IpAddr,
cloud_destroy_ts: Option<i64>,
is_destroying_or_draining: bool,
}

#[worker(name = "cluster-server-dns-create")]
Expand Down Expand Up @@ -44,18 +44,6 @@ async fn inner(
) -> GlobalResult<()> {
let server_id = unwrap_ref!(ctx.server_id).as_uuid();

let server = sql_fetch_one!(
[ctx, Server, @tx tx]
"
SELECT
datacenter_id, public_ip, cloud_destroy_ts
FROM db_cluster.servers
WHERE server_id = $1
",
server_id,
)
.await?;

// Lock row
sql_execute!(
[ctx, @tx tx]
Expand All @@ -70,8 +58,22 @@ async fn inner(
)
.await?;

if server.cloud_destroy_ts.is_some() {
tracing::info!("server marked for deletion, not creating dns record");
let server = sql_fetch_one!(
[ctx, Server, @tx tx]
"
SELECT
datacenter_id,
public_ip,
(cloud_destroy_ts IS NOT NULL OR drain_ts IS NOT NULL) AS is_destroying_or_draining
FROM db_cluster.servers
WHERE server_id = $1
",
server_id,
)
.await?;

if server.is_destroying_or_draining {
tracing::info!("server marked for deletion/drain, not creating dns record");
return Ok(());
}

Expand Down
11 changes: 6 additions & 5 deletions svc/pkg/cluster/worker/src/workers/server_dns_delete.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use proto::backend::pkg::*;
use crate::util::CloudflareError;

#[derive(sqlx::FromRow)]
struct DnsRecord {
struct DnsRecordRow {
dns_record_id: Option<String>,
secondary_dns_record_id: Option<String>,
}
Expand Down Expand Up @@ -44,7 +44,7 @@ async fn inner(
let server_id = unwrap_ref!(ctx.server_id).as_uuid();

let row = sql_fetch_optional!(
[ctx, DnsRecord, @tx tx]
[ctx, DnsRecordRow, @tx tx]
"
SELECT dns_record_id, secondary_dns_record_id
FROM db_cluster.servers_cloudflare
Expand All @@ -57,7 +57,8 @@ async fn inner(
util::timestamp::now(),
)
.await?;
let Some(DnsRecord {

let Some(DnsRecordRow {
dns_record_id,
secondary_dns_record_id,
}) = row
Expand Down Expand Up @@ -87,7 +88,7 @@ async fn inner(
tracing::warn!(%zone_id, %record_id, "dns record not found");
} else {
res?;
tracing::warn!(%record_id, "deleted dns record");
tracing::info!(%record_id, "deleted dns record");
}
} else {
tracing::warn!("server has no primary dns record");
Expand All @@ -102,7 +103,7 @@ async fn inner(
})
.await?;

tracing::warn!(%record_id, "deleted secondary dns record");
tracing::info!(%record_id, "deleted secondary dns record");
} else {
tracing::warn!("server has no secondary dns record");
}
Expand Down
30 changes: 4 additions & 26 deletions svc/pkg/cluster/worker/src/workers/server_install/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,22 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
}

if let Some(server_id) = ctx.server_id {
let (is_destroying,) = sql_fetch_one!(
let (is_destroying_or_draining,) = sql_fetch_one!(
[ctx, (bool,)]
"
SELECT EXISTS(
SELECT 1
FROM db_cluster.servers
WHERE server_id = $1 AND
cloud_destroy_ts IS NOT NULL
(cloud_destroy_ts IS NOT NULL OR drain_ts IS NOT NULL)
)
",
server_id.as_uuid(),
)
.await?;

if is_destroying {
tracing::info!("server marked for deletion, not installing");
if is_destroying_or_draining {
tracing::info!("server marked for deletion/drain, not installing");
return Ok(());
}
}
Expand Down Expand Up @@ -133,28 +133,6 @@ async fn worker(ctx: &OperationContext<cluster::msg::server_install::Message>) -
})
.await??;

// Check if destroyed again after installing
if let Some(server_id) = ctx.server_id {
let (is_destroying,) = sql_fetch_one!(
[ctx, (bool,)]
"
SELECT EXISTS(
SELECT 1
FROM db_cluster.servers
WHERE server_id = $1 AND
cloud_destroy_ts IS NOT NULL
)
",
server_id.as_uuid(),
)
.await?;

if is_destroying {
tracing::info!("server marked for deletion, not marking as installed");
return Ok(());
}
}

let request_id = unwrap_ref!(ctx.request_id).as_uuid();
msg!([ctx] cluster::msg::server_install_complete(request_id) {
request_id: ctx.request_id,
Expand Down