Skip to content

fix: allocation sizes for nomad #1127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

fix: allocation sizes for nomad #1127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 29 additions & 27 deletions lib/bolt/core/src/context/service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,33 +510,35 @@ impl ServiceContextData {
);
}

let can_depend =
if self.is_monolith_worker() {
matches!(
dep.config().kind,
ServiceKind::Database { .. }
| ServiceKind::Cache { .. } | ServiceKind::Operation { .. }
| ServiceKind::Package { .. }
| ServiceKind::Consumer { .. }
)
} else if matches!(self.config().kind, ServiceKind::Api { .. }) {
matches!(
dep.config().kind,
ServiceKind::Database { .. }
| ServiceKind::Cache { .. } | ServiceKind::Operation { .. }
| ServiceKind::Package { .. }
| ServiceKind::ApiRoutes { .. }
| ServiceKind::Consumer { .. }
)
} else {
matches!(
dep.config().kind,
ServiceKind::Database { .. }
| ServiceKind::Cache { .. } | ServiceKind::Operation { .. }
| ServiceKind::Package { .. }
| ServiceKind::Consumer { .. }
)
};
let can_depend = if self.is_monolith_worker() {
matches!(
dep.config().kind,
ServiceKind::Database { .. }
| ServiceKind::Cache { .. }
| ServiceKind::Operation { .. }
| ServiceKind::Package { .. }
| ServiceKind::Consumer { .. }
)
} else if matches!(self.config().kind, ServiceKind::Api { .. }) {
matches!(
dep.config().kind,
ServiceKind::Database { .. }
| ServiceKind::Cache { .. }
| ServiceKind::Operation { .. }
| ServiceKind::Package { .. }
| ServiceKind::ApiRoutes { .. }
| ServiceKind::Consumer { .. }
)
} else {
matches!(
dep.config().kind,
ServiceKind::Database { .. }
| ServiceKind::Cache { .. }
| ServiceKind::Operation { .. }
| ServiceKind::Package { .. }
| ServiceKind::Consumer { .. }
)
};

if !can_depend {
panic!(
Expand Down
8 changes: 4 additions & 4 deletions svc/pkg/cluster/src/workflows/datacenter/scale.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,12 @@ async fn inner(
.map(TryInto::try_into)
.collect::<GlobalResult<Vec<Server>>>()?;

// Sort job servers by memory usage
// Sort job servers by allocated memory
servers.sort_by_key(|server| memory_by_server.get(&server.server_id));

// TODO: remove
tracing::info!(server_ids=?servers.iter().map(|s| s.server_id).collect::<Vec<_>>(), ?memory_by_server, "server topo");

// TODO: RVT-3732 Sort gg and ats servers by cpu usage
// servers.sort_by_key

Expand Down Expand Up @@ -388,7 +391,6 @@ async fn scale_down_job_servers(

let drain_candidates = nomad_servers
.iter()
.rev()
.take(drain_count)
.map(|server| server.server_id);

Expand Down Expand Up @@ -420,7 +422,6 @@ async fn scale_down_gg_servers<'a, I: Iterator<Item = &'a Server> + DoubleEndedI
tracing::info!(count=%drain_count, "draining gg servers");

let drain_candidates = installed_servers
.rev()
.take(drain_count)
.map(|server| server.server_id);

Expand Down Expand Up @@ -455,7 +456,6 @@ async fn scale_down_ats_servers<
tracing::info!(count=%drain_count, "draining ats servers");

let drain_candidates = installed_servers
.rev()
.take(drain_count)
.map(|server| server.server_id);

Expand Down
14 changes: 3 additions & 11 deletions svc/pkg/mm/worker/src/workers/lobby_create/nomad_job.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ pub fn gen_lobby_docker_job(
// Nomad configures CPU based on MHz, not millicores. We havel to calculate the CPU share
// by knowing how many MHz are on the client.
CPU: if tier.rivet_cores_numerator < tier.rivet_cores_denominator {
Some((tier.cpu - util_job::TASK_CLEANUP_CPU as u64).try_into()?)
Some(tier.cpu.try_into()?)
} else {
None
},
Expand All @@ -103,18 +103,10 @@ pub fn gen_lobby_docker_job(
} else {
None
},
memory_mb: Some(
(TryInto::<i64>::try_into(memory)? / (1024 * 1024)
- util_job::TASK_CLEANUP_MEMORY as i64)
.try_into()?,
),
memory_mb: Some(tier.memory.try_into()?),
// Allow oversubscribing memory by 50% of the reserved
// memory if using less than the node's total memory
memory_max_mb: Some(
(TryInto::<i64>::try_into(memory_max)? / (1024 * 1024)
- util_job::TASK_CLEANUP_MEMORY as i64)
.try_into()?,
),
memory_max_mb: Some(tier.memory_max.try_into()?),
disk_mb: Some(tier.disk as i32), // TODO: Is this deprecated?
..Resources::new()
};
Expand Down
11 changes: 1 addition & 10 deletions svc/pkg/tier/ops/list/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,7 @@ async fn handle(ctx: OperationContext<tier::list::Request>) -> GlobalResult<tier
.find(|it| it.hardware_id == hardware),
"datacenter hardware stats not found"
);
let _config = JobNodeConfig::from_linode(instance_type);

let config = JobNodeConfig::from_linode(&linode::types::InstanceType {
hardware_id: "".to_string(),
vcpus: 8,
memory: 2u64.pow(14),
disk: 2u64.pow(15) * 10,
transfer: 6_000,
network_out: 6_000,
});
let config = JobNodeConfig::from_linode(instance_type);

Ok(tier::list::response::Region {
region_id: Some(datacenter_id.into()),
Expand Down