Skip to content

Commit

Permalink
YT-19456: JobController config refactor pt.2 JRM now has its own config
Browse files Browse the repository at this point in the history
Configs diff:
New config path ```"job_resource_manager"``` has been added to the cluster node configs (both static and dynamic). Following options were moved there from ```"exec_node/job_controller"```:
```"resource_limits"``` -- static
```"cpu_model"``` -- static
```"start_port"``` -- static
```"port_count"``` -- static
```"port_set"``` -- static
```"cpu_to_vcpu_factor"``` -- static and dynamic
```"enable_cpu_to_vcpu_factor"``` -- dynamic
```"cpu_model_to_cpu_to_vcpu_factor"``` -- dynamic
```"profiling_period"``` -- copied to dynamic, not moved
```"memory_pressure_detector"``` -- dynamic
```"mapped_memory_controller"``` -- dynamic
```"free_memory_watermark"``` -- dynamic
  • Loading branch information
E1pp committed Nov 29, 2023
1 parent c113ec8 commit cf8a4c5
Showing 1 changed file with 12 additions and 6 deletions.
18 changes: 12 additions & 6 deletions pkg/ytconfig/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,12 @@ type GpuManager struct {
}

type JobController struct {
ResourceLimitsLegacy JobResourceLimits `yson:"resource_limits"`
GpuManager GpuManager `yson:"gpu_manager"`
}

type JobResourceManager struct {
ResourceLimits JobResourceLimits `yson:"resource_limits"`
GpuManager GpuManager `yson:"gpu_manager"`
}

type JobProxy struct {
Expand Down Expand Up @@ -135,10 +139,11 @@ type DataNodeServer struct {

type ExecNodeServer struct {
NodeServer
ExecAgent ExecAgent `yson:"exec_agent"`
DataNode DataNode `yson:"data_node"`
TabletNode TabletNode `yson:"tablet_node"`
CachingObjectService Cache `yson:"caching_object_service"`
JobResourceManager JobResourceManager `yson:"job_resource_manager"`
ExecAgent ExecAgent `yson:"exec_agent"`
DataNode DataNode `yson:"data_node"`
TabletNode TabletNode `yson:"tablet_node"`
CachingObjectService Cache `yson:"caching_object_service"`
}

type TabletNodeServer struct {
Expand Down Expand Up @@ -349,7 +354,7 @@ func getExecNodeServerCarcass(spec *ytv1.ExecNodesSpec, usePorto bool) (ExecNode

if c.ResourceLimits.TotalCpu != nil {
// Dummy heuristic.
c.ExecAgent.JobController.ResourceLimits.UserSlots = int(5 * *c.ResourceLimits.TotalCpu)
c.ExecAgent.JobController.ResourceLimitsLegacy.UserSlots = int(5 * *c.ResourceLimits.TotalCpu)
}

c.ExecAgent.SlotManager.JobEnvironment.StartUID = consts.StartUID
Expand Down Expand Up @@ -377,6 +382,7 @@ func getExecNodeServerCarcass(spec *ytv1.ExecNodesSpec, usePorto bool) (ExecNode
jobProxyLoggingBuilder.logging.FlushPeriod = 3000
c.ExecAgent.JobProxyLoggingLegacy = jobProxyLoggingBuilder.logging
c.ExecAgent.JobProxy.JobProxyLogging = jobProxyLoggingBuilder.logging
c.JobResourceManager.ResourceLimits = c.ExecAgent.JobController.ResourceLimitsLegacy

return c, nil
}
Expand Down

0 comments on commit cf8a4c5

Please sign in to comment.