Permalink
Show file tree
Hide file tree
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Worker level task metrics (#12446)
* * fix metric name inconsistency * * add task slot metrics for middle managers * * add new WorkerTaskCountStatsMonitor to report task count metrics from worker * * more stuff * * remove unused variable * * more stuff * * add javadocs * * fix checkstyle * * fix hadoop test failure * * cleanup * * add more code coverage in tests * * fix test failure * * add docs * * increase code coverage * * fix spelling * * fix failing tests * * remove dead code * * fix spelling
- Loading branch information
Showing
10 changed files
with
454 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| @@ -0,0 +1,79 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.druid.server.metrics; | ||
|
|
||
| import com.google.inject.Inject; | ||
| import com.google.inject.Injector; | ||
| import org.apache.druid.discovery.NodeRole; | ||
| import org.apache.druid.guice.annotations.Self; | ||
| import org.apache.druid.java.util.emitter.service.ServiceEmitter; | ||
| import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; | ||
| import org.apache.druid.java.util.metrics.AbstractMonitor; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| public class WorkerTaskCountStatsMonitor extends AbstractMonitor | ||
| { | ||
| private final WorkerTaskCountStatsProvider statsProvider; | ||
| private final String workerCategory; | ||
| private final String workerVersion; | ||
| private final boolean isMiddleManager; | ||
|
|
||
| @Inject | ||
| public WorkerTaskCountStatsMonitor( | ||
| Injector injector, | ||
| @Self Set<NodeRole> nodeRoles | ||
| ) | ||
| { | ||
| this.isMiddleManager = nodeRoles.contains(NodeRole.MIDDLE_MANAGER); | ||
| if (isMiddleManager) { | ||
| this.statsProvider = injector.getInstance(WorkerTaskCountStatsProvider.class); | ||
| this.workerCategory = statsProvider.getWorkerCategory(); | ||
| this.workerVersion = statsProvider.getWorkerVersion(); | ||
| } else { | ||
| this.statsProvider = null; | ||
| this.workerCategory = null; | ||
| this.workerVersion = null; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public boolean doMonitor(ServiceEmitter emitter) | ||
| { | ||
| if (isMiddleManager) { | ||
| emit(emitter, "worker/task/failed/count", statsProvider.getWorkerFailedTaskCount()); | ||
| emit(emitter, "worker/task/success/count", statsProvider.getWorkerSuccessfulTaskCount()); | ||
| emit(emitter, "worker/taskSlot/idle/count", statsProvider.getWorkerIdleTaskSlotCount()); | ||
| emit(emitter, "worker/taskSlot/total/count", statsProvider.getWorkerTotalTaskSlotCount()); | ||
| emit(emitter, "worker/taskSlot/used/count", statsProvider.getWorkerUsedTaskSlotCount()); | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| private void emit(ServiceEmitter emitter, String metricName, Long value) | ||
| { | ||
| if (value != null) { | ||
| final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); | ||
| builder.setDimension("category", workerCategory); | ||
| builder.setDimension("version", workerVersion); | ||
| emitter.emit(builder.build(metricName, value)); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.apache.druid.server.metrics; | ||
|
|
||
| /** | ||
| * Proides task / task count status at the level of individual worker nodes. These merics | ||
| * are repoerted by workers, like middle-managers. | ||
| */ | ||
| public interface WorkerTaskCountStatsProvider | ||
| { | ||
| /** | ||
| * The number of failed tasks run on worker during emission period. | ||
| */ | ||
| Long getWorkerFailedTaskCount(); | ||
|
|
||
| /** | ||
| * The number of successful tasks run on worker during emission period. | ||
| */ | ||
| Long getWorkerSuccessfulTaskCount(); | ||
|
|
||
| /** | ||
| * The number of idle task slots on worker. | ||
| */ | ||
| Long getWorkerIdleTaskSlotCount(); | ||
|
|
||
| /** | ||
| * The number of total task slots on worker. | ||
| */ | ||
| Long getWorkerTotalTaskSlotCount(); | ||
|
|
||
| /** | ||
| * The number of used task slots on worker. | ||
| */ | ||
| Long getWorkerUsedTaskSlotCount(); | ||
|
|
||
|
|
||
| /** | ||
| * The worker category. | ||
| */ | ||
| String getWorkerCategory(); | ||
|
|
||
| /** | ||
| * The worker version. | ||
| */ | ||
| String getWorkerVersion(); | ||
| } |
Oops, something went wrong.