From 2188a07e5bea1da26bf679ca0ece26ab596d3438 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Thu, 19 Mar 2015 11:49:07 -0700 Subject: [PATCH] YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee --- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 + .../api/protocolrecords/AllocateResponse.java | 16 +- .../TimelineWriteResponse.java | 20 +- .../hadoop/yarn/conf/YarnConfiguration.java | 20 +- .../src/main/proto/yarn_service_protos.proto | 2 +- .../pom.xml | 9 + .../distributedshell/ApplicationMaster.java | 52 ++--- .../applications/distributedshell/Client.java | 8 +- .../TestDistributedShell.java | 6 +- .../hadoop/yarn/client/api/AMRMClient.java | 6 +- .../client/api/async/AMRMClientAsync.java | 4 +- .../api/async/impl/AMRMClientAsyncImpl.java | 20 +- .../impl/pb/AllocateResponsePBImpl.java | 16 +- .../yarn/client/api/TimelineClient.java | 2 +- .../client/api/impl/TimelineClientImpl.java | 25 ++- .../src/main/resources/yarn-default.xml | 14 +- .../hadoop/yarn/TestContainerLaunchRPC.java | 2 +- .../hadoop-yarn-server-common/pom.xml | 2 +- ...java => CollectorNodemanagerProtocol.java} | 37 ++-- ...va => CollectorNodemanagerProtocolPB.java} | 8 +- ...ectorNodemanagerProtocolPBClientImpl.java} | 50 ++--- ...ctorNodemanagerProtocolPBServiceImpl.java} | 38 ++-- .../protocolrecords/NodeHeartbeatRequest.java | 13 +- .../NodeHeartbeatResponse.java | 8 +- ...ava => ReportNewCollectorInfoRequest.java} | 42 ++-- ...va => ReportNewCollectorInfoResponse.java} | 8 +- .../impl/pb/NodeHeartbeatRequestPBImpl.java | 58 +++--- .../impl/pb/NodeHeartbeatResponsePBImpl.java | 56 +++--- ...ReportNewAggregatorsInfoRequestPBImpl.java | 142 ------------- .../ReportNewCollectorInfoRequestPBImpl.java | 142 +++++++++++++ ...ReportNewCollectorInfoResponsePBImpl.java} | 30 +-- .../server/api/records/AppAggregatorsMap.java | 33 ---- .../server/api/records/AppCollectorsMap.java | 46 +++++ ...BImpl.java => AppCollectorsMapPBImpl.java} | 76 +++---- ...to => collectornodemanager_protocol.proto} | 6 +- .../yarn_server_common_service_protos.proto | 18 +- .../java/org/apache/hadoop/yarn/TestRPC.java | 116 +++++------ .../hadoop/yarn/TestYarnServerApiClasses.java | 24 +-- .../yarn/server/nodemanager/Context.java | 14 +- .../yarn/server/nodemanager/NodeManager.java | 56 +++--- .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../NMCollectorService.java} | 73 ++++--- .../application/ApplicationImpl.java | 9 +- .../amrmproxy/BaseAMRMProxyTest.java | 4 +- .../ApplicationMasterService.java | 12 +- .../ResourceTrackerService.java | 72 +++---- .../server/resourcemanager/rmapp/RMApp.java | 22 +-- ...nt.java => RMAppCollectorUpdateEvent.java} | 21 +- .../resourcemanager/rmapp/RMAppEventType.java | 4 +- .../resourcemanager/rmapp/RMAppImpl.java | 60 +++--- .../applicationsmanager/MockAsm.java | 6 +- .../resourcemanager/rmapp/MockRMApp.java | 8 +- .../hadoop-yarn-server-tests/pom.xml | 5 + .../TestTimelineServiceClientIntegration.java | 52 ++++- .../pom.xml | 10 + .../AppLevelTimelineCollector.java} | 8 +- .../PerNodeTimelineCollectorsAuxService.java} | 87 ++++---- .../TimelineCollector.java} | 12 +- .../TimelineCollectorManager.java} | 187 +++++++++--------- .../TimelineCollectorWebService.java} | 41 ++-- .../storage/FileSystemTimelineWriterImpl.java | 6 +- .../TestAppLevelTimelineCollector.java} | 4 +- ...tPerNodeTimelineCollectorsAuxService.java} | 48 +++-- .../TestTimelineCollectorManager.java} | 45 +++-- .../TestFileSystemTimelineWriterImpl.java | 43 ++-- 65 files changed, 1105 insertions(+), 990 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/{AggregatorNodemanagerProtocol.java => CollectorNodemanagerProtocol.java} (61%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/{AggregatorNodemanagerProtocolPB.java => CollectorNodemanagerProtocolPB.java} (77%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/{AggregatorNodemanagerProtocolPBClientImpl.java => CollectorNodemanagerProtocolPBClientImpl.java} (70%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/{AggregatorNodemanagerProtocolPBServiceImpl.java => CollectorNodemanagerProtocolPBServiceImpl.java} (57%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/{ReportNewAggregatorsInfoRequest.java => ReportNewCollectorInfoRequest.java} (55%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/{ReportNewAggregatorsInfoResponse.java => ReportNewCollectorInfoResponse.java} (81%) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/{ReportNewAggregatorsInfoResponsePBImpl.java => ReportNewCollectorInfoResponsePBImpl.java} (74%) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/{AppAggregatorsMapPBImpl.java => AppCollectorsMapPBImpl.java} (71%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/{aggregatornodemanager_protocol.proto => collectornodemanager_protocol.proto} (81%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/{aggregatormanager/NMAggregatorService.java => collectormanager/NMCollectorService.java} (54%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/{RMAppAggregatorUpdateEvent.java => RMAppCollectorUpdateEvent.java} (71%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/AppLevelTimelineAggregator.java => collector/AppLevelTimelineCollector.java} (85%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/PerNodeTimelineAggregatorsAuxService.java => collector/PerNodeTimelineCollectorsAuxService.java} (66%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/TimelineAggregator.java => collector/TimelineCollector.java} (94%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/TimelineAggregatorsCollection.java => collector/TimelineCollectorManager.java} (58%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/TimelineAggregatorWebService.java => collector/TimelineCollectorWebService.java} (82%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/TestAppLevelTimelineAggregator.java => collector/TestAppLevelTimelineCollector.java} (87%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/TestPerNodeTimelineAggregatorsAuxService.java => collector/TestPerNodeTimelineCollectorsAuxService.java} (73%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/{aggregator/TestTimelineAggregatorsCollection.java => collector/TestTimelineCollectorManager.java} (65%) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 41aef33f89b1b..e4b445d6b702e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -47,6 +47,10 @@ commons-logging commons-logging + + javax.xml.bind + jaxb-api + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java index c61b8106af00c..2567a51f87afc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java @@ -127,7 +127,7 @@ public static AllocateResponse newInstance(int responseId, response.setAMRMToken(amRMToken); return response; } - + @Public @Unstable public static AllocateResponse newInstance(int responseId, @@ -137,13 +137,13 @@ public static AllocateResponse newInstance(int responseId, PreemptionMessage preempt, List nmTokens, Token amRMToken, List increasedContainers, List decreasedContainers, - String aggregatorAddr) { + String collectorAddr) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens, increasedContainers, decreasedContainers); response.setAMRMToken(amRMToken); - response.setAggregatorAddr(aggregatorAddr); + response.setCollectorAddr(collectorAddr); return response; } @@ -349,16 +349,16 @@ public abstract void setDecreasedContainers( public abstract void setApplicationPriority(Priority priority); /** - * The address of aggregator that belong to this app + * The address of collector that belong to this app * - * @return The address of aggregator that belong to this attempt + * @return The address of collector that belong to this attempt */ @Public @Unstable - public abstract String getAggregatorAddr(); - + public abstract String getCollectorAddr(); + @Private @Unstable - public abstract void setAggregatorAddr(String aggregatorAddr); + public abstract void setCollectorAddr(String collectorAddr); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java index 82ecdbd99e5c8..4739d8fcdb172 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java @@ -47,7 +47,7 @@ public TimelineWriteResponse() { /** * Get a list of {@link TimelineWriteError} instances - * + * * @return a list of {@link TimelineWriteError} instances */ @XmlElement(name = "errors") @@ -57,7 +57,7 @@ public List getErrors() { /** * Add a single {@link TimelineWriteError} instance into the existing list - * + * * @param error * a single {@link TimelineWriteError} instance */ @@ -67,7 +67,7 @@ public void addError(TimelineWriteError error) { /** * Add a list of {@link TimelineWriteError} instances into the existing list - * + * * @param errors * a list of {@link TimelineWriteError} instances */ @@ -77,7 +77,7 @@ public void addErrors(List errors) { /** * Set the list to the given list of {@link TimelineWriteError} instances - * + * * @param errors * a list of {@link TimelineWriteError} instances */ @@ -107,7 +107,7 @@ public static class TimelineWriteError { /** * Get the entity Id - * + * * @return the entity Id */ @XmlElement(name = "entity") @@ -117,7 +117,7 @@ public String getEntityId() { /** * Set the entity Id - * + * * @param entityId * the entity Id */ @@ -127,7 +127,7 @@ public void setEntityId(String entityId) { /** * Get the entity type - * + * * @return the entity type */ @XmlElement(name = "entitytype") @@ -137,7 +137,7 @@ public String getEntityType() { /** * Set the entity type - * + * * @param entityType * the entity type */ @@ -147,7 +147,7 @@ public void setEntityType(String entityType) { /** * Get the error code - * + * * @return an error code */ @XmlElement(name = "errorcode") @@ -157,7 +157,7 @@ public int getErrorCode() { /** * Set the error code to the given error code - * + * * @param errorCode * an error code */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fc7e9152be1ca..6de2ce825e076 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -814,10 +814,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; /** Number of threads container manager uses.*/ - public static final String NM_AGGREGATOR_SERVICE_THREAD_COUNT = - NM_PREFIX + "aggregator-service.thread-count"; - public static final int DEFAULT_NM_AGGREGATOR_SERVICE_THREAD_COUNT = 5; - + public static final String NM_COLLECTOR_SERVICE_THREAD_COUNT = + NM_PREFIX + "collector-service.thread-count"; + public static final int DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT = 5; + /** Number of threads used in cleanup.*/ public static final String NM_DELETE_THREAD_COUNT = NM_PREFIX + "delete.thread-count"; @@ -845,13 +845,13 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT; - /** Address where the aggregator service IPC is.*/ - public static final String NM_AGGREGATOR_SERVICE_ADDRESS = - NM_PREFIX + "aggregator-service.address"; - public static final int DEFAULT_NM_AGGREGATOR_SERVICE_PORT = 8048; - public static final String DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS = + /** Address where the collector service IPC is.*/ + public static final String NM_COLLECTOR_SERVICE_ADDRESS = + NM_PREFIX + "collector-service.address"; + public static final int DEFAULT_NM_COLLECTOR_SERVICE_PORT = 8048; + public static final String DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS = "0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT; - + /** Interval in between cache cleanups.*/ public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = NM_PREFIX + "localizer.cache.cleanup.interval-ms"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 6b2d878574a6e..dddb0f0f1ede6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -89,7 +89,7 @@ message AllocateResponseProto { repeated ContainerProto decreased_containers = 11; optional hadoop.common.TokenProto am_rm_token = 12; optional PriorityProto application_priority = 13; - optional string aggregator_addr = 14; + optional string collector_addr = 14; } enum SchedulerResourceTypes { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index fc41ff8e945e1..c0e21773e5616 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -70,6 +70,15 @@ commons-io commons-io + + org.apache.hadoop + hadoop-yarn-server-applicationhistoryservice + + + org.apache.hadoop + hadoop-yarn-server-timelineservice + test + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 4833c5cc52c6e..46724d693bcf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -40,10 +40,10 @@ import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -222,12 +222,12 @@ public static enum DSEntity { private int appMasterRpcPort = -1; // Tracking url to which app master publishes info for clients to monitor private String appMasterTrackingUrl = ""; - + private boolean newTimelineService = false; - + // For posting entities in new timeline service in a non-blocking way // TODO replace with event loop in TimelineClient. - private static ExecutorService threadPool = + private static ExecutorService threadPool = Executors.newCachedThreadPool( new ThreadFactoryBuilder().setNameFormat("TimelineService #%d") .build()); @@ -331,9 +331,9 @@ public static void main(String[] args) { } appMaster.run(); result = appMaster.finish(); - + threadPool.shutdown(); - + while (!threadPool.isTerminated()) { // wait for all posting thread to finish try { if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) { @@ -427,7 +427,7 @@ public boolean init(String[] args) throws ParseException, IOException { opts.addOption("container_retry_interval", true, "Interval between each retry, unit is milliseconds"); opts.addOption("debug", false, "Dump out debug information"); - opts.addOption("timeline_service_version", true, + opts.addOption("timeline_service_version", true, "Version for timeline service"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -583,7 +583,7 @@ public boolean init(String[] args) throws ParseException, IOException { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { if (cliParser.hasOption("timeline_service_version")) { - String timelineServiceVersion = + String timelineServiceVersion = cliParser.getOptionValue("timeline_service_version", "v1"); if (timelineServiceVersion.trim().equalsIgnoreCase("v1")) { newTimelineService = false; @@ -655,7 +655,7 @@ public void run() throws YarnException, IOException, InterruptedException { amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); - + containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); @@ -840,7 +840,7 @@ protected boolean finish() { if(timelineClient != null) { timelineClient.stop(); } - + return success; } @@ -1392,11 +1392,11 @@ Thread createLaunchContainerThread(Container allocatedContainer, } private static void publishContainerStartEventOnNewTimelineService( - final TimelineClient timelineClient, final Container container, + final TimelineClient timelineClient, final Container container, final String domainId, final UserGroupInformation ugi) { Runnable publishWrapper = new Runnable() { public void run() { - publishContainerStartEventOnNewTimelineServiceBase(timelineClient, + publishContainerStartEventOnNewTimelineServiceBase(timelineClient, container, domainId, ugi); } }; @@ -1406,14 +1406,14 @@ public void run() { private static void publishContainerStartEventOnNewTimelineServiceBase( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { - final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.setId(container.getId().toString()); entity.setType(DSEntity.DS_CONTAINER.toString()); //entity.setDomainId(domainId); entity.addInfo("user", ugi.getShortUserName()); - - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = + + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setId(DSEvent.DS_CONTAINER_START.toString()); @@ -1435,29 +1435,29 @@ public TimelinePutResponse run() throws Exception { e instanceof UndeclaredThrowableException ? e.getCause() : e); } } - + private static void publishContainerEndEventOnNewTimelineService( final TimelineClient timelineClient, final ContainerStatus container, final String domainId, final UserGroupInformation ugi) { Runnable publishWrapper = new Runnable() { public void run() { - publishContainerEndEventOnNewTimelineServiceBase(timelineClient, + publishContainerEndEventOnNewTimelineServiceBase(timelineClient, container, domainId, ugi); } }; threadPool.execute(publishWrapper); } - + private static void publishContainerEndEventOnNewTimelineServiceBase( final TimelineClient timelineClient, final ContainerStatus container, final String domainId, final UserGroupInformation ugi) { - final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.setId(container.getContainerId().toString()); entity.setType(DSEntity.DS_CONTAINER.toString()); //entity.setDomainId(domainId); entity.addInfo("user", ugi.getShortUserName()); - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setId(DSEvent.DS_CONTAINER_END.toString()); @@ -1482,28 +1482,28 @@ public TimelinePutResponse run() throws Exception { private static void publishApplicationAttemptEventOnNewTimelineService( final TimelineClient timelineClient, final String appAttemptId, - final DSEvent appEvent, final String domainId, + final DSEvent appEvent, final String domainId, final UserGroupInformation ugi) { - + Runnable publishWrapper = new Runnable() { public void run() { - publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient, + publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient, appAttemptId, appEvent, domainId, ugi); } }; threadPool.execute(publishWrapper); } - + private static void publishApplicationAttemptEventOnNewTimelineServiceBase( final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) { - final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.setId(appAttemptId); entity.setType(DSEntity.DS_APP_ATTEMPT.toString()); //entity.setDomainId(domainId); entity.addInfo("user", ugi.getShortUserName()); - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); event.setId(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index af424241c1e6f..164c6a5d800a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -189,7 +189,7 @@ public class Client { // Command line options private Options opts; - + private String timelineServiceVersion; private static final String shellCommandPath = "shellCommands"; @@ -373,11 +373,11 @@ public boolean init(String[] args) throws ParseException { throw new IllegalArgumentException("Invalid virtual cores specified for application master, exiting." + " Specified virtual cores=" + amVCores); } - + if (cliParser.hasOption("timeline_service_version")) { - timelineServiceVersion = + timelineServiceVersion = cliParser.getOptionValue("timeline_service_version", "v1"); - if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") || + if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") || timelineServiceVersion.trim().equalsIgnoreCase("v2"))) { throw new IllegalArgumentException( "timeline_service_version is not set properly, should be 'v1' or 'v2'"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index aa60808929f26..d344df9d01996 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -79,7 +79,7 @@ import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.server.timelineservice.aggregator.PerNodeTimelineAggregatorsAuxService; +import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.junit.After; import org.junit.Assert; @@ -100,7 +100,7 @@ public class TestDistributedShell { protected YarnConfiguration conf = null; private static final int NUM_NMS = 1; private static final float DEFAULT_TIMELINE_VERSION = 1.0f; - private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_aggregator"; + private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector"; protected final static String APPMASTER_JAR = JarFinder.getJar(ApplicationMaster.class); @@ -161,7 +161,7 @@ private void setupInternal(int numNodeManager, float timelineVersion) // enable aux-service based timeline aggregators conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME); conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME - + ".class", PerNodeTimelineAggregatorsAuxService.class.getName()); + + ".class", PerNodeTimelineCollectorsAuxService.class.getName()); } else { Assert.fail("Wrong timeline version number: " + timelineVersion); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index e401e389981b0..7f64dd52f9083 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -51,7 +51,7 @@ public abstract class AMRMClient extends AbstractService { private static final Log LOG = LogFactory.getLog(AMRMClient.class); - + private TimelineClient timelineClient; /** @@ -468,7 +468,7 @@ public NMTokenCache getNMTokenCache() { public void registerTimelineClient(TimelineClient timelineClient) { this.timelineClient = timelineClient; } - + /** * Get registered timeline client. * @return @@ -476,7 +476,7 @@ public void registerTimelineClient(TimelineClient timelineClient) { public TimelineClient getRegisteredTimeineClient() { return this.timelineClient; } - + /** * Wait for check to return true for each 1000 ms. * See also {@link #waitFor(com.google.common.base.Supplier, int)} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java index 9fb1184eaccf8..7cdda1be251a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java @@ -293,7 +293,7 @@ public abstract void requestContainerResourceChange( * @return Current number of nodes in the cluster */ public abstract int getClusterNodeCount(); - + /** * Register TimelineClient to AMRMClient. * @param timelineClient @@ -301,7 +301,7 @@ public abstract void requestContainerResourceChange( public void registerTimelineClient(TimelineClient timelineClient) { client.registerTimelineClient(timelineClient); } - + /** * Get registered timeline client. * @return diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index f4a5d4d481782..212f7213236e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -67,8 +67,8 @@ public class AMRMClientAsyncImpl private volatile boolean keepRunning; private volatile float progress; - private volatile String aggregatorAddr; - + private volatile String collectorAddr; + private volatile Throwable savedException; /** @@ -354,15 +354,15 @@ public void run() { if (!allocated.isEmpty()) { handler.onContainersAllocated(allocated); } - - String aggregatorAddress = response.getAggregatorAddr(); + + String collectorAddress = response.getCollectorAddr(); TimelineClient timelineClient = client.getRegisteredTimeineClient(); - if (timelineClient != null && aggregatorAddress != null - && !aggregatorAddress.isEmpty()) { - if (aggregatorAddr == null || - !aggregatorAddr.equals(aggregatorAddress)) { - aggregatorAddr = aggregatorAddress; - timelineClient.setTimelineServiceAddress(aggregatorAddress); + if (timelineClient != null && collectorAddress != null + && !collectorAddress.isEmpty()) { + if (collectorAddr == null || + !collectorAddr.equals(collectorAddress)) { + collectorAddr = collectorAddress; + timelineClient.setTimelineServiceAddress(collectorAddress); } } progress = handler.getProgress(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java index ca060bbb3b1a5..5ef974ef6c8de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java @@ -384,22 +384,22 @@ public synchronized void setAMRMToken(Token amRMToken) { } this.amrmToken = amRMToken; } - + @Override - public String getAggregatorAddr() { + public String getCollectorAddr() { AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; - return p.getAggregatorAddr(); + return p.getCollectorAddr(); } - + @Override - public void setAggregatorAddr(String aggregatorAddr) { + public void setCollectorAddr(String collectorAddr) { maybeInitBuilder(); - if (aggregatorAddr == null) { - builder.clearAggregatorAddr(); + if (collectorAddr == null) { + builder.clearCollectorAddr(); return; } - builder.setAggregatorAddr(aggregatorAddr); + builder.setCollectorAddr(collectorAddr); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index b6f816c1103df..d629bb74d1715 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -242,5 +242,5 @@ public abstract void putEntitiesAsync( * the timeline service address */ public abstract void setTimelineServiceAddress(String address); - + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index a9d76a8fffe4b..d4335009b2be1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -119,12 +119,11 @@ public class TimelineClientImpl extends TimelineClient { private TimelineWriter timelineWriter; private volatile String timelineServiceAddress; - + // Retry parameters for identifying new timeline service // TODO consider to merge with connection retry private int maxServiceRetries; private long serviceRetryInterval; - private boolean timelineServiceV2 = false; @Private @@ -330,7 +329,7 @@ protected void serviceInit(Configuration conf) throws Exception { conf.getFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_VERSION); LOG.info("Timeline service address: " + getTimelineServiceAddress()); - } + } super.serviceInit(conf); } @@ -410,16 +409,16 @@ public void putDomain(TimelineDomain domain) throws IOException, YarnException { timelineWriter.putDomain(domain); } - + // Used for new timeline service only @Private - public void putObjects(String path, MultivaluedMap params, + public void putObjects(String path, MultivaluedMap params, Object obj) throws IOException, YarnException { - - // timelineServiceAddress could haven't be initialized yet + + // timelineServiceAddress could haven't be initialized yet // or stale (only for new timeline service) int retries = pollTimelineServiceAddress(this.maxServiceRetries); - + // timelineServiceAddress could be stale, add retry logic here. boolean needRetry = true; while (needRetry) { @@ -436,13 +435,13 @@ public void putObjects(String path, MultivaluedMap params, } } } - + /** * Check if reaching to maximum of retries. * @param retries * @param e */ - private void checkRetryWithSleep(int retries, Exception e) throws + private void checkRetryWithSleep(int retries, Exception e) throws YarnException, IOException { if (retries > 0) { try { @@ -452,8 +451,8 @@ private void checkRetryWithSleep(int retries, Exception e) throws } } else { LOG.error( - "TimelineClient has reached to max retry times :" + - this.maxServiceRetries + " for service address: " + + "TimelineClient has reached to max retry times :" + + this.maxServiceRetries + " for service address: " + timelineServiceAddress); if (e instanceof YarnException) { throw (YarnException)e; @@ -499,7 +498,7 @@ private void putObjects( public void setTimelineServiceAddress(String address) { this.timelineServiceAddress = address; } - + private String getTimelineServiceAddress() { return this.timelineServiceAddress; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a477f1855480b..2781da0c95ed8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -965,10 +965,10 @@ yarn.nodemanager.container-manager.thread-count 20 - + - Number of threads aggregator service uses. - yarn.nodemanager.aggregator-service.thread-count + Number of threads collector service uses. + yarn.nodemanager.collector-service.thread-count 5 @@ -1046,11 +1046,11 @@ yarn.nodemanager.localizer.address ${yarn.nodemanager.hostname}:8040 - - + + - Address where the aggregator service IPC is. - yarn.nodemanager.aggregator-service.address + Address where the collector service IPC is. + yarn.nodemanager.collector-service.address ${yarn.nodemanager.hostname}:8048 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index 18719e6e79fda..31bea2d81d754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -135,7 +135,7 @@ private void testRPCTimeout(String rpcClass) throws Exception { Assert.fail("timeout exception should have occurred!"); } - + public static Token newContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index b8ef142c6fffd..95bb89b04f5bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -150,7 +150,7 @@ yarn_server_common_service_protos.proto ResourceTracker.proto SCMUploader.proto - aggregatornodemanager_protocol.proto + collectornodemanager_protocol.proto ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java similarity index 61% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java index 53bdb4e8db4a1..26c121a71b06c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java @@ -21,36 +21,37 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; /** - *

The protocol between an TimelineAggregatorsCollection and a - * NodeManager to report a new application aggregator get launched. + *

The protocol between an TimelineCollectorManager and a + * NodeManager to report a new application collector get launched. *

- * + * */ @Private -public interface AggregatorNodemanagerProtocol { +public interface CollectorNodemanagerProtocol { /** - * + * *

- * The TimelineAggregatorsCollection provides a list of mapping - * between application and aggregator's address in - * {@link ReportNewAggregatorsInfoRequest} to a NodeManager to - * register aggregator's info, include: applicationId and REST URI to - * access aggregator. NodeManager will add them into registered aggregators + * The TimelineCollectorManager provides a list of mapping + * between application and collector's address in + * {@link ReportNewCollectorInfoRequest} to a NodeManager to + * register collector's info, include: applicationId and REST URI to + * access collector. NodeManager will add them into registered collectors * and register them into ResourceManager afterwards. *

- * - * @param request the request of registering a new aggregator or a list of aggregators - * @return + * + * @param request the request of registering a new collector or a list of + * collectors + * @return * @throws YarnException * @throws IOException */ - ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( - ReportNewAggregatorsInfoRequest request) + ReportNewCollectorInfoResponse reportNewCollectorInfo( + ReportNewCollectorInfoRequest request) throws YarnException, IOException; - + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java similarity index 77% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java index 4df80a5a2d06f..655e9890a216a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java @@ -20,14 +20,14 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.ipc.ProtocolInfo; -import org.apache.hadoop.yarn.proto.AggregatorNodemanagerProtocol.AggregatorNodemanagerProtocolService; +import org.apache.hadoop.yarn.proto.CollectorNodemanagerProtocol.CollectorNodemanagerProtocolService; @Private @Unstable @ProtocolInfo( - protocolName = "org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB", + protocolName = "org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB", protocolVersion = 1) -public interface AggregatorNodemanagerProtocolPB extends - AggregatorNodemanagerProtocolService.BlockingInterface { +public interface CollectorNodemanagerProtocolPB extends + CollectorNodemanagerProtocolService.BlockingInterface { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java similarity index 70% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java index 6e777e7898082..276a540c77a19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java @@ -30,18 +30,18 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoResponsePBImpl; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl; import com.google.protobuf.ServiceException; -public class AggregatorNodemanagerProtocolPBClientImpl implements - AggregatorNodemanagerProtocol, Closeable { +public class CollectorNodemanagerProtocolPBClientImpl implements + CollectorNodemanagerProtocol, Closeable { // Not a documented config. Only used for tests internally static final String NM_COMMAND_TIMEOUT = YarnConfiguration.YARN_PREFIX @@ -51,39 +51,39 @@ public class AggregatorNodemanagerProtocolPBClientImpl implements * Maximum of 1 minute timeout for a Node to react to the command */ static final int DEFAULT_COMMAND_TIMEOUT = 60000; - - private AggregatorNodemanagerProtocolPB proxy; - + + private CollectorNodemanagerProtocolPB proxy; + @Private - public AggregatorNodemanagerProtocolPBClientImpl(long clientVersion, + public CollectorNodemanagerProtocolPBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { - RPC.setProtocolEngine(conf, AggregatorNodemanagerProtocolPB.class, + RPC.setProtocolEngine(conf, CollectorNodemanagerProtocolPB.class, ProtobufRpcEngine.class); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT); proxy = - (AggregatorNodemanagerProtocolPB) RPC.getProxy( - AggregatorNodemanagerProtocolPB.class, + (CollectorNodemanagerProtocolPB) RPC.getProxy( + CollectorNodemanagerProtocolPB.class, clientVersion, addr, ugi, conf, NetUtils.getDefaultSocketFactory(conf), expireIntvl); } - + @Override - public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( - ReportNewAggregatorsInfoRequest request) throws YarnException, IOException { - - ReportNewAggregatorsInfoRequestProto requestProto = - ((ReportNewAggregatorsInfoRequestPBImpl) request).getProto(); + public ReportNewCollectorInfoResponse reportNewCollectorInfo( + ReportNewCollectorInfoRequest request) throws YarnException, IOException { + + ReportNewCollectorInfoRequestProto requestProto = + ((ReportNewCollectorInfoRequestPBImpl) request).getProto(); try { - return new ReportNewAggregatorsInfoResponsePBImpl( - proxy.reportNewAggregatorInfo(null, requestProto)); + return new ReportNewCollectorInfoResponsePBImpl( + proxy.reportNewCollectorInfo(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } } - + @Override public void close() { if (this.proxy != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java similarity index 57% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java index 87bce1685715e..3f42732bc35fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java @@ -19,38 +19,36 @@ import java.io.IOException; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoResponseProto; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoResponsePBImpl; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; -public class AggregatorNodemanagerProtocolPBServiceImpl implements - AggregatorNodemanagerProtocolPB { +public class CollectorNodemanagerProtocolPBServiceImpl implements + CollectorNodemanagerProtocolPB { - private AggregatorNodemanagerProtocol real; - - public AggregatorNodemanagerProtocolPBServiceImpl(AggregatorNodemanagerProtocol impl) { + private CollectorNodemanagerProtocol real; + + public CollectorNodemanagerProtocolPBServiceImpl(CollectorNodemanagerProtocol impl) { this.real = impl; } @Override - public ReportNewAggregatorsInfoResponseProto reportNewAggregatorInfo( - RpcController arg0, ReportNewAggregatorsInfoRequestProto proto) + public ReportNewCollectorInfoResponseProto reportNewCollectorInfo( + RpcController arg0, ReportNewCollectorInfoRequestProto proto) throws ServiceException { - ReportNewAggregatorsInfoRequestPBImpl request = - new ReportNewAggregatorsInfoRequestPBImpl(proto); + ReportNewCollectorInfoRequestPBImpl request = + new ReportNewCollectorInfoRequestPBImpl(proto); try { - ReportNewAggregatorsInfoResponse response = real.reportNewAggregatorInfo(request); - return ((ReportNewAggregatorsInfoResponsePBImpl)response).getProto(); + ReportNewCollectorInfoResponse response = real.reportNewCollectorInfo(request); + return ((ReportNewCollectorInfoResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java index 09cb9e3dd0b7a..c795e556c46e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java @@ -43,11 +43,11 @@ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus, nodeHeartbeatRequest.setNodeLabels(nodeLabels); return nodeHeartbeatRequest; } - + public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus, MasterKey lastKnownContainerTokenMasterKey, MasterKey lastKnownNMTokenMasterKey, Set nodeLabels, - Map registeredAggregators) { + Map registeredCollectors) { NodeHeartbeatRequest nodeHeartbeatRequest = Records.newRecord(NodeHeartbeatRequest.class); nodeHeartbeatRequest.setNodeStatus(nodeStatus); @@ -56,7 +56,7 @@ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus, nodeHeartbeatRequest .setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey); nodeHeartbeatRequest.setNodeLabels(nodeLabels); - nodeHeartbeatRequest.setRegisteredAggregators(registeredAggregators); + nodeHeartbeatRequest.setRegisteredCollectors(registeredCollectors); return nodeHeartbeatRequest; } @@ -78,7 +78,8 @@ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus, public abstract void setLogAggregationReportsForApps( List logAggregationReportsForApps); - // This tells RM registered aggregators' address info on this node - public abstract Map getRegisteredAggregators(); - public abstract void setRegisteredAggregators(Map appAggregatorsMap); + // This tells RM registered collectors' address info on this node + public abstract Map getRegisteredCollectors(); + public abstract void setRegisteredCollectors(Map appCollectorsMap); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java index f5d0043a6a417..09cafaf0660ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java @@ -40,10 +40,10 @@ public interface NodeHeartbeatResponse { List getContainersToBeRemovedFromNM(); List getApplicationsToCleanup(); - - // This tells NM the aggregators' address info of related Apps - Map getAppAggregatorsMap(); - void setAppAggregatorsMap(Map appAggregatorsMap); + + // This tells NM the collectors' address info of related apps + Map getAppCollectorsMap(); + void setAppCollectorsMap(Map appCollectorsMap); void setResponseId(int responseId); void setNodeAction(NodeAction action); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java similarity index 55% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java index ae538a2edc41e..3498de90a3cbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java @@ -22,32 +22,32 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; +import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; import org.apache.hadoop.yarn.util.Records; @Private -public abstract class ReportNewAggregatorsInfoRequest { - - public static ReportNewAggregatorsInfoRequest newInstance( - List appAggregatorsList) { - ReportNewAggregatorsInfoRequest request = - Records.newRecord(ReportNewAggregatorsInfoRequest.class); - request.setAppAggregatorsList(appAggregatorsList); +public abstract class ReportNewCollectorInfoRequest { + + public static ReportNewCollectorInfoRequest newInstance( + List appCollectorsList) { + ReportNewCollectorInfoRequest request = + Records.newRecord(ReportNewCollectorInfoRequest.class); + request.setAppCollectorsList(appCollectorsList); return request; } - - public static ReportNewAggregatorsInfoRequest newInstance( - ApplicationId id, String aggregatorAddr) { - ReportNewAggregatorsInfoRequest request = - Records.newRecord(ReportNewAggregatorsInfoRequest.class); - request.setAppAggregatorsList( - Arrays.asList(AppAggregatorsMap.newInstance(id, aggregatorAddr))); + + public static ReportNewCollectorInfoRequest newInstance( + ApplicationId id, String collectorAddr) { + ReportNewCollectorInfoRequest request = + Records.newRecord(ReportNewCollectorInfoRequest.class); + request.setAppCollectorsList( + Arrays.asList(AppCollectorsMap.newInstance(id, collectorAddr))); return request; } - - public abstract List getAppAggregatorsList(); - - public abstract void setAppAggregatorsList( - List appAggregatorsList); - + + public abstract List getAppCollectorsList(); + + public abstract void setAppCollectorsList( + List appCollectorsList); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java similarity index 81% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java index 3b847d6dcbc92..4157c47af3ce4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java @@ -20,12 +20,12 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.util.Records; -public abstract class ReportNewAggregatorsInfoResponse { +public abstract class ReportNewCollectorInfoResponse { @Private - public static ReportNewAggregatorsInfoResponse newInstance() { - ReportNewAggregatorsInfoResponse response = - Records.newRecord(ReportNewAggregatorsInfoResponse.class); + public static ReportNewCollectorInfoResponse newInstance() { + ReportNewCollectorInfoResponse response = + Records.newRecord(ReportNewCollectorInfoResponse.class); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java index 02e0a3da76937..fa0cf5cd43e2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java @@ -34,8 +34,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregationReportProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto; @@ -58,7 +58,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { private Set labels = null; private List logAggregationReportsForApps = null; - Map registeredAggregators = null; + Map registeredCollectors = null; public NodeHeartbeatRequestPBImpl() { builder = NodeHeartbeatRequestProto.newBuilder(); @@ -114,8 +114,8 @@ private void mergeLocalToBuilder() { if (this.logAggregationReportsForApps != null) { addLogAggregationStatusForAppsToProto(); } - if (this.registeredAggregators != null) { - addRegisteredAggregatorsToProto(); + if (this.registeredCollectors != null) { + addRegisteredCollectorsToProto(); } } @@ -158,13 +158,13 @@ private LogAggregationReportProto convertToProtoFormat( return ((LogAggregationReportPBImpl) value).getProto(); } - private void addRegisteredAggregatorsToProto() { + private void addRegisteredCollectorsToProto() { maybeInitBuilder(); - builder.clearRegisteredAggregators(); - for (Map.Entry entry : registeredAggregators.entrySet()) { - builder.addRegisteredAggregators(AppAggregatorsMapProto.newBuilder() + builder.clearRegisteredCollectors(); + for (Map.Entry entry : registeredCollectors.entrySet()) { + builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder() .setAppId(convertToProtoFormat(entry.getKey())) - .setAppAggregatorAddr(entry.getValue())); + .setAppCollectorAddr(entry.getValue())); } } @@ -248,35 +248,35 @@ public void setLastKnownNMTokenMasterKey(MasterKey masterKey) { builder.clearLastKnownNmTokenMasterKey(); this.lastKnownNMTokenMasterKey = masterKey; } - + @Override - public Map getRegisteredAggregators() { - if (this.registeredAggregators != null) { - return this.registeredAggregators; + public Map getRegisteredCollectors() { + if (this.registeredCollectors != null) { + return this.registeredCollectors; } - initRegisteredAggregators(); - return registeredAggregators; + initRegisteredCollectors(); + return registeredCollectors; } - - private void initRegisteredAggregators() { + + private void initRegisteredCollectors() { NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getRegisteredAggregatorsList(); - this.registeredAggregators = new HashMap (); - for (AppAggregatorsMapProto c : list) { + List list = p.getRegisteredCollectorsList(); + this.registeredCollectors = new HashMap (); + for (AppCollectorsMapProto c : list) { ApplicationId appId = convertFromProtoFormat(c.getAppId()); - this.registeredAggregators.put(appId, c.getAppAggregatorAddr()); + this.registeredCollectors.put(appId, c.getAppCollectorAddr()); } } - + @Override - public void setRegisteredAggregators( - Map registeredAggregators) { - if (registeredAggregators == null || registeredAggregators.isEmpty()) { + public void setRegisteredCollectors( + Map registeredCollectors) { + if (registeredCollectors == null || registeredCollectors.isEmpty()) { return; } maybeInitBuilder(); - this.registeredAggregators = new HashMap(); - this.registeredAggregators.putAll(registeredAggregators); + this.registeredCollectors = new HashMap(); + this.registeredCollectors.putAll(registeredCollectors); } private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) { @@ -286,11 +286,11 @@ private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) { private NodeStatusProto convertToProtoFormat(NodeStatus t) { return ((NodeStatusPBImpl)t).getProto(); } - + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { return new ApplicationIdPBImpl(p); } - + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { return ((ApplicationIdPBImpl) t).getProto(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java index cb20615098060..356414fe12666 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java @@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.SystemCredentialsForAppsProto; @@ -69,7 +69,7 @@ public class NodeHeartbeatResponsePBImpl extends private List applicationsToCleanup = null; private Map systemCredentials = null; private Resource resource = null; - Map appAggregatorsMap = null; + Map appCollectorsMap = null; private MasterKey containerTokenMasterKey = null; private MasterKey nmTokenMasterKey = null; @@ -127,8 +127,8 @@ private void mergeLocalToBuilder() { if (this.resource != null) { builder.setResource(convertToProtoFormat(this.resource)); } - if (this.appAggregatorsMap != null) { - addAppAggregatorsMapToProto(); + if (this.appCollectorsMap != null) { + addAppCollectorsMapToProto(); } } @@ -142,14 +142,14 @@ private void addSystemCredentialsToProto() { entry.getValue().duplicate()))); } } - - private void addAppAggregatorsMapToProto() { + + private void addAppCollectorsMapToProto() { maybeInitBuilder(); - builder.clearAppAggregatorsMap(); - for (Map.Entry entry : appAggregatorsMap.entrySet()) { - builder.addAppAggregatorsMap(AppAggregatorsMapProto.newBuilder() + builder.clearAppCollectorsMap(); + for (Map.Entry entry : appCollectorsMap.entrySet()) { + builder.addAppCollectorsMap(AppCollectorsMapProto.newBuilder() .setAppId(convertToProtoFormat(entry.getKey())) - .setAppAggregatorAddr(entry.getValue())); + .setAppCollectorAddr(entry.getValue())); } } @@ -565,14 +565,14 @@ public Map getSystemCredentialsForApps() { initSystemCredentials(); return systemCredentials; } - + @Override - public Map getAppAggregatorsMap() { - if (this.appAggregatorsMap != null) { - return this.appAggregatorsMap; + public Map getAppCollectorsMap() { + if (this.appCollectorsMap != null) { + return this.appCollectorsMap; } - initAppAggregatorsMap(); - return appAggregatorsMap; + initAppCollectorsMap(); + return appCollectorsMap; } private void initSystemCredentials() { @@ -585,14 +585,14 @@ private void initSystemCredentials() { this.systemCredentials.put(appId, byteBuffer); } } - - private void initAppAggregatorsMap() { + + private void initAppCollectorsMap() { NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getAppAggregatorsMapList(); - this.appAggregatorsMap = new HashMap (); - for (AppAggregatorsMapProto c : list) { + List list = p.getAppCollectorsMapList(); + this.appCollectorsMap = new HashMap (); + for (AppCollectorsMapProto c : list) { ApplicationId appId = convertFromProtoFormat(c.getAppId()); - this.appAggregatorsMap.put(appId, c.getAppAggregatorAddr()); + this.appCollectorsMap.put(appId, c.getAppCollectorAddr()); } } @@ -606,16 +606,16 @@ public void setSystemCredentialsForApps( this.systemCredentials = new HashMap(); this.systemCredentials.putAll(systemCredentials); } - + @Override - public void setAppAggregatorsMap( - Map appAggregatorsMap) { - if (appAggregatorsMap == null || appAggregatorsMap.isEmpty()) { + public void setAppCollectorsMap( + Map appCollectorsMap) { + if (appCollectorsMap == null || appCollectorsMap.isEmpty()) { return; } maybeInitBuilder(); - this.appAggregatorsMap = new HashMap(); - this.appAggregatorsMap.putAll(appAggregatorsMap); + this.appCollectorsMap = new HashMap(); + this.appCollectorsMap.putAll(appCollectorsMap); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java deleted file mode 100644 index eb7beef2d5c41..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; -import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; -import org.apache.hadoop.yarn.server.api.records.impl.pb.AppAggregatorsMapPBImpl; - -public class ReportNewAggregatorsInfoRequestPBImpl extends - ReportNewAggregatorsInfoRequest { - - ReportNewAggregatorsInfoRequestProto proto = - ReportNewAggregatorsInfoRequestProto.getDefaultInstance(); - - ReportNewAggregatorsInfoRequestProto.Builder builder = null; - boolean viaProto = false; - - private List aggregatorsList = null; - - public ReportNewAggregatorsInfoRequestPBImpl() { - builder = ReportNewAggregatorsInfoRequestProto.newBuilder(); - } - - public ReportNewAggregatorsInfoRequestPBImpl( - ReportNewAggregatorsInfoRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public ReportNewAggregatorsInfoRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void mergeLocalToBuilder() { - if (aggregatorsList != null) { - addLocalAggregatorsToProto(); - } - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ReportNewAggregatorsInfoRequestProto.newBuilder(proto); - } - viaProto = false; - } - - private void addLocalAggregatorsToProto() { - maybeInitBuilder(); - builder.clearAppAggregators(); - List protoList = - new ArrayList(); - for (AppAggregatorsMap m : this.aggregatorsList) { - protoList.add(convertToProtoFormat(m)); - } - builder.addAllAppAggregators(protoList); - } - - private void initLocalAggregatorsList() { - ReportNewAggregatorsInfoRequestProtoOrBuilder p = viaProto ? proto : builder; - List aggregatorsList = - p.getAppAggregatorsList(); - this.aggregatorsList = new ArrayList(); - for (AppAggregatorsMapProto m : aggregatorsList) { - this.aggregatorsList.add(convertFromProtoFormat(m)); - } - } - - @Override - public List getAppAggregatorsList() { - if (this.aggregatorsList == null) { - initLocalAggregatorsList(); - } - return this.aggregatorsList; - } - - @Override - public void setAppAggregatorsList(List appAggregatorsList) { - maybeInitBuilder(); - if (appAggregatorsList == null) { - builder.clearAppAggregators(); - } - this.aggregatorsList = appAggregatorsList; - } - - private AppAggregatorsMapPBImpl convertFromProtoFormat( - AppAggregatorsMapProto p) { - return new AppAggregatorsMapPBImpl(p); - } - - private AppAggregatorsMapProto convertToProtoFormat( - AppAggregatorsMap m) { - return ((AppAggregatorsMapPBImpl) m).getProto(); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java new file mode 100644 index 0000000000000..5dd8f17feddcd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; +import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; +import org.apache.hadoop.yarn.server.api.records.impl.pb.AppCollectorsMapPBImpl; + +public class ReportNewCollectorInfoRequestPBImpl extends + ReportNewCollectorInfoRequest { + + ReportNewCollectorInfoRequestProto proto = + ReportNewCollectorInfoRequestProto.getDefaultInstance(); + + ReportNewCollectorInfoRequestProto.Builder builder = null; + boolean viaProto = false; + + private List collectorsList = null; + + public ReportNewCollectorInfoRequestPBImpl() { + builder = ReportNewCollectorInfoRequestProto.newBuilder(); + } + + public ReportNewCollectorInfoRequestPBImpl( + ReportNewCollectorInfoRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public ReportNewCollectorInfoRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (collectorsList != null) { + addLocalCollectorsToProto(); + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ReportNewCollectorInfoRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalCollectorsToProto() { + maybeInitBuilder(); + builder.clearAppCollectors(); + List protoList = + new ArrayList(); + for (AppCollectorsMap m : this.collectorsList) { + protoList.add(convertToProtoFormat(m)); + } + builder.addAllAppCollectors(protoList); + } + + private void initLocalCollectorsList() { + ReportNewCollectorInfoRequestProtoOrBuilder p = viaProto ? proto : builder; + List collectorsList = + p.getAppCollectorsList(); + this.collectorsList = new ArrayList(); + for (AppCollectorsMapProto m : collectorsList) { + this.collectorsList.add(convertFromProtoFormat(m)); + } + } + + @Override + public List getAppCollectorsList() { + if (this.collectorsList == null) { + initLocalCollectorsList(); + } + return this.collectorsList; + } + + @Override + public void setAppCollectorsList(List appCollectorsList) { + maybeInitBuilder(); + if (appCollectorsList == null) { + builder.clearAppCollectors(); + } + this.collectorsList = appCollectorsList; + } + + private AppCollectorsMapPBImpl convertFromProtoFormat( + AppCollectorsMapProto p) { + return new AppCollectorsMapPBImpl(p); + } + + private AppCollectorsMapProto convertToProtoFormat( + AppCollectorsMap m) { + return ((AppCollectorsMapPBImpl) m).getProto(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java similarity index 74% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java index 0f0925a85c205..7c90675b6ad49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java @@ -19,33 +19,33 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoResponseProto; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; import com.google.protobuf.TextFormat; @Private @Unstable -public class ReportNewAggregatorsInfoResponsePBImpl extends - ReportNewAggregatorsInfoResponse { +public class ReportNewCollectorInfoResponsePBImpl extends + ReportNewCollectorInfoResponse { + + ReportNewCollectorInfoResponseProto proto = + ReportNewCollectorInfoResponseProto.getDefaultInstance(); + + ReportNewCollectorInfoResponseProto.Builder builder = null; - ReportNewAggregatorsInfoResponseProto proto = - ReportNewAggregatorsInfoResponseProto.getDefaultInstance(); - - ReportNewAggregatorsInfoResponseProto.Builder builder = null; - boolean viaProto = false; - - public ReportNewAggregatorsInfoResponsePBImpl() { - builder = ReportNewAggregatorsInfoResponseProto.newBuilder(); + + public ReportNewCollectorInfoResponsePBImpl() { + builder = ReportNewCollectorInfoResponseProto.newBuilder(); } - public ReportNewAggregatorsInfoResponsePBImpl(ReportNewAggregatorsInfoResponseProto proto) { + public ReportNewCollectorInfoResponsePBImpl(ReportNewCollectorInfoResponseProto proto) { this.proto = proto; viaProto = true; } - - public ReportNewAggregatorsInfoResponseProto getProto() { + + public ReportNewCollectorInfoResponseProto getProto() { proto = viaProto ? proto : builder.build(); viaProto = true; return proto; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java deleted file mode 100644 index 67c377de254bb..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.apache.hadoop.yarn.server.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.util.Records; - - -@Private -public abstract class AppAggregatorsMap { - - public static AppAggregatorsMap newInstance( - ApplicationId id, String aggregatorAddr) { - AppAggregatorsMap appAggregatorMap = - Records.newRecord(AppAggregatorsMap.class); - appAggregatorMap.setApplicationId(id); - appAggregatorMap.setAggregatorAddr(aggregatorAddr); - return appAggregatorMap; - } - - public abstract ApplicationId getApplicationId(); - - public abstract void setApplicationId( - ApplicationId id); - - public abstract String getAggregatorAddr(); - - public abstract void setAggregatorAddr( - String addr); - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java new file mode 100644 index 0000000000000..07e1d92898fbf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.Records; + + +@Private +public abstract class AppCollectorsMap { + + public static AppCollectorsMap newInstance( + ApplicationId id, String collectorAddr) { + AppCollectorsMap appCollectorsMap = + Records.newRecord(AppCollectorsMap.class); + appCollectorsMap.setApplicationId(id); + appCollectorsMap.setCollectorAddr(collectorAddr); + return appCollectorsMap; + } + + public abstract ApplicationId getApplicationId(); + + public abstract void setApplicationId(ApplicationId id); + + public abstract String getCollectorAddr(); + + public abstract void setCollectorAddr(String addr); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java similarity index 71% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java index 32903e2e58eba..eb3bde4df6c14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java @@ -21,37 +21,37 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; -import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; +import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProtoOrBuilder; import com.google.protobuf.TextFormat; @Private @Unstable -public class AppAggregatorsMapPBImpl extends AppAggregatorsMap { +public class AppCollectorsMapPBImpl extends AppCollectorsMap { - AppAggregatorsMapProto proto = - AppAggregatorsMapProto.getDefaultInstance(); - - AppAggregatorsMapProto.Builder builder = null; + AppCollectorsMapProto proto = + AppCollectorsMapProto.getDefaultInstance(); + + AppCollectorsMapProto.Builder builder = null; boolean viaProto = false; - + private ApplicationId appId = null; - private String aggregatorAddr = null; - - public AppAggregatorsMapPBImpl() { - builder = AppAggregatorsMapProto.newBuilder(); + private String collectorAddr = null; + + public AppCollectorsMapPBImpl() { + builder = AppCollectorsMapProto.newBuilder(); } - public AppAggregatorsMapPBImpl(AppAggregatorsMapProto proto) { + public AppCollectorsMapPBImpl(AppCollectorsMapProto proto) { this.proto = proto; viaProto = true; } - - public AppAggregatorsMapProto getProto() { + + public AppCollectorsMapProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -77,24 +77,24 @@ public boolean equals(Object other) { public String toString() { return TextFormat.shortDebugString(getProto()); } - + @Override public ApplicationId getApplicationId() { - AppAggregatorsMapProtoOrBuilder p = viaProto ? proto : builder; + AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder; if (this.appId == null && p.hasAppId()) { this.appId = convertFromProtoFormat(p.getAppId()); } return this.appId; } - + @Override - public String getAggregatorAddr() { - AppAggregatorsMapProtoOrBuilder p = viaProto ? proto : builder; - if (this.aggregatorAddr == null - && p.hasAppAggregatorAddr()) { - this.aggregatorAddr = p.getAppAggregatorAddr(); + public String getCollectorAddr() { + AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder; + if (this.collectorAddr == null + && p.hasAppCollectorAddr()) { + this.collectorAddr = p.getAppCollectorAddr(); } - return this.aggregatorAddr; + return this.collectorAddr; } @Override @@ -105,31 +105,31 @@ public void setApplicationId(ApplicationId appId) { } this.appId = appId; } - + @Override - public void setAggregatorAddr(String aggregatorAddr) { + public void setCollectorAddr(String collectorAddr) { maybeInitBuilder(); - if (aggregatorAddr == null) { - builder.clearAppAggregatorAddr(); + if (collectorAddr == null) { + builder.clearAppCollectorAddr(); } - this.aggregatorAddr = aggregatorAddr; + this.collectorAddr = collectorAddr; } - + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { return new ApplicationIdPBImpl(p); } - + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { return ((ApplicationIdPBImpl) t).getProto(); } - + private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = AppAggregatorsMapProto.newBuilder(proto); + builder = AppCollectorsMapProto.newBuilder(proto); } viaProto = false; } - + private void mergeLocalToProto() { if (viaProto) { maybeInitBuilder(); @@ -138,13 +138,13 @@ private void mergeLocalToProto() { proto = builder.build(); viaProto = true; } - + private void mergeLocalToBuilder() { if (this.appId != null) { builder.setAppId(convertToProtoFormat(this.appId)); } - if (this.aggregatorAddr != null) { - builder.setAppAggregatorAddr(this.aggregatorAddr); + if (this.collectorAddr != null) { + builder.setAppCollectorAddr(this.collectorAddr); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto similarity index 81% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto index d7b05c1417324..654a9f2168ca3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto @@ -17,13 +17,13 @@ */ option java_package = "org.apache.hadoop.yarn.proto"; -option java_outer_classname = "AggregatorNodemanagerProtocol"; +option java_outer_classname = "CollectorNodemanagerProtocol"; option java_generic_services = true; option java_generate_equals_and_hash = true; package hadoop.yarn; import "yarn_server_common_service_protos.proto"; -service AggregatorNodemanagerProtocolService { - rpc reportNewAggregatorInfo (ReportNewAggregatorsInfoRequestProto) returns (ReportNewAggregatorsInfoResponseProto); +service CollectorNodemanagerProtocolService { + rpc reportNewCollectorInfo (ReportNewCollectorInfoRequestProto) returns (ReportNewCollectorInfoResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index 625928b5d08ee..af00748417062 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -84,7 +84,7 @@ message NodeHeartbeatRequestProto { optional MasterKeyProto last_known_nm_token_master_key = 3; optional NodeLabelsProto nodeLabels = 4; repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5; - repeated AppAggregatorsMapProto registered_aggregators = 6; + repeated AppCollectorsMapProto registered_collectors = 6; } message LogAggregationReportProto { @@ -109,7 +109,7 @@ message NodeHeartbeatResponseProto { repeated SignalContainerRequestProto containers_to_signal = 13; optional ResourceProto resource = 14; optional ContainerQueuingLimitProto container_queuing_limit = 15; - repeated AppAggregatorsMapProto app_aggregators_map = 16; + repeated AppCollectorsMapProto app_collectors_map = 16; } message ContainerQueuingLimitProto { @@ -123,21 +123,21 @@ message SystemCredentialsForAppsProto { } //////////////////////////////////////////////////////////////////////// -////// From aggregator_nodemanager_protocol //////////////////////////// +////// From collector_nodemanager_protocol //////////////////////////// //////////////////////////////////////////////////////////////////////// -message AppAggregatorsMapProto { +message AppCollectorsMapProto { optional ApplicationIdProto appId = 1; - optional string appAggregatorAddr = 2; + optional string appCollectorAddr = 2; } ////////////////////////////////////////////////////// -/////// aggregator_nodemanager_protocol ////////////// +/////// collector_nodemanager_protocol ////////////// ////////////////////////////////////////////////////// -message ReportNewAggregatorsInfoRequestProto { - repeated AppAggregatorsMapProto app_aggregators = 1; +message ReportNewCollectorInfoRequestProto { + repeated AppCollectorsMapProto app_collectors = 1; } -message ReportNewAggregatorsInfoResponseProto { +message ReportNewCollectorInfoResponseProto { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 3dc92ce9484df..1f47dfda13ec1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -63,10 +63,10 @@ import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; -import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; +import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -76,15 +76,15 @@ public class TestRPC { private static final String EXCEPTION_MSG = "test error"; private static final String EXCEPTION_CAUSE = "exception cause"; private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - - public static final String ILLEGAL_NUMBER_MESSAGE = - "aggregators' number in ReportNewAggregatorsInfoRequest is not ONE."; - - public static final String DEFAULT_AGGREGATOR_ADDR = "localhost:0"; - - public static final ApplicationId DEFAULT_APP_ID = + + public static final String ILLEGAL_NUMBER_MESSAGE = + "collectors' number in ReportNewCollectorInfoRequest is not ONE."; + + public static final String DEFAULT_COLLECTOR_ADDR = "localhost:0"; + + public static final ApplicationId DEFAULT_APP_ID = ApplicationId.newInstance(0, 0); - + @Test public void testUnknownCall() { Configuration conf = new Configuration(); @@ -116,17 +116,17 @@ public void testUnknownCall() { server.stop(); } } - + @Test - public void testRPCOnAggregatorNodeManagerProtocol() throws IOException { + public void testRPCOnCollectorNodeManagerProtocol() throws IOException { Configuration conf = new Configuration(); conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class .getName()); YarnRPC rpc = YarnRPC.create(conf); String bindAddr = "localhost:0"; InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); - Server server = rpc.getServer(AggregatorNodemanagerProtocol.class, - new DummyNMAggregatorService(), addr, conf, null, 1); + Server server = rpc.getServer(CollectorNodemanagerProtocol.class, + new DummyNMCollectorService(), addr, conf, null, 1); server.start(); // Test unrelated protocol wouldn't get response @@ -145,31 +145,31 @@ public void testRPCOnAggregatorNodeManagerProtocol() throws IOException { } catch (Exception e) { e.printStackTrace(); } - - // Test AggregatorNodemanagerProtocol get proper response - AggregatorNodemanagerProtocol proxy = (AggregatorNodemanagerProtocol)rpc.getProxy( - AggregatorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf); - // Verify request with DEFAULT_APP_ID and DEFAULT_AGGREGATOR_ADDR get + + // Test CollectorNodemanagerProtocol get proper response + CollectorNodemanagerProtocol proxy = (CollectorNodemanagerProtocol)rpc.getProxy( + CollectorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf); + // Verify request with DEFAULT_APP_ID and DEFAULT_COLLECTOR_ADDR get // normally response. try { - ReportNewAggregatorsInfoRequest request = - ReportNewAggregatorsInfoRequest.newInstance( - DEFAULT_APP_ID, DEFAULT_AGGREGATOR_ADDR); - proxy.reportNewAggregatorInfo(request); + ReportNewCollectorInfoRequest request = + ReportNewCollectorInfoRequest.newInstance( + DEFAULT_APP_ID, DEFAULT_COLLECTOR_ADDR); + proxy.reportNewCollectorInfo(request); } catch (YarnException e) { Assert.fail("RPC call failured is not expected here."); } - - // Verify empty request get YarnException back (by design in - // DummyNMAggregatorService) + + // Verify empty request get YarnException back (by design in + // DummyNMCollectorService) try { - proxy.reportNewAggregatorInfo(Records - .newRecord(ReportNewAggregatorsInfoRequest.class)); + proxy.reportNewCollectorInfo(Records + .newRecord(ReportNewCollectorInfoRequest.class)); Assert.fail("Excepted RPC call to fail with YarnException."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains(ILLEGAL_NUMBER_MESSAGE)); } - + server.stop(); } @@ -177,21 +177,21 @@ public void testRPCOnAggregatorNodeManagerProtocol() throws IOException { public void testHadoopProtoRPC() throws Exception { test(HadoopYarnProtoRPC.class.getName()); } - + private void test(String rpcClass) throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass); YarnRPC rpc = YarnRPC.create(conf); String bindAddr = "localhost:0"; InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); - Server server = rpc.getServer(ContainerManagementProtocol.class, + Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1); server.start(); RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class); - ContainerManagementProtocol proxy = (ContainerManagementProtocol) - rpc.getProxy(ContainerManagementProtocol.class, + ContainerManagementProtocol proxy = (ContainerManagementProtocol) + rpc.getProxy(ContainerManagementProtocol.class, NetUtils.getConnectAddress(server), conf); - ContainerLaunchContext containerLaunchContext = + ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); ApplicationId applicationId = ApplicationId.newInstance(0, 0); @@ -255,7 +255,7 @@ public class DummyContainerManager implements ContainerManagementProtocol { public GetContainerStatusesResponse getContainerStatuses( GetContainerStatusesRequest request) throws YarnException { - GetContainerStatusesResponse response = + GetContainerStatusesResponse response = recordFactory.newRecordInstance(GetContainerStatusesResponse.class); response.setContainerStatuses(statuses); return response; @@ -287,9 +287,9 @@ public StartContainersResponse startContainers( } @Override - public StopContainersResponse stopContainers(StopContainersRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - Exception e = new Exception(EXCEPTION_MSG, + Exception e = new Exception(EXCEPTION_MSG, new Exception(EXCEPTION_CAUSE)); throw new YarnException(e); } @@ -332,32 +332,32 @@ public static Token newContainerToken(NodeId nodeId, byte[] password, .buildTokenService(addr).toString()); return containerToken; } - - // A dummy implementation for AggregatorNodemanagerProtocol for test purpose, - // it only can accept one appID, aggregatorAddr pair or throw exceptions - public class DummyNMAggregatorService - implements AggregatorNodemanagerProtocol { - + + // A dummy implementation for CollectorNodemanagerProtocol for test purpose, + // it only can accept one appID, collectorAddr pair or throw exceptions + public class DummyNMCollectorService + implements CollectorNodemanagerProtocol { + @Override - public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( - ReportNewAggregatorsInfoRequest request) + public ReportNewCollectorInfoResponse reportNewCollectorInfo( + ReportNewCollectorInfoRequest request) throws YarnException, IOException { - List appAggregators = request.getAppAggregatorsList(); - if (appAggregators.size() == 1) { - // check default appID and aggregatorAddr - AppAggregatorsMap appAggregator = appAggregators.get(0); - Assert.assertEquals(appAggregator.getApplicationId(), + List appCollectors = request.getAppCollectorsList(); + if (appCollectors.size() == 1) { + // check default appID and collectorAddr + AppCollectorsMap appCollector = appCollectors.get(0); + Assert.assertEquals(appCollector.getApplicationId(), DEFAULT_APP_ID); - Assert.assertEquals(appAggregator.getAggregatorAddr(), - DEFAULT_AGGREGATOR_ADDR); + Assert.assertEquals(appCollector.getCollectorAddr(), + DEFAULT_COLLECTOR_ADDR); } else { throw new YarnException(ILLEGAL_NUMBER_MESSAGE); } - - ReportNewAggregatorsInfoResponse response = - recordFactory.newRecordInstance(ReportNewAggregatorsInfoResponse.class); + + ReportNewCollectorInfoResponse response = + recordFactory.newRecordInstance(ReportNewCollectorInfoResponse.class); return response; } } - + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 4c65948deeb91..475ce58578aff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -109,14 +109,14 @@ public void testNodeHeartbeatRequestPBImpl() { original.setLastKnownNMTokenMasterKey(getMasterKey()); original.setNodeStatus(getNodeStatus()); original.setNodeLabels(getValidNodeLabels()); - Map aggregators = getAggregators(); - original.setRegisteredAggregators(aggregators); + Map collectors = getCollectors(); + original.setRegisteredCollectors(collectors); NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl( original.getProto()); assertEquals(1, copy.getLastKnownContainerTokenMasterKey().getKeyId()); assertEquals(1, copy.getLastKnownNMTokenMasterKey().getKeyId()); assertEquals("localhost", copy.getNodeStatus().getNodeId().getHost()); - assertEquals(aggregators, copy.getRegisteredAggregators()); + assertEquals(collectors, copy.getRegisteredCollectors()); // check labels are coming with valid values Assert.assertTrue(original.getNodeLabels() .containsAll(copy.getNodeLabels())); @@ -153,8 +153,8 @@ public void testNodeHeartbeatResponsePBImpl() { original.setNextHeartBeatInterval(1000); original.setNodeAction(NodeAction.NORMAL); original.setResponseId(100); - Map aggregators = getAggregators(); - original.setAppAggregatorsMap(aggregators); + Map collectors = getCollectors(); + original.setAppCollectorsMap(collectors); NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl( original.getProto()); @@ -164,7 +164,7 @@ public void testNodeHeartbeatResponsePBImpl() { assertEquals(1, copy.getContainerTokenMasterKey().getKeyId()); assertEquals(1, copy.getNMTokenMasterKey().getKeyId()); assertEquals("testDiagnosticMessage", copy.getDiagnosticsMessage()); - assertEquals(aggregators, copy.getAppAggregatorsMap()); + assertEquals(collectors, copy.getAppCollectorsMap()); assertEquals(false, copy.getAreNodeLabelsAcceptedByRM()); } @@ -344,15 +344,15 @@ private HashSet getValidNodeLabels() { return nodeLabels; } - private Map getAggregators() { + private Map getCollectors() { ApplicationId appID = ApplicationId.newInstance(1L, 1); - String aggregatorAddr = "localhost:0"; - Map aggregatorMap = + String collectorAddr = "localhost:0"; + Map collectorMap = new HashMap(); - aggregatorMap.put(appID, aggregatorAddr); - return aggregatorMap; + collectorMap.put(appID, collectorAddr); + return collectorMap; } - + private ContainerStatus getContainerStatus(int applicationId, int containerID, int appAttemptId) { ContainerStatus status = recordFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index aa8655c52fc8c..af896c853c961 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -70,19 +70,19 @@ interface QueuingContext { ConcurrentMap getApplications(); Map getSystemCredentialsForApps(); - + /** - * Get the registered aggregators that located on this NM. + * Get the registered collectors that located on this NM. * @return registered */ - Map getRegisteredAggregators(); - + Map getRegisteredCollectors(); + /** - * Return the known aggregators which get from RM for all active applications + * Return the known collectors which get from RM for all active applications * running on this NM. - * @return known aggregators. + * @return known collectors. */ - Map getKnownAggregators(); + Map getKnownCollectors(); ConcurrentMap getContainers(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 271795c746ee3..0792c370aafb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -60,7 +60,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager; -import org.apache.hadoop.yarn.server.nodemanager.aggregatormanager.NMAggregatorService; +import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -99,7 +99,7 @@ public class NodeManager extends CompositeService private Context context; private AsyncDispatcher dispatcher; private ContainerManagerImpl containerManager; - private NMAggregatorService nmAggregatorService; + private NMCollectorService nmCollectorService; private NodeStatusUpdater nodeStatusUpdater; private NodeResourceMonitor nodeResourceMonitor; private static CompositeServiceShutdownHook nodeManagerShutdownHook; @@ -184,9 +184,9 @@ protected ContainerManagerImpl createContainerManager(Context context, metrics, dirsHandler); } } - - protected NMAggregatorService createNMAggregatorService(Context context) { - return new NMAggregatorService(context); + + protected NMCollectorService createNMCollectorService(Context context) { + return new NMCollectorService(context); } protected WebServer createWebServer(Context nmContext, @@ -379,9 +379,9 @@ protected void serviceInit(Configuration conf) throws Exception { metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); DefaultMetricsSystem.initialize("NodeManager"); - - this.nmAggregatorService = createNMAggregatorService(context); - addService(nmAggregatorService); + + this.nmCollectorService = createNMCollectorService(context); + addService(nmCollectorService); // StatusUpdater should be added last so that it get started last // so that we make sure everything is up before registering with RM. @@ -474,11 +474,11 @@ public static class NMContext implements Context { protected final ConcurrentMap containers = new ConcurrentSkipListMap(); - - protected Map registeredAggregators = + + protected Map registeredCollectors = new ConcurrentHashMap(); - - protected Map knownAggregators = + + protected Map knownCollectors = new ConcurrentHashMap(); protected final ConcurrentMap getRegisteredAggregators() { - return this.registeredAggregators; + public Map getRegisteredCollectors() { + return this.registeredCollectors; } - public void addRegisteredAggregators( - Map newRegisteredAggregators) { - this.registeredAggregators.putAll(newRegisteredAggregators); - // Update to knownAggregators as well so it can immediately be consumed by + public void addRegisteredCollectors( + Map newRegisteredCollectors) { + this.registeredCollectors.putAll(newRegisteredCollectors); + // Update to knownCollectors as well so it can immediately be consumed by // this NM's TimelineClient. - this.knownAggregators.putAll(newRegisteredAggregators); + this.knownCollectors.putAll(newRegisteredCollectors); } - + @Override - public Map getKnownAggregators() { - return this.knownAggregators; + public Map getKnownCollectors() { + return this.knownCollectors; } - public void addKnownAggregators( - Map knownAggregators) { - this.knownAggregators.putAll(knownAggregators); + public void addKnownCollectors( + Map knownCollectors) { + this.knownCollectors.putAll(knownCollectors); } } @@ -781,10 +781,10 @@ Dispatcher getNMDispatcher(){ public Context getNMContext() { return this.context; } - + // For testing - NMAggregatorService getNMAggregatorService() { - return this.nmAggregatorService; + NMCollectorService getNMCollectorService() { + return this.nmCollectorService; } public static void main(String[] args) throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index f54469305a218..eebc1e0952a32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -814,7 +814,7 @@ public void run() { NodeStatusUpdaterImpl.this.context .getNMTokenSecretManager().getCurrentKey(), nodeLabelsForHeartbeat, - NodeStatusUpdaterImpl.this.context.getRegisteredAggregators()); + NodeStatusUpdaterImpl.this.context.getRegisteredCollectors()); if (logAggregationEnabled) { // pull log aggregation status for application running in this NM @@ -907,8 +907,9 @@ public void run() { } } - Map knownAggregators = response.getAppAggregatorsMap(); - ((NodeManager.NMContext)context).addKnownAggregators(knownAggregators); + Map knownCollectors = + response.getAppCollectorsMap(); + ((NodeManager.NMContext)context).addKnownCollectors(knownCollectors); } catch (ConnectException e) { //catch and throw the exception if tried MAX wait time to connect RM diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java similarity index 54% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java index 17150bac499fa..009fa63357f16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.nodemanager.aggregatormanager; +package org.apache.hadoop.yarn.server.nodemanager.collectormanager; import java.io.IOException; import java.net.InetSocketAddress; @@ -25,46 +25,43 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.service.CompositeService; - import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; -import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse; +import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; -public class NMAggregatorService extends CompositeService implements - AggregatorNodemanagerProtocol { +public class NMCollectorService extends CompositeService implements + CollectorNodemanagerProtocol { - private static final Log LOG = LogFactory.getLog(NMAggregatorService.class); + private static final Log LOG = LogFactory.getLog(NMCollectorService.class); final Context context; - + private Server server; - public NMAggregatorService(Context context) { - - super(NMAggregatorService.class.getName()); + public NMCollectorService(Context context) { + + super(NMCollectorService.class.getName()); this.context = context; } @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); - - InetSocketAddress aggregatorServerAddress = conf.getSocketAddr( + + InetSocketAddress collectorServerAddress = conf.getSocketAddr( YarnConfiguration.NM_BIND_HOST, - YarnConfiguration.NM_AGGREGATOR_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_PORT); + YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT); Configuration serverConf = new Configuration(conf); @@ -72,42 +69,42 @@ protected void serviceStart() throws Exception { YarnRPC rpc = YarnRPC.create(conf); server = - rpc.getServer(AggregatorNodemanagerProtocol.class, this, - aggregatorServerAddress, serverConf, + rpc.getServer(CollectorNodemanagerProtocol.class, this, + collectorServerAddress, serverConf, this.context.getNMTokenSecretManager(), - conf.getInt(YarnConfiguration.NM_AGGREGATOR_SERVICE_THREAD_COUNT, - YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_THREAD_COUNT)); + conf.getInt(YarnConfiguration.NM_COLLECTOR_SERVICE_THREAD_COUNT, + YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT)); server.start(); // start remaining services super.serviceStart(); - LOG.info("NMAggregatorService started at " + aggregatorServerAddress); + LOG.info("NMCollectorService started at " + collectorServerAddress); } - + @Override public void serviceStop() throws Exception { if (server != null) { server.stop(); } - // TODO may cleanup app aggregators running on this NM in future. + // TODO may cleanup app collectors running on this NM in future. super.serviceStop(); } @Override - public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( - ReportNewAggregatorsInfoRequest request) throws IOException { - List newAggregatorsList = request.getAppAggregatorsList(); - if (newAggregatorsList != null && !newAggregatorsList.isEmpty()) { - Map newAggregatorsMap = + public ReportNewCollectorInfoResponse reportNewCollectorInfo( + ReportNewCollectorInfoRequest request) throws IOException { + List newCollectorsList = request.getAppCollectorsList(); + if (newCollectorsList != null && !newCollectorsList.isEmpty()) { + Map newCollectorsMap = new HashMap(); - for (AppAggregatorsMap aggregator : newAggregatorsList) { - newAggregatorsMap.put(aggregator.getApplicationId(), aggregator.getAggregatorAddr()); + for (AppCollectorsMap collector : newCollectorsList) { + newCollectorsMap.put(collector.getApplicationId(), collector.getCollectorAddr()); } - ((NodeManager.NMContext)context).addRegisteredAggregators(newAggregatorsMap); + ((NodeManager.NMContext)context).addRegisteredCollectors(newCollectorsMap); } - - return ReportNewAggregatorsInfoResponse.newInstance(); + + return ReportNewCollectorInfoResponse.newInstance(); } - + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 6864b0ddf7fc4..c9c6f6bf4900d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -496,10 +496,11 @@ public void transition(ApplicationImpl app, ApplicationEvent event) { new LogHandlerAppFinishedEvent(app.appId)); app.context.getNMTokenSecretManager().appFinished(app.getAppId()); - // Remove aggregator info for finished apps. - // TODO check we remove related aggregators info in failure cases (YARN-3038) - app.context.getRegisteredAggregators().remove(app.getAppId()); - app.context.getKnownAggregators().remove(app.getAppId()); + // Remove collectors info for finished apps. + // TODO check we remove related collectors info in failure cases + // (YARN-3038) + app.context.getRegisteredCollectors().remove(app.getAppId()); + app.context.getKnownCollectors().remove(app.getAppId()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index a974267730cd8..70bd8f4c1fd59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -618,12 +618,12 @@ public Map getSystemCredentialsForApps() { } @Override - public Map getRegisteredAggregators() { + public Map getRegisteredCollectors() { return null; } @Override - public Map getKnownAggregators() { + public Map getKnownCollectors() { return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 805794871e01a..d0a8a6e9b0208 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -311,8 +311,8 @@ public FinishApplicationMasterResponse finishApplicationMaster( RMApp rmApp = rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); - // Remove aggregator address when app get finished. - rmApp.removeAggregatorAddr(); + // Remove collector address when app get finished. + rmApp.removeCollectorAddr(); // checking whether the app exits in RMStateStore at first not to throw // ApplicationDoesNotExistInCacheException before and after // RM work-preserving restart. @@ -573,10 +573,10 @@ public AllocateResponse allocate(AllocateRequest request) allocateResponse.setDecreasedContainers(allocation.getDecreasedContainers()); allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes()); - - // add aggregator address for this application - allocateResponse.setAggregatorAddr( - this.rmContext.getRMApps().get(applicationId).getAggregatorAddr()); + + // add collector address for this application + allocateResponse.setCollectorAddr( + this.rmContext.getRMApps().get(applicationId).getCollectorAddr()); // add preemption to the allocateResponse message (if any) allocateResponse diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 4b712c0a1d163..683aa9143be23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -70,7 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppAggregatorUpdateEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppCollectorUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -516,11 +516,11 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) return YarnServerBuilderUtils.newNodeHeartbeatResponse(NodeAction.RESYNC, message); } - - // Check & update aggregators info from request. + + // Check & update collectors info from request. // TODO make sure it won't have race condition issue for AM failed over case // that the older registration could possible override the newer one. - updateAppAggregatorsMap(request); + updateAppCollectorsMap(request); // Heartbeat response NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils @@ -538,13 +538,14 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) if (!systemCredentials.isEmpty()) { nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials); } - - // Return aggregators' map that NM needs to know - // TODO we should optimize this to only include aggreator info that NM + + // Return collectors' map that NM needs to know + // TODO we should optimize this to only include collector info that NM // doesn't know yet. - List keepAliveApps = remoteNodeStatus.getKeepAliveApplications(); + List keepAliveApps = + remoteNodeStatus.getKeepAliveApplications(); if (keepAliveApps != null) { - setAppAggregatorsMapToResponse(keepAliveApps, nodeHeartBeatResponse); + setAppCollectorsMapToResponse(keepAliveApps, nodeHeartBeatResponse); } // 4. Send status to RMNode, saving the latest response. @@ -589,48 +590,49 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) } return nodeHeartBeatResponse; } - - private void setAppAggregatorsMapToResponse( + + private void setAppCollectorsMapToResponse( List liveApps, NodeHeartbeatResponse response) { - Map liveAppAggregatorsMap = new + Map liveAppCollectorsMap = new ConcurrentHashMap(); Map rmApps = rmContext.getRMApps(); for (ApplicationId appId : liveApps) { - String appAggregatorAddr = rmApps.get(appId).getAggregatorAddr(); - if (appAggregatorAddr != null) { - liveAppAggregatorsMap.put(appId, appAggregatorAddr); + String appCollectorAddr = rmApps.get(appId).getCollectorAddr(); + if (appCollectorAddr != null) { + liveAppCollectorsMap.put(appId, appCollectorAddr); } else { - // Log a debug info if aggregator address is not found. + // Log a debug info if collector address is not found. if (LOG.isDebugEnabled()) { - LOG.debug("Aggregator for applicaton: " + appId + " hasn't registered yet!"); + LOG.debug("Collector for applicaton: " + appId + + " hasn't registered yet!"); } } } - response.setAppAggregatorsMap(liveAppAggregatorsMap); + response.setAppCollectorsMap(liveAppCollectorsMap); } - - private void updateAppAggregatorsMap(NodeHeartbeatRequest request) { - Map registeredAggregatorsMap = - request.getRegisteredAggregators(); - if (registeredAggregatorsMap != null - && !registeredAggregatorsMap.isEmpty()) { + + private void updateAppCollectorsMap(NodeHeartbeatRequest request) { + Map registeredCollectorsMap = + request.getRegisteredCollectors(); + if (registeredCollectorsMap != null + && !registeredCollectorsMap.isEmpty()) { Map rmApps = rmContext.getRMApps(); - for (Map.Entry entry: - registeredAggregatorsMap.entrySet()) { + for (Map.Entry entry: + registeredCollectorsMap.entrySet()) { ApplicationId appId = entry.getKey(); - String aggregatorAddr = entry.getValue(); - if (aggregatorAddr != null && !aggregatorAddr.isEmpty()) { + String collectorAddr = entry.getValue(); + if (collectorAddr != null && !collectorAddr.isEmpty()) { RMApp rmApp = rmApps.get(appId); if (rmApp == null) { - LOG.warn("Cannot update aggregator info because application ID: " + + LOG.warn("Cannot update collector info because application ID: " + appId + " is not found in RMContext!"); } else { - String previousAggregatorAddr = rmApp.getAggregatorAddr(); - if (previousAggregatorAddr == null || - previousAggregatorAddr != aggregatorAddr) { - // sending aggregator update event. - RMAppAggregatorUpdateEvent event = - new RMAppAggregatorUpdateEvent(appId, aggregatorAddr); + String previousCollectorAddr = rmApp.getCollectorAddr(); + if (previousCollectorAddr == null || + previousCollectorAddr != collectorAddr) { + // sending collector update event. + RMAppCollectorUpdateEvent event = + new RMAppCollectorUpdateEvent(appId, collectorAddr); rmContext.getDispatcher().getEventHandler().handle(event); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 25b2314ab075b..8a795cde598d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -175,23 +175,23 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the tracking url for the application master. */ String getTrackingUrl(); - + /** - * The aggregator address for the application. - * @return the address for the application's aggregator. + * The collector address for the application. + * @return the address for the application's collector. */ - String getAggregatorAddr(); - + String getCollectorAddr(); + /** - * Set aggregator address for the application - * @param aggregatorAddr the address of aggregator + * Set collector address for the application + * @param collectorAddr the address of collector */ - void setAggregatorAddr(String aggregatorAddr); - + void setCollectorAddr(String collectorAddr); + /** - * Remove aggregator address when application is finished or killed. + * Remove collector address when application is finished or killed. */ - void removeAggregatorAddr(); + void removeCollectorAddr(); /** * The original tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java similarity index 71% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java index b43de44da952f..698c9b5c4a468 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java @@ -20,17 +20,18 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; -public class RMAppAggregatorUpdateEvent extends RMAppEvent { +public class RMAppCollectorUpdateEvent extends RMAppEvent { - private final String appAggregatorAddr; - - public RMAppAggregatorUpdateEvent(ApplicationId appId, String appAggregatorAddr) { - super(appId, RMAppEventType.AGGREGATOR_UPDATE); - this.appAggregatorAddr = appAggregatorAddr; + private final String appCollectorAddr; + + public RMAppCollectorUpdateEvent(ApplicationId appId, + String appCollectorAddr) { + super(appId, RMAppEventType.COLLECTOR_UPDATE); + this.appCollectorAddr = appCollectorAddr; } - - public String getAppAggregatorAddr(){ - return this.appAggregatorAddr; + + public String getAppCollectorAddr(){ + return this.appCollectorAddr; } - + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index 6e9460aefea8c..2b42638a9f08a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -30,9 +30,9 @@ public enum RMAppEventType { // Source: Scheduler APP_ACCEPTED, - + // TODO add source later - AGGREGATOR_UPDATE, + COLLECTOR_UPDATE, // Source: RMAppAttempt ATTEMPT_REGISTERED, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index cf50c3d466abf..ac6a585abdffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -152,7 +152,7 @@ public class RMAppImpl implements RMApp, Recoverable { private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; private int nextAttemptId = 1; - private String aggregatorAddr; + private String collectorAddr; // This field isn't protected by readlock now. private volatile RMAppAttempt currentAttempt; private String queue; @@ -201,7 +201,7 @@ public class RMAppImpl implements RMApp, Recoverable { .addTransition(RMAppState.NEW, RMAppState.NEW, RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) .addTransition(RMAppState.NEW, RMAppState.NEW, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) .addTransition(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppEventType.START, new RMAppNewlySavingTransition()) .addTransition(RMAppState.NEW, EnumSet.of(RMAppState.SUBMITTED, @@ -219,7 +219,7 @@ RMAppEventType.RECOVER, new RMAppRecoveredTransition()) .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING, RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) .addTransition(RMAppState.NEW_SAVING, RMAppState.SUBMITTED, RMAppEventType.APP_NEW_SAVED, new AddApplicationToSchedulerTransition()) .addTransition(RMAppState.NEW_SAVING, RMAppState.FINAL_SAVING, @@ -239,7 +239,7 @@ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) .addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING, RMAppEventType.APP_REJECTED, new FinalSavingTransition( @@ -257,7 +257,7 @@ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING, RMAppEventType.ATTEMPT_REGISTERED, new RMAppStateUpdateTransition( YarnApplicationState.RUNNING)) @@ -286,7 +286,7 @@ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, RMAppEventType.MOVE, new RMAppMoveTransition()) .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) .addTransition(RMAppState.RUNNING, RMAppState.FINAL_SAVING, RMAppEventType.ATTEMPT_UNREGISTERED, new FinalSavingTransition( @@ -317,7 +317,7 @@ RMAppEventType.KILL, new KillAttemptTransition()) RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) .addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) // ignorable transitions .addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING, EnumSet.of(RMAppEventType.NODE_UPDATE, RMAppEventType.KILL, @@ -330,7 +330,7 @@ RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) .addTransition(RMAppState.FINISHING, RMAppState.FINISHING, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) // ignorable transitions .addTransition(RMAppState.FINISHING, RMAppState.FINISHING, EnumSet.of(RMAppEventType.NODE_UPDATE, @@ -343,7 +343,7 @@ RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) .addTransition(RMAppState.KILLING, RMAppState.KILLING, - RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) + RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition()) .addTransition(RMAppState.KILLING, RMAppState.FINAL_SAVING, RMAppEventType.ATTEMPT_KILLED, new FinalSavingTransition( @@ -577,20 +577,20 @@ public String getQueue() { public void setQueue(String queue) { this.queue = queue; } - + @Override - public String getAggregatorAddr() { - return this.aggregatorAddr; + public String getCollectorAddr() { + return this.collectorAddr; } - + @Override - public void setAggregatorAddr(String aggregatorAddr) { - this.aggregatorAddr = aggregatorAddr; + public void setCollectorAddr(String collectorAddr) { + this.collectorAddr = collectorAddr; } - + @Override - public void removeAggregatorAddr() { - this.aggregatorAddr = null; + public void removeCollectorAddr() { + this.collectorAddr = null; } @Override @@ -864,8 +864,8 @@ public void recover(RMState state) { // send the ATS create Event sendATSCreateEvent(this, this.startTime); - //TODO recover aggregator address. - //this.aggregatorAddr = appState.getAggregatorAddr(); + //TODO recover collector address. + //this.collectorAddr = appState.getCollectorAddr(); RMAppAttemptImpl preAttempt = null; for (ApplicationAttemptId attemptId : @@ -938,22 +938,22 @@ public void transition(RMAppImpl app, RMAppEvent event) { }; } - private static final class RMAppAggregatorUpdateTransition + private static final class RMAppCollectorUpdateTransition extends RMAppTransition { - + public void transition(RMAppImpl app, RMAppEvent event) { - LOG.info("Updating aggregator info for app: " + app.getApplicationId()); - - RMAppAggregatorUpdateEvent appAggregatorUpdateEvent = - (RMAppAggregatorUpdateEvent) event; - // Update aggregator address - app.setAggregatorAddr(appAggregatorUpdateEvent.getAppAggregatorAddr()); - + LOG.info("Updating collector info for app: " + app.getApplicationId()); + + RMAppCollectorUpdateEvent appCollectorUpdateEvent = + (RMAppCollectorUpdateEvent) event; + // Update collector address + app.setCollectorAddr(appCollectorUpdateEvent.getAppCollectorAddr()); + // TODO persistent to RMStateStore for recover // Save to RMStateStore }; } - + private static final class RMAppNodeUpdateTransition extends RMAppTransition { public void transition(RMAppImpl app, RMAppEvent event) { RMAppNodeUpdateEvent nodeUpdateEvent = (RMAppNodeUpdateEvent) event; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 85d6129f2543f..19ee0b17c4617 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -95,15 +95,15 @@ public StringBuilder getDiagnostics() { throw new UnsupportedOperationException("Not supported yet."); } @Override - public String getAggregatorAddr() { + public String getCollectorAddr() { throw new UnsupportedOperationException("Not supported yet."); } @Override - public void setAggregatorAddr(String aggregatorAddr) { + public void setCollectorAddr(String collectorAddr) { throw new UnsupportedOperationException("Not supported yet."); } @Override - public void removeAggregatorAddr() { + public void removeCollectorAddr() { throw new UnsupportedOperationException("Not supported yet."); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index e4d846441a854..62a5c5282ae1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -302,17 +302,17 @@ public CallerContext getCallerContext() { throw new UnsupportedOperationException("Not supported yet."); } - public String getAggregatorAddr() { + public String getCollectorAddr() { throw new UnsupportedOperationException("Not supported yet."); } - + @Override - public void removeAggregatorAddr() { + public void removeCollectorAddr() { throw new UnsupportedOperationException("Not supported yet."); } @Override - public void setAggregatorAddr(String aggregatorAddr) { + public void setCollectorAddr(String collectorAddr) { throw new UnsupportedOperationException("Not supported yet."); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index 22542a090f686..6bd54c29cba39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -56,6 +56,11 @@ junit test + + org.mockito + mockito-all + test + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index 32ee5d8a6c120..fab131cc7d85b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -1,25 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.yarn.server.timelineservice; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.timelineservice.aggregator.PerNodeTimelineAggregatorsAuxService; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; +import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.fail; - public class TestTimelineServiceClientIntegration { - private static PerNodeTimelineAggregatorsAuxService auxService; + private static TimelineCollectorManager collectorManager; + private static PerNodeTimelineCollectorsAuxService auxService; @BeforeClass public static void setupClass() throws Exception { try { - auxService = PerNodeTimelineAggregatorsAuxService.launchServer(new String[0]); + collectorManager = new MyTimelineCollectorManager(); + auxService = + PerNodeTimelineCollectorsAuxService.launchServer(new String[0], + collectorManager); auxService.addApplication(ApplicationId.newInstance(0, 1)); } catch (ExitUtil.ExitException e) { fail(); @@ -38,6 +63,9 @@ public void testPutEntities() throws Exception { TimelineClient client = TimelineClient.createTimelineClient(ApplicationId.newInstance(0, 1)); try { + // set the timeline service address manually + client.setTimelineServiceAddress( + collectorManager.getRestServerBindAddress()); client.init(new YarnConfiguration()); client.start(); TimelineEntity entity = new TimelineEntity(); @@ -45,10 +73,20 @@ public void testPutEntities() throws Exception { entity.setId("test entity id"); client.putEntities(entity); client.putEntitiesAsync(entity); - } catch(Exception e) { - fail(); } finally { client.stop(); } } + + private static class MyTimelineCollectorManager extends + TimelineCollectorManager { + public MyTimelineCollectorManager() { + super(); + } + + @Override + protected CollectorNodemanagerProtocol getNMCollectorService() { + return mock(CollectorNodemanagerProtocol.class); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index e5384a30a94ba..ddaf06aa93e22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -56,6 +56,11 @@ hadoop-yarn-common + + org.apache.hadoop + hadoop-yarn-server-common + + org.apache.hadoop hadoop-yarn-server-applicationhistoryservice @@ -71,6 +76,11 @@ guice + + commons-io + commons-io + + javax.servlet servlet-api diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java similarity index 85% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java index 95ec9f8e40a3b..7d59876a95e07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -30,12 +30,12 @@ */ @Private @Unstable -public class AppLevelTimelineAggregator extends TimelineAggregator { +public class AppLevelTimelineCollector extends TimelineCollector { private final String applicationId; // TODO define key metadata such as flow metadata, user, and queue - public AppLevelTimelineAggregator(String applicationId) { - super(AppLevelTimelineAggregator.class.getName() + " - " + applicationId); + public AppLevelTimelineCollector(String applicationId) { + super(AppLevelTimelineCollector.class.getName() + " - " + applicationId); this.applicationId = applicationId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java similarity index 66% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java index 19920fd6a396a..59ecef14b2e5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; import java.nio.ByteBuffer; @@ -42,85 +42,85 @@ import com.google.common.annotations.VisibleForTesting; /** - * The top-level server for the per-node timeline aggregator collection. Currently + * The top-level server for the per-node timeline collector manager. Currently * it is defined as an auxiliary service to accommodate running within another * daemon (e.g. node manager). */ @Private @Unstable -public class PerNodeTimelineAggregatorsAuxService extends AuxiliaryService { +public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService { private static final Log LOG = - LogFactory.getLog(PerNodeTimelineAggregatorsAuxService.class); + LogFactory.getLog(PerNodeTimelineCollectorsAuxService.class); private static final int SHUTDOWN_HOOK_PRIORITY = 30; - private final TimelineAggregatorsCollection aggregatorCollection; + private final TimelineCollectorManager collectorManager; - public PerNodeTimelineAggregatorsAuxService() { + public PerNodeTimelineCollectorsAuxService() { // use the same singleton - this(TimelineAggregatorsCollection.getInstance()); + this(TimelineCollectorManager.getInstance()); } - @VisibleForTesting PerNodeTimelineAggregatorsAuxService( - TimelineAggregatorsCollection aggregatorCollection) { - super("timeline_aggregator"); - this.aggregatorCollection = aggregatorCollection; + @VisibleForTesting PerNodeTimelineCollectorsAuxService( + TimelineCollectorManager collectorsManager) { + super("timeline_collector"); + this.collectorManager = collectorsManager; } @Override protected void serviceInit(Configuration conf) throws Exception { - aggregatorCollection.init(conf); + collectorManager.init(conf); super.serviceInit(conf); } @Override protected void serviceStart() throws Exception { - aggregatorCollection.start(); + collectorManager.start(); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - aggregatorCollection.stop(); + collectorManager.stop(); super.serviceStop(); } // these methods can be used as the basis for future service methods if the - // per-node aggregator runs separate from the node manager + // per-node collector runs separate from the node manager /** - * Creates and adds an app level aggregator for the specified application id. - * The aggregator is also initialized and started. If the service already + * Creates and adds an app level collector for the specified application id. + * The collector is also initialized and started. If the service already * exists, no new service is created. * * @return whether it was added successfully */ public boolean addApplication(ApplicationId appId) { - AppLevelTimelineAggregator aggregator = - new AppLevelTimelineAggregator(appId.toString()); - return (aggregatorCollection.putIfAbsent(appId, aggregator) - == aggregator); + AppLevelTimelineCollector collector = + new AppLevelTimelineCollector(appId.toString()); + return (collectorManager.putIfAbsent(appId, collector) + == collector); } /** - * Removes the app level aggregator for the specified application id. The - * aggregator is also stopped as a result. If the aggregator does not exist, no + * Removes the app level collector for the specified application id. The + * collector is also stopped as a result. If the collector does not exist, no * change is made. * * @return whether it was removed successfully */ public boolean removeApplication(ApplicationId appId) { String appIdString = appId.toString(); - return aggregatorCollection.remove(appIdString); + return collectorManager.remove(appIdString); } /** - * Creates and adds an app level aggregator for the specified application id. - * The aggregator is also initialized and started. If the aggregator already - * exists, no new aggregator is created. + * Creates and adds an app level collector for the specified application id. + * The collector is also initialized and started. If the collector already + * exists, no new collector is created. */ @Override public void initializeContainer(ContainerInitializationContext context) { // intercept the event of the AM container being created and initialize the - // app level aggregator service + // app level collector service if (isApplicationMaster(context)) { ApplicationId appId = context.getContainerId(). getApplicationAttemptId().getApplicationId(); @@ -129,14 +129,14 @@ public void initializeContainer(ContainerInitializationContext context) { } /** - * Removes the app level aggregator for the specified application id. The - * aggregator is also stopped as a result. If the aggregator does not exist, no + * Removes the app level collector for the specified application id. The + * collector is also stopped as a result. If the collector does not exist, no * change is made. */ @Override public void stopContainer(ContainerTerminationContext context) { // intercept the event of the AM container being stopped and remove the app - // level aggregator service + // level collector service if (isApplicationMaster(context)) { ApplicationId appId = context.getContainerId(). getApplicationAttemptId().getApplicationId(); @@ -154,7 +154,7 @@ private boolean isApplicationMaster(ContainerContext context) { @VisibleForTesting boolean hasApplication(String appId) { - return aggregatorCollection.containsKey(appId); + return collectorManager.containsKey(appId); } @Override @@ -173,30 +173,33 @@ public ByteBuffer getMetaData() { } @VisibleForTesting - public static PerNodeTimelineAggregatorsAuxService launchServer(String[] args) { + public static PerNodeTimelineCollectorsAuxService + launchServer(String[] args, TimelineCollectorManager collectorManager) { Thread .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); - StringUtils.startupShutdownMessage(PerNodeTimelineAggregatorsAuxService.class, args, - LOG); - PerNodeTimelineAggregatorsAuxService auxService = null; + StringUtils.startupShutdownMessage( + PerNodeTimelineCollectorsAuxService.class, args, LOG); + PerNodeTimelineCollectorsAuxService auxService = null; try { - auxService = new PerNodeTimelineAggregatorsAuxService(); + auxService = collectorManager == null ? + new PerNodeTimelineCollectorsAuxService() : + new PerNodeTimelineCollectorsAuxService(collectorManager); ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService), SHUTDOWN_HOOK_PRIORITY); YarnConfiguration conf = new YarnConfiguration(); auxService.init(conf); auxService.start(); } catch (Throwable t) { - LOG.fatal("Error starting PerNodeAggregatorServer", t); - ExitUtil.terminate(-1, "Error starting PerNodeAggregatorServer"); + LOG.fatal("Error starting PerNodeTimelineCollectorServer", t); + ExitUtil.terminate(-1, "Error starting PerNodeTimelineCollectorServer"); } return auxService; } private static class ShutdownHook implements Runnable { - private final PerNodeTimelineAggregatorsAuxService auxService; + private final PerNodeTimelineCollectorsAuxService auxService; - public ShutdownHook(PerNodeTimelineAggregatorsAuxService auxService) { + public ShutdownHook(PerNodeTimelineCollectorsAuxService auxService) { this.auxService = auxService; } @@ -206,6 +209,6 @@ public void run() { } public static void main(String[] args) { - launchServer(args); + launchServer(args, null); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java similarity index 94% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java index dbd089534c325..6e20e6999e325 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; import java.io.IOException; @@ -27,12 +27,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; -import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; /** * Service that handles writes to the timeline service and writes them to the * backing storage. @@ -42,12 +42,12 @@ */ @Private @Unstable -public abstract class TimelineAggregator extends CompositeService { - private static final Log LOG = LogFactory.getLog(TimelineAggregator.class); +public abstract class TimelineCollector extends CompositeService { + private static final Log LOG = LogFactory.getLog(TimelineCollector.class); private TimelineWriter writer; - public TimelineAggregator(String name) { + public TimelineCollector(String name) { super(name); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java similarity index 58% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java index d6e2a1882a441..3691162e3f65c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java @@ -16,11 +16,14 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; + +import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER; import java.io.IOException; -import java.net.URI; import java.net.InetSocketAddress; +import java.net.URI; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -39,64 +42,64 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; -import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; -import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER; +import com.google.common.annotations.VisibleForTesting; /** - * Class that manages adding and removing aggregators and their lifecycle. It - * provides thread safety access to the aggregators inside. + * Class that manages adding and removing collectors and their lifecycle. It + * provides thread safety access to the collectors inside. * * It is a singleton, and instances should be obtained via * {@link #getInstance()}. */ @Private @Unstable -public class TimelineAggregatorsCollection extends CompositeService { +public class TimelineCollectorManager extends CompositeService { private static final Log LOG = - LogFactory.getLog(TimelineAggregatorsCollection.class); - private static final TimelineAggregatorsCollection INSTANCE = - new TimelineAggregatorsCollection(); + LogFactory.getLog(TimelineCollectorManager.class); + private static final TimelineCollectorManager INSTANCE = + new TimelineCollectorManager(); // access to this map is synchronized with the map itself - private final Map aggregators = + private final Map collectors = Collections.synchronizedMap( - new HashMap()); + new HashMap()); - // REST server for this aggregator collection + // REST server for this collector manager private HttpServer2 timelineRestServer; - + private String timelineRestServerBindAddress; - - private AggregatorNodemanagerProtocol nmAggregatorService; - - private InetSocketAddress nmAggregatorServiceAddress; - static final String AGGREGATOR_COLLECTION_ATTR_KEY = "aggregator.collection"; + private CollectorNodemanagerProtocol nmCollectorService; + + private InetSocketAddress nmCollectorServiceAddress; + + static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager"; - static TimelineAggregatorsCollection getInstance() { + static TimelineCollectorManager getInstance() { return INSTANCE; } - TimelineAggregatorsCollection() { - super(TimelineAggregatorsCollection.class.getName()); + @VisibleForTesting + protected TimelineCollectorManager() { + super(TimelineCollectorManager.class.getName()); } @Override public void serviceInit(Configuration conf) throws Exception { - this.nmAggregatorServiceAddress = conf.getSocketAddr( + this.nmCollectorServiceAddress = conf.getSocketAddr( YarnConfiguration.NM_BIND_HOST, - YarnConfiguration.NM_AGGREGATOR_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_PORT); - + YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT); + } - + @Override protected void serviceStart() throws Exception { startWebApp(); @@ -112,95 +115,95 @@ protected void serviceStop() throws Exception { } /** - * Put the aggregator into the collection if an aggregator mapped by id does + * Put the collector into the collection if an collector mapped by id does * not exist. * * @throws YarnRuntimeException if there was any exception in initializing and * starting the app level service - * @return the aggregator associated with id after the potential put. + * @return the collector associated with id after the potential put. */ - public TimelineAggregator putIfAbsent(ApplicationId appId, - TimelineAggregator aggregator) { + public TimelineCollector putIfAbsent(ApplicationId appId, + TimelineCollector collector) { String id = appId.toString(); - TimelineAggregator aggregatorInTable; - boolean aggregatorIsNew = false; - synchronized (aggregators) { - aggregatorInTable = aggregators.get(id); - if (aggregatorInTable == null) { + TimelineCollector collectorInTable; + boolean collectorIsNew = false; + synchronized (collectors) { + collectorInTable = collectors.get(id); + if (collectorInTable == null) { try { // initialize, start, and add it to the collection so it can be // cleaned up when the parent shuts down - aggregator.init(getConfig()); - aggregator.start(); - aggregators.put(id, aggregator); - LOG.info("the aggregator for " + id + " was added"); - aggregatorInTable = aggregator; - aggregatorIsNew = true; + collector.init(getConfig()); + collector.start(); + collectors.put(id, collector); + LOG.info("the collector for " + id + " was added"); + collectorInTable = collector; + collectorIsNew = true; } catch (Exception e) { throw new YarnRuntimeException(e); } } else { - String msg = "the aggregator for " + id + " already exists!"; + String msg = "the collector for " + id + " already exists!"; LOG.error(msg); } - + } - // Report to NM if a new aggregator is added. - if (aggregatorIsNew) { + // Report to NM if a new collector is added. + if (collectorIsNew) { try { - reportNewAggregatorToNM(appId); + reportNewCollectorToNM(appId); } catch (Exception e) { // throw exception here as it cannot be used if failed report to NM - LOG.error("Failed to report a new aggregator for application: " + appId + - " to NM Aggregator Services."); + LOG.error("Failed to report a new collector for application: " + appId + + " to the NM Collector Service."); throw new YarnRuntimeException(e); } } - - return aggregatorInTable; + + return collectorInTable; } /** - * Removes the aggregator for the specified id. The aggregator is also stopped - * as a result. If the aggregator does not exist, no change is made. + * Removes the collector for the specified id. The collector is also stopped + * as a result. If the collector does not exist, no change is made. * * @return whether it was removed successfully */ public boolean remove(String id) { - synchronized (aggregators) { - TimelineAggregator aggregator = aggregators.remove(id); - if (aggregator == null) { - String msg = "the aggregator for " + id + " does not exist!"; + synchronized (collectors) { + TimelineCollector collector = collectors.remove(id); + if (collector == null) { + String msg = "the collector for " + id + " does not exist!"; LOG.error(msg); return false; } else { // stop the service to do clean up - aggregator.stop(); - LOG.info("the aggregator service for " + id + " was removed"); + collector.stop(); + LOG.info("the collector service for " + id + " was removed"); return true; } } } /** - * Returns the aggregator for the specified id. + * Returns the collector for the specified id. * - * @return the aggregator or null if it does not exist + * @return the collector or null if it does not exist */ - public TimelineAggregator get(String id) { - return aggregators.get(id); + public TimelineCollector get(String id) { + return collectors.get(id); } /** - * Returns whether the aggregator for the specified id exists in this + * Returns whether the collector for the specified id exists in this * collection. */ public boolean containsKey(String id) { - return aggregators.containsKey(id); + return collectors.containsKey(id); } /** - * Launch the REST web server for this aggregator collection + * Launch the REST web server for this collector manager */ private void startWebApp() { Configuration conf = getConfig(); @@ -211,7 +214,7 @@ private void startWebApp() { WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress( NetUtils.createSocketAddr(bindAddress)); - LOG.info("Instantiating the per-node aggregator webapp at " + + LOG.info("Instantiating the per-node collector webapp at " + timelineRestServerBindAddress); try { Configuration confForInfoServer = new Configuration(conf); @@ -232,40 +235,44 @@ private void startWebApp() { options, new String[] {"/*"}); timelineRestServer.addJerseyResourcePackage( - TimelineAggregatorWebService.class.getPackage().getName() + ";" + TimelineCollectorWebService.class.getPackage().getName() + ";" + GenericExceptionHandler.class.getPackage().getName() + ";" + YarnJacksonJaxbJsonProvider.class.getPackage().getName(), "/*"); - timelineRestServer.setAttribute(AGGREGATOR_COLLECTION_ATTR_KEY, - TimelineAggregatorsCollection.getInstance()); + timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY, + TimelineCollectorManager.getInstance()); timelineRestServer.start(); } catch (Exception e) { - String msg = "The per-node aggregator webapp failed to start."; + String msg = "The per-node collector webapp failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } } - - private void reportNewAggregatorToNM(ApplicationId appId) + + private void reportNewCollectorToNM(ApplicationId appId) throws YarnException, IOException { - this.nmAggregatorService = getNMAggregatorService(); - ReportNewAggregatorsInfoRequest request = - ReportNewAggregatorsInfoRequest.newInstance(appId, + this.nmCollectorService = getNMCollectorService(); + ReportNewCollectorInfoRequest request = + ReportNewCollectorInfoRequest.newInstance(appId, this.timelineRestServerBindAddress); - LOG.info("Report a new aggregator for application: " + appId + - " to NM Aggregator Services."); - nmAggregatorService.reportNewAggregatorInfo(request); + LOG.info("Report a new collector for application: " + appId + + " to the NM Collector Service."); + nmCollectorService.reportNewCollectorInfo(request); } - - // protected for test - protected AggregatorNodemanagerProtocol getNMAggregatorService(){ + + @VisibleForTesting + protected CollectorNodemanagerProtocol getNMCollectorService() { Configuration conf = getConfig(); final YarnRPC rpc = YarnRPC.create(conf); - + // TODO Security settings. - return (AggregatorNodemanagerProtocol) rpc.getProxy( - AggregatorNodemanagerProtocol.class, - nmAggregatorServiceAddress, conf); + return (CollectorNodemanagerProtocol) rpc.getProxy( + CollectorNodemanagerProtocol.class, + nmCollectorServiceAddress, conf); + } + + @VisibleForTesting + public String getRestServerBindAddress() { + return timelineRestServerBindAddress; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java similarity index 82% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java index 7d42f94eb29a4..5adae7131f401 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java @@ -16,12 +16,18 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -36,7 +42,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.ForbiddenException; @@ -53,9 +58,9 @@ @Unstable @Singleton @Path("/ws/v2/timeline") -public class TimelineAggregatorWebService { +public class TimelineCollectorWebService { private static final Log LOG = - LogFactory.getLog(TimelineAggregatorWebService.class); + LogFactory.getLog(TimelineCollectorWebService.class); private @Context ServletContext context; @@ -95,12 +100,12 @@ public AboutInfo about( @Context HttpServletRequest req, @Context HttpServletResponse res) { init(res); - return new AboutInfo("Timeline API"); + return new AboutInfo("Timeline Collector API"); } /** - * Accepts writes to the aggregator, and returns a response. It simply routes - * the request to the app level aggregator. It expects an application as a + * Accepts writes to the collector, and returns a response. It simply routes + * the request to the app level collector. It expects an application as a * context. */ @PUT @@ -128,12 +133,12 @@ public Response putEntities( if (appId == null) { return Response.status(Response.Status.BAD_REQUEST).build(); } - TimelineAggregator service = getAggregatorService(req, appId); - if (service == null) { + TimelineCollector collector = getCollector(req, appId); + if (collector == null) { LOG.error("Application not found"); throw new NotFoundException(); // different exception? } - service.postEntities(entities, callerUgi); + collector.postEntities(entities, callerUgi); return Response.ok().build(); } catch (Exception e) { LOG.error("Error putting entities", e); @@ -143,8 +148,6 @@ public Response putEntities( } private String parseApplicationId(String appId) { - // Make sure the appId is not null and is valid - ApplicationId appID; try { if (appId != null) { return ConverterUtils.toApplicationId(appId.trim()).toString(); @@ -156,13 +159,13 @@ private String parseApplicationId(String appId) { } } - private TimelineAggregator - getAggregatorService(HttpServletRequest req, String appIdToParse) { + private TimelineCollector + getCollector(HttpServletRequest req, String appIdToParse) { String appIdString = parseApplicationId(appIdToParse); - final TimelineAggregatorsCollection aggregatorCollection = - (TimelineAggregatorsCollection) context.getAttribute( - TimelineAggregatorsCollection.AGGREGATOR_COLLECTION_ATTR_KEY); - return aggregatorCollection.get(appIdString); + final TimelineCollectorManager collectorManager = + (TimelineCollectorManager) context.getAttribute( + TimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY); + return collectorManager.get(appIdString); } private void init(HttpServletResponse response) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java index 4a57e9768cbdf..f5603f626cdd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java @@ -65,7 +65,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService * Stores the entire information in {@link TimelineEntity} to the * timeline store. Any errors occurring for individual write request objects * will be reported in the response. - * + * * @param data * a {@link TimelineEntity} object * @return {@link TimelineWriteResponse} object. @@ -116,10 +116,10 @@ private void write(TimelineEntity entity, * Aggregates the entity information to the timeline store based on which * track this entity is to be rolled up to The tracks along which aggregations * are to be done are given by {@link TimelineAggregationTrack} - * + * * Any errors occurring for individual write request objects will be reported * in the response. - * + * * @param data * a {@link TimelineEntity} object * a {@link TimelineAggregationTrack} enum value diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java similarity index 87% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java index 8f95814a0352f..74c81a75bfea6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java @@ -16,8 +16,8 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; -public class TestAppLevelTimelineAggregator { +public class TestAppLevelTimelineCollector { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java similarity index 73% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java index 1c89ead58f573..3b203523bfaac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,14 +32,15 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; import org.junit.Test; -public class TestPerNodeTimelineAggregatorsAuxService { +public class TestPerNodeTimelineCollectorsAuxService { private ApplicationAttemptId appAttemptId; - public TestPerNodeTimelineAggregatorsAuxService() { + public TestPerNodeTimelineCollectorsAuxService() { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -47,7 +48,8 @@ public TestPerNodeTimelineAggregatorsAuxService() { @Test public void testAddApplication() throws Exception { - PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication(); + PerNodeTimelineCollectorsAuxService auxService = + createCollectorAndAddApplication(); // auxService should have a single app assertTrue(auxService.hasApplication( appAttemptId.getApplicationId().toString())); @@ -56,7 +58,7 @@ public void testAddApplication() throws Exception { @Test public void testAddApplicationNonAMContainer() throws Exception { - PerNodeTimelineAggregatorsAuxService auxService = createAggregator(); + PerNodeTimelineCollectorsAuxService auxService = createCollector(); ContainerId containerId = getContainerId(2L); // not an AM ContainerInitializationContext context = @@ -70,7 +72,8 @@ public void testAddApplicationNonAMContainer() throws Exception { @Test public void testRemoveApplication() throws Exception { - PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication(); + PerNodeTimelineCollectorsAuxService auxService = + createCollectorAndAddApplication(); // auxService should have a single app String appIdStr = appAttemptId.getApplicationId().toString(); assertTrue(auxService.hasApplication(appIdStr)); @@ -87,7 +90,8 @@ public void testRemoveApplication() throws Exception { @Test public void testRemoveApplicationNonAMContainer() throws Exception { - PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication(); + PerNodeTimelineCollectorsAuxService auxService = + createCollectorAndAddApplication(); // auxService should have a single app String appIdStr = appAttemptId.getApplicationId().toString(); assertTrue(auxService.hasApplication(appIdStr)); @@ -105,10 +109,11 @@ public void testRemoveApplicationNonAMContainer() throws Exception { @Test(timeout = 60000) public void testLaunch() throws Exception { ExitUtil.disableSystemExit(); - PerNodeTimelineAggregatorsAuxService auxService = null; + PerNodeTimelineCollectorsAuxService auxService = null; try { auxService = - PerNodeTimelineAggregatorsAuxService.launchServer(new String[0]); + PerNodeTimelineCollectorsAuxService.launchServer(new String[0], + createCollectorManager()); } catch (ExitUtil.ExitException e) { assertEquals(0, e.status); ExitUtil.resetFirstExitException(); @@ -120,8 +125,9 @@ public void testLaunch() throws Exception { } } - private PerNodeTimelineAggregatorsAuxService createAggregatorAndAddApplication() { - PerNodeTimelineAggregatorsAuxService auxService = createAggregator(); + private PerNodeTimelineCollectorsAuxService + createCollectorAndAddApplication() { + PerNodeTimelineCollectorsAuxService auxService = createCollector(); // create an AM container ContainerId containerId = getAMContainerId(); ContainerInitializationContext context = @@ -131,15 +137,23 @@ private PerNodeTimelineAggregatorsAuxService createAggregatorAndAddApplication() return auxService; } - private PerNodeTimelineAggregatorsAuxService createAggregator() { - TimelineAggregatorsCollection - aggregatorsCollection = spy(new TimelineAggregatorsCollection()); - doReturn(new Configuration()).when(aggregatorsCollection).getConfig(); - PerNodeTimelineAggregatorsAuxService auxService = - spy(new PerNodeTimelineAggregatorsAuxService(aggregatorsCollection)); + private PerNodeTimelineCollectorsAuxService createCollector() { + TimelineCollectorManager collectorManager = createCollectorManager(); + PerNodeTimelineCollectorsAuxService auxService = + spy(new PerNodeTimelineCollectorsAuxService(collectorManager)); return auxService; } + private TimelineCollectorManager createCollectorManager() { + TimelineCollectorManager collectorManager = + spy(new TimelineCollectorManager()); + doReturn(new Configuration()).when(collectorManager).getConfig(); + CollectorNodemanagerProtocol nmCollectorService = + mock(CollectorNodemanagerProtocol.class); + doReturn(nmCollectorService).when(collectorManager).getNMCollectorService(); + return collectorManager; + } + private ContainerId getAMContainerId() { return getContainerId(1L); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java similarity index 65% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java index dd64629e3b0a9..541665be16d1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java @@ -16,11 +16,12 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.server.timelineservice.aggregator; +package org.apache.hadoop.yarn.server.timelineservice.collector; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import java.util.ArrayList; @@ -30,18 +31,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.junit.Test; -public class TestTimelineAggregatorsCollection { +public class TestTimelineCollectorManager { @Test(timeout=60000) public void testMultithreadedAdd() throws Exception { - final TimelineAggregatorsCollection aggregatorCollection = - spy(new TimelineAggregatorsCollection()); - doReturn(new Configuration()).when(aggregatorCollection).getConfig(); + final TimelineCollectorManager collectorManager = createCollectorManager(); final int NUM_APPS = 5; List> tasks = new ArrayList>(); @@ -49,9 +48,9 @@ public void testMultithreadedAdd() throws Exception { final ApplicationId appId = ApplicationId.newInstance(0L, i); Callable task = new Callable() { public Boolean call() { - AppLevelTimelineAggregator aggregator = - new AppLevelTimelineAggregator(appId.toString()); - return (aggregatorCollection.putIfAbsent(appId, aggregator) == aggregator); + AppLevelTimelineCollector collector = + new AppLevelTimelineCollector(appId.toString()); + return (collectorManager.putIfAbsent(appId, collector) == collector); } }; tasks.add(task); @@ -67,15 +66,14 @@ public Boolean call() { } // check the keys for (int i = 0; i < NUM_APPS; i++) { - assertTrue(aggregatorCollection.containsKey(String.valueOf(i))); + final ApplicationId appId = ApplicationId.newInstance(0L, i); + assertTrue(collectorManager.containsKey(appId.toString())); } } @Test public void testMultithreadedAddAndRemove() throws Exception { - final TimelineAggregatorsCollection aggregatorCollection = - spy(new TimelineAggregatorsCollection()); - doReturn(new Configuration()).when(aggregatorCollection).getConfig(); + final TimelineCollectorManager collectorManager = createCollectorManager(); final int NUM_APPS = 5; List> tasks = new ArrayList>(); @@ -83,11 +81,11 @@ public void testMultithreadedAddAndRemove() throws Exception { final ApplicationId appId = ApplicationId.newInstance(0L, i); Callable task = new Callable() { public Boolean call() { - AppLevelTimelineAggregator aggregator = - new AppLevelTimelineAggregator(appId.toString()); + AppLevelTimelineCollector collector = + new AppLevelTimelineCollector(appId.toString()); boolean successPut = - (aggregatorCollection.putIfAbsent(appId, aggregator) == aggregator); - return successPut && aggregatorCollection.remove(appId.toString()); + (collectorManager.putIfAbsent(appId, collector) == collector); + return successPut && collectorManager.remove(appId.toString()); } }; tasks.add(task); @@ -103,7 +101,18 @@ public Boolean call() { } // check the keys for (int i = 0; i < NUM_APPS; i++) { - assertFalse(aggregatorCollection.containsKey(String.valueOf(i))); + final ApplicationId appId = ApplicationId.newInstance(0L, i); + assertFalse(collectorManager.containsKey(appId.toString())); } } + + private TimelineCollectorManager createCollectorManager() { + final TimelineCollectorManager collectorManager = + spy(new TimelineCollectorManager()); + doReturn(new Configuration()).when(collectorManager).getConfig(); + CollectorNodemanagerProtocol nmCollectorService = + mock(CollectorNodemanagerProtocol.class); + doReturn(nmCollectorService).when(collectorManager).getNMCollectorService(); + return collectorManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java index f72045460963d..7f919f02d80f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java @@ -27,12 +27,12 @@ import java.nio.file.Paths; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.junit.Test; -import org.apache.commons.io.FileUtils; public class TestFileSystemTimelineWriterImpl { @@ -42,9 +42,6 @@ public class TestFileSystemTimelineWriterImpl { */ @Test public void testWriteEntityToFile() throws Exception { - String name = "unit_test_BaseAggregator_testWriteEntityToFile_" - + Long.toString(System.currentTimeMillis()); - TimelineEntities te = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); String id = "hello"; @@ -55,25 +52,27 @@ public void testWriteEntityToFile() throws Exception { entity.setModifiedTime(1425016502000L); te.addEntity(entity); - FileSystemTimelineWriterImpl fsi = new FileSystemTimelineWriterImpl(); - fsi.serviceInit(new Configuration()); - fsi.write(te); + try (FileSystemTimelineWriterImpl fsi = + new FileSystemTimelineWriterImpl()) { + fsi.serviceInit(new Configuration()); + fsi.write(te); - String fileName = fsi.getOutputRoot() + "/" + type + "/" + id - + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; - Path path = Paths.get(fileName); - File f = new File(fileName); - assertTrue(f.exists() && !f.isDirectory()); - List data = Files.readAllLines(path, StandardCharsets.UTF_8); - // ensure there's only one entity + 1 new line - assertTrue(data.size() == 2); - String d = data.get(0); - // confirm the contents same as what was written - assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity)); + String fileName = fsi.getOutputRoot() + "/" + type + "/" + id + + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; + Path path = Paths.get(fileName); + File f = new File(fileName); + assertTrue(f.exists() && !f.isDirectory()); + List data = Files.readAllLines(path, StandardCharsets.UTF_8); + // ensure there's only one entity + 1 new line + assertTrue(data.size() == 2); + String d = data.get(0); + // confirm the contents same as what was written + assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity)); - // delete the directory - File outputDir = new File(fsi.getOutputRoot()); - FileUtils.deleteDirectory(outputDir); - assertTrue(!(f.exists())); + // delete the directory + File outputDir = new File(fsi.getOutputRoot()); + FileUtils.deleteDirectory(outputDir); + assertTrue(!(f.exists())); + } } }