From c2a526a1306900a9bbd3d70ff58b410ff64666dc Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Wed, 3 Aug 2022 05:07:09 +0800 Subject: [PATCH] YARN-8973. [Router] Add missing methods in RMWebProtocol. (#4664) --- .../webapp/RMWebServiceProtocol.java | 28 +++++++++++++++++++ .../webapp/DefaultRequestInterceptorREST.java | 20 +++++++++++++ .../webapp/FederationInterceptorREST.java | 14 ++++++++++ .../router/webapp/RouterWebServices.java | 28 +++++++++++++++++++ .../webapp/BaseRouterWebServicesTest.java | 12 ++++++++ .../webapp/MockRESTRequestInterceptor.java | 13 +++++++++ .../PassThroughRESTRequestInterceptor.java | 14 ++++++++++ .../router/webapp/TestRouterWebServices.java | 6 ++++ 8 files changed, 135 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 41fc4ea870985..6c0309969eebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** *

@@ -745,4 +746,31 @@ RMQueueAclInfo checkUserAccessToQueue(String queue, String username, */ Response signalToContainer(String containerId, String command, HttpServletRequest req) throws AuthorizationException; + + /** + * This method updates the Scheduler configuration, and it is reachable by + * using {@link RMWSConsts#SCHEDULER_CONF}. + * + * @param mutationInfo th information for making scheduler configuration + * changes (supports adding, removing, or updating a queue, as well + * as global scheduler conf changes) + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException if the user is not authorized to invoke this + * method + * @throws InterruptedException if interrupted + */ + Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) throws AuthorizationException, InterruptedException; + + /** + * This method retrieves all the Scheduler configuration, and it is reachable + * by using {@link RMWSConsts#SCHEDULER_CONF}. + * + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException if the user is not authorized to invoke this + * method. + */ + Response getSchedulerConfiguration(HttpServletRequest hsr) throws AuthorizationException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 21fd2be854618..c07056ce8a100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** @@ -565,6 +566,25 @@ public ContainerInfo getContainer(HttpServletRequest req, null, null, getConf(), client); } + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest req) + throws AuthorizationException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + Response.class, HTTPMethods.PUT, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF, + mutationInfo, null, getConf(), client); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest req) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + Response.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF, + null, null, getConf(), client); + } + @Override public void setNextInterceptor(RESTRequestInterceptor next) { throw new YarnRuntimeException("setNextInterceptor is being called on " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index bd5286021e178..2bc6d60b4e820 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -101,6 +101,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -1417,6 +1418,19 @@ public ContainerInfo getContainer(HttpServletRequest req, throw new NotImplementedException("Code is not implemented"); } + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + throw new NotImplementedException("Code is not implemented"); + } + @Override public void setNextInterceptor(RESTRequestInterceptor next) { throw new YarnRuntimeException("setNextInterceptor is being called on " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index 6beafc8183f6d..8faf0619f3f7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.LRUCacheHashMap; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -956,6 +957,33 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, appAttemptId, containerId); } + @PUT + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor() + .updateSchedulerConfiguration(mutationInfo, hsr); + } + + @GET + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor().getSchedulerConfiguration(hsr); + } + @VisibleForTesting protected void setResponse(HttpServletResponse response) { this.response = response; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index 025f44d4bd209..a4294bc36109a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -395,4 +395,16 @@ private HttpServletRequest createHttpServletRequest(String user) { when(request.getRemoteUser()).thenReturn(user); return request; } + + protected Response updateSchedulerConfiguration(String user) + throws IOException, InterruptedException { + return routerWebService.updateSchedulerConfiguration(null, + createHttpServletRequest(user)); + } + + protected Response getSchedulerConfiguration(String user) + throws IOException, InterruptedException { + return routerWebService. + getSchedulerConfiguration(createHttpServletRequest(user)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index 68b8db6f334c1..5951676a6d8e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * This class mocks the RESTRequestInterceptor. @@ -373,4 +374,16 @@ public Response signalToContainer(String containerId, String command, HttpServletRequest req) { return Response.status(Status.OK).build(); } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) throws AuthorizationException, InterruptedException { + return Response.status(Status.OK).build(); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + return Response.status(Status.OK).build(); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 219020f2bbd17..84a6de3205f5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * Mock interceptor that does not do anything other than forwarding it to the @@ -379,4 +380,17 @@ public Response signalToContainer(String containerId, String command, HttpServletRequest req) throws AuthorizationException { return getNextInterceptor().signalToContainer(containerId, command, req); } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + return getNextInterceptor().updateSchedulerConfiguration(mutationInfo, hsr); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + return getNextInterceptor().getSchedulerConfiguration(hsr); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java index 7491cbc2a9fc7..5bf8db04b06cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java @@ -200,6 +200,12 @@ public void testRouterWebServicesE2E() throws Exception { ContainerInfo containerInfo = getContainer(user); Assert.assertNotNull(containerInfo); + + Response response19 = updateSchedulerConfiguration(user); + Assert.assertNotNull(response19); + + Response response20 = getSchedulerConfiguration(user); + Assert.assertNotNull(response20); } /**