YARN-7218. Decouple YARN Services REST API namespace from RM. (Contri…

…buted by Eric Yang)
apache · Nov 17, 2017 · 0940e4f · 0940e4f
1 parent 5f0b238
commit 0940e4f
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 48 deletions.
diff --git a/.../hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/.../hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -62,8 +62,8 @@ public void testPathAnnotation() {
         this.apiServer.getClass().isAnnotationPresent(Path.class));
     final Path path = this.apiServer.getClass()
         .getAnnotation(Path.class);
-    assertEquals("The path has /ws/v1 annotation", path.value(),
-        "/ws/v1");
+    assertEquals("The path has /v1 annotation", path.value(),
+        "/v1");
   }
 
   @Test

diff --git a/...arn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/...arn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -26,6 +26,8 @@
 import java.util.List;
 import java.util.Objects;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
@@ -46,22 +48,53 @@
 @ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.")
 @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
 @XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
 @JsonInclude(JsonInclude.Include.NON_NULL)
 public class Component implements Serializable {
   private static final long serialVersionUID = -8430058381509087805L;
 
+  @JsonProperty("name")
   private String name = null;
+
+  @JsonProperty("dependencies")
   private List<String> dependencies = new ArrayList<String>();
+
+  @JsonProperty("readiness_check")
+  @XmlElement(name = "readiness_check")
   private ReadinessCheck readinessCheck = null;
+
+  @JsonProperty("artifact")
   private Artifact artifact = null;
+
+  @JsonProperty("launch_command")
+  @XmlElement(name = "launch_command")
   private String launchCommand = null;
+
+  @JsonProperty("resource")
   private Resource resource = null;
+
+  @JsonProperty("number_of_containers")
+  @XmlElement(name = "number_of_containers")
   private Long numberOfContainers = null;
+
+  @JsonProperty("run_privileged_container")
+  @XmlElement(name = "run_privileged_container")
   private Boolean runPrivilegedContainer = false;
+
+  @JsonProperty("placement_policy")
+  @XmlElement(name = "placement_policy")
   private PlacementPolicy placementPolicy = null;
+
+  @JsonProperty("state")
   private ComponentState state = ComponentState.FLEXING;
+
+  @JsonProperty("configuration")
   private Configuration configuration = new Configuration();
+
+  @JsonProperty("quicklinks")
   private List<String> quicklinks = new ArrayList<String>();
+
+  @JsonProperty("containers")
   private List<Container> containers =
       Collections.synchronizedList(new ArrayList<Container>());
 
@@ -74,7 +107,6 @@ public Component name(String name) {
   }
 
   @ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).")
-  @JsonProperty("name")
   public String getName() {
     return name;
   }
@@ -95,7 +127,6 @@ public Component dependencies(List<String> dependencies) {
   }
 
   @ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.")
-  @JsonProperty("dependencies")
   public List<String> getDependencies() {
     return dependencies;
   }
@@ -113,12 +144,10 @@ public Component readinessCheck(ReadinessCheck readinessCheck) {
   }
 
   @ApiModelProperty(example = "null", value = "Readiness check for this component.")
-  @JsonProperty("readiness_check")
   public ReadinessCheck getReadinessCheck() {
     return readinessCheck;
   }
 
-  @XmlElement(name = "readiness_check")
   public void setReadinessCheck(ReadinessCheck readinessCheck) {
     this.readinessCheck = readinessCheck;
   }
@@ -133,7 +162,6 @@ public Component artifact(Artifact artifact) {
   }
 
   @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.")
-  @JsonProperty("artifact")
   public Artifact getArtifact() {
     return artifact;
   }
@@ -153,12 +181,10 @@ public Component launchCommand(String launchCommand) {
   }
 
   @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
-  @JsonProperty("launch_command")
   public String getLaunchCommand() {
     return launchCommand;
   }
 
-  @XmlElement(name = "launch_command")
   public void setLaunchCommand(String launchCommand) {
     this.launchCommand = launchCommand;
   }
@@ -173,7 +199,6 @@ public Component resource(Resource resource) {
   }
 
   @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.")
-  @JsonProperty("resource")
   public Resource getResource() {
     return resource;
   }
@@ -192,18 +217,15 @@ public Component numberOfContainers(Long numberOfContainers) {
   }
 
   @ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.")
-  @JsonProperty("number_of_containers")
   public Long getNumberOfContainers() {
     return numberOfContainers;
   }
 
-  @XmlElement(name = "number_of_containers")
   public void setNumberOfContainers(Long numberOfContainers) {
     this.numberOfContainers = numberOfContainers;
   }
 
   @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
-  @JsonProperty("containers")
   public List<Container> getContainers() {
     return containers;
   }
@@ -237,12 +259,10 @@ public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
   }
 
   @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
-  @JsonProperty("run_privileged_container")
   public Boolean getRunPrivilegedContainer() {
     return runPrivilegedContainer;
   }
 
-  @XmlElement(name = "run_privileged_container")
   public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
     this.runPrivilegedContainer = runPrivilegedContainer;
   }
@@ -259,12 +279,10 @@ public Component placementPolicy(PlacementPolicy placementPolicy) {
   }
 
   @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.")
-  @JsonProperty("placement_policy")
   public PlacementPolicy getPlacementPolicy() {
     return placementPolicy;
   }
 
-  @XmlElement(name = "placement_policy")
   public void setPlacementPolicy(PlacementPolicy placementPolicy) {
     this.placementPolicy = placementPolicy;
   }
@@ -278,7 +296,6 @@ public Component configuration(Configuration configuration) {
   }
 
   @ApiModelProperty(example = "null", value = "Config properties for this component.")
-  @JsonProperty("configuration")
   public Configuration getConfiguration() {
     return configuration;
   }
@@ -297,7 +314,6 @@ public Component quicklinks(List<String> quicklinks) {
   }
 
   @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.")
-  @JsonProperty("quicklinks")
   public List<String> getQuicklinks() {
     return quicklinks;
   }
@@ -312,7 +328,6 @@ public Component state(ComponentState state) {
   }
 
   @ApiModelProperty(example = "null", value = "State of the component.")
-  @JsonProperty("state")
   public ComponentState getState() {
     return state;
   }

diff --git a/...ervices-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/...ervices-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,7 +25,11 @@
 import java.util.Map;
 import java.util.Objects;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlType;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -43,6 +47,8 @@
 @InterfaceStability.Unstable
 @ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.")
 @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
 public class ReadinessCheck implements Serializable {
   private static final long serialVersionUID = -3836839816887186801L;
 
@@ -68,8 +74,14 @@ public String toString() {
     }
   }
 
+  @JsonProperty("type")
+  @XmlElement(name = "type")
   private TypeEnum type = null;
+  @JsonProperty("properties")
+  @XmlElement(name = "properties")
   private Map<String, String> properties = new HashMap<String, String>();
+  @JsonProperty("artifact")
+  @XmlElement(name = "artifact")
   private Artifact artifact = null;
 
   /**
@@ -82,7 +94,6 @@ public ReadinessCheck type(TypeEnum type) {
   }
 
   @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
-  @JsonProperty("type")
   public TypeEnum getType() {
     return type;
   }
@@ -129,7 +140,6 @@ public ReadinessCheck artifact(Artifact artifact) {
   }
 
   @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.")
-  @JsonProperty("artifact")
   public Artifact getArtifact() {
     return artifact;
   }

diff --git a/...-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/...-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -26,6 +26,8 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import java.util.ArrayList;
@@ -43,6 +45,7 @@
 @ApiModel(description = "An Service resource has the following attributes.")
 @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
 @XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
     "lifetime", "containers" })
@@ -53,9 +56,15 @@ public class Service extends BaseResource {
   private String id = null;
   private Artifact artifact = null;
   private Resource resource = null;
+  @JsonProperty("launch_time")
+  @XmlElement(name = "launch_time")
   private Date launchTime = null;
+  @JsonProperty("number_of_running_containers")
+  @XmlElement(name = "number_of_running_containers")
   private Long numberOfRunningContainers = null;
   private Long lifetime = null;
+  @JsonProperty("placement_policy")
+  @XmlElement(name = "placement_policy")
   private PlacementPolicy placementPolicy = null;
   private List<Component> components = new ArrayList<>();
   private Configuration configuration = new Configuration();
@@ -148,12 +157,10 @@ public Service launchTime(Date launchTime) {
   }
 
   @ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.")
-  @JsonProperty("launch_time")
   public Date getLaunchTime() {
     return launchTime == null ? null : (Date) launchTime.clone();
   }
 
-  @XmlElement(name = "launch_time")
   public void setLaunchTime(Date launchTime) {
     this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
   }
@@ -171,12 +178,10 @@ public Service numberOfRunningContainers(Long numberOfRunningContainers) {
   }
 
   @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
-  @JsonProperty("number_of_running_containers")
   public Long getNumberOfRunningContainers() {
     return numberOfRunningContainers;
   }
 
-  @XmlElement(name = "number_of_running_containers")
   public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
     this.numberOfRunningContainers = numberOfRunningContainers;
   }
@@ -215,12 +220,10 @@ public Service placementPolicy(PlacementPolicy placementPolicy) {
   }
 
   @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
-  @JsonProperty("placement_policy")
   public PlacementPolicy getPlacementPolicy() {
     return placementPolicy;
   }
 
-  @XmlElement(name = "placement_policy")
   public void setPlacementPolicy(PlacementPolicy placementPolicy) {
     this.placementPolicy = placementPolicy;
   }

diff --git a/...arn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/...arn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -20,7 +20,7 @@
 public interface RestApiConstants {
 
   // Rest endpoints
-  String CONTEXT_ROOT = "/ws/v1";
+  String CONTEXT_ROOT = "/v1";
   String VERSION = "/services/version";
   String SERVICE_ROOT_PATH = "/services";
   String SERVICE_PATH = "/services/{service_name}";

diff --git a/...t/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/...t/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -394,16 +394,22 @@ public WebApp start() {
     }
 
     public WebApp start(WebApp webapp) {
-      return start(webapp, null);
+      return start(webapp, null, null);
     }
 
-    public WebApp start(WebApp webapp, WebAppContext ui2Context) {
+    public WebApp start(WebApp webapp, WebAppContext ui2Context,
+        Map<String, String> services) {
       WebApp webApp = build(webapp);
       HttpServer2 httpServer = webApp.httpServer();
       if (ui2Context != null) {
         addFiltersForNewContext(ui2Context);
         httpServer.addHandlerAtFront(ui2Context);
       }
+      if (services!=null) {
+        String packageName = services.get("PackageName");
+        String pathSpec = services.get("PathSpec");
+        httpServer.addJerseyResourcePackage(packageName, pathSpec);
+      }
       try {
         httpServer.start();
         LOG.info("Web app " + name + " started at "

diff --git a/...emanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/...emanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -126,7 +126,9 @@
 import java.security.PrivilegedExceptionAction;
 import java.security.SecureRandom;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -1062,7 +1064,7 @@ public static HttpServer2.Builder httpServerTemplateForRM(Configuration conf,
   }
 
   protected void startWepApp() {
-
+    Map<String, String> serviceConfig = null;
     Configuration conf = getConfig();
 
     RMWebAppUtil.setupSecurityAndFilters(conf,
@@ -1128,7 +1130,15 @@ protected void startWepApp() {
       }
     }
 
-    webApp = builder.start(new RMWebApp(this), uiWebAppContext);
+    if (getConfig().getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
+        false)) {
+      serviceConfig = new HashMap<String, String>();
+      String apiPackages = "org.apache.hadoop.yarn.service.webapp;" +
+          "org.apache.hadoop.yarn.webapp";
+      serviceConfig.put("PackageName", apiPackages);
+      serviceConfig.put("PathSpec", "/app/*");
+    }
+    webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig);
   }
 
   private String getWebAppsPath(String appName) {