From 0940e4f692441f16e742666ac925f71a083eab27 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Fri, 17 Nov 2017 12:28:12 -0500 Subject: [PATCH] YARN-7218. Decouple YARN Services REST API namespace from RM. (Contributed by Eric Yang) --- .../hadoop/yarn/service/TestApiServer.java | 4 +- .../yarn/service/api/records/Component.java | 51 ++++++++++++------- .../service/api/records/ReadinessCheck.java | 14 ++++- .../yarn/service/api/records/Service.java | 15 +++--- .../yarn/service/conf/RestApiConstants.java | 2 +- .../apache/hadoop/yarn/webapp/WebApps.java | 10 +++- .../resourcemanager/ResourceManager.java | 14 ++++- .../resourcemanager/webapp/RMWebApp.java | 15 ------ 8 files changed, 77 insertions(+), 48 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java index 2b224747ae1ea..896b2f699b256 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java @@ -62,8 +62,8 @@ public void testPathAnnotation() { this.apiServer.getClass().isAnnotationPresent(Path.class)); final Path path = this.apiServer.getClass() .getAnnotation(Path.class); - assertEquals("The path has /ws/v1 annotation", path.value(), - "/ws/v1"); + assertEquals("The path has /v1 annotation", path.value(), + "/v1"); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java index fe9c043106cfa..ce0e0cfde8c81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.Objects; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; @@ -46,22 +48,53 @@ @ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.") @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") @XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) @JsonInclude(JsonInclude.Include.NON_NULL) public class Component implements Serializable { private static final long serialVersionUID = -8430058381509087805L; + @JsonProperty("name") private String name = null; + + @JsonProperty("dependencies") private List dependencies = new ArrayList(); + + @JsonProperty("readiness_check") + @XmlElement(name = "readiness_check") private ReadinessCheck readinessCheck = null; + + @JsonProperty("artifact") private Artifact artifact = null; + + @JsonProperty("launch_command") + @XmlElement(name = "launch_command") private String launchCommand = null; + + @JsonProperty("resource") private Resource resource = null; + + @JsonProperty("number_of_containers") + @XmlElement(name = "number_of_containers") private Long numberOfContainers = null; + + @JsonProperty("run_privileged_container") + @XmlElement(name = "run_privileged_container") private Boolean runPrivilegedContainer = false; + + @JsonProperty("placement_policy") + @XmlElement(name = "placement_policy") private PlacementPolicy placementPolicy = null; + + @JsonProperty("state") private ComponentState state = ComponentState.FLEXING; + + @JsonProperty("configuration") private Configuration configuration = new Configuration(); + + @JsonProperty("quicklinks") private List quicklinks = new ArrayList(); + + @JsonProperty("containers") private List containers = Collections.synchronizedList(new ArrayList()); @@ -74,7 +107,6 @@ public Component name(String name) { } @ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).") - @JsonProperty("name") public String getName() { return name; } @@ -95,7 +127,6 @@ public Component dependencies(List dependencies) { } @ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.") - @JsonProperty("dependencies") public List getDependencies() { return dependencies; } @@ -113,12 +144,10 @@ public Component readinessCheck(ReadinessCheck readinessCheck) { } @ApiModelProperty(example = "null", value = "Readiness check for this component.") - @JsonProperty("readiness_check") public ReadinessCheck getReadinessCheck() { return readinessCheck; } - @XmlElement(name = "readiness_check") public void setReadinessCheck(ReadinessCheck readinessCheck) { this.readinessCheck = readinessCheck; } @@ -133,7 +162,6 @@ public Component artifact(Artifact artifact) { } @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.") - @JsonProperty("artifact") public Artifact getArtifact() { return artifact; } @@ -153,12 +181,10 @@ public Component launchCommand(String launchCommand) { } @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).") - @JsonProperty("launch_command") public String getLaunchCommand() { return launchCommand; } - @XmlElement(name = "launch_command") public void setLaunchCommand(String launchCommand) { this.launchCommand = launchCommand; } @@ -173,7 +199,6 @@ public Component resource(Resource resource) { } @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.") - @JsonProperty("resource") public Resource getResource() { return resource; } @@ -192,18 +217,15 @@ public Component numberOfContainers(Long numberOfContainers) { } @ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.") - @JsonProperty("number_of_containers") public Long getNumberOfContainers() { return numberOfContainers; } - @XmlElement(name = "number_of_containers") public void setNumberOfContainers(Long numberOfContainers) { this.numberOfContainers = numberOfContainers; } @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.") - @JsonProperty("containers") public List getContainers() { return containers; } @@ -237,12 +259,10 @@ public Component runPrivilegedContainer(Boolean runPrivilegedContainer) { } @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).") - @JsonProperty("run_privileged_container") public Boolean getRunPrivilegedContainer() { return runPrivilegedContainer; } - @XmlElement(name = "run_privileged_container") public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) { this.runPrivilegedContainer = runPrivilegedContainer; } @@ -259,12 +279,10 @@ public Component placementPolicy(PlacementPolicy placementPolicy) { } @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.") - @JsonProperty("placement_policy") public PlacementPolicy getPlacementPolicy() { return placementPolicy; } - @XmlElement(name = "placement_policy") public void setPlacementPolicy(PlacementPolicy placementPolicy) { this.placementPolicy = placementPolicy; } @@ -278,7 +296,6 @@ public Component configuration(Configuration configuration) { } @ApiModelProperty(example = "null", value = "Config properties for this component.") - @JsonProperty("configuration") public Configuration getConfiguration() { return configuration; } @@ -297,7 +314,6 @@ public Component quicklinks(List quicklinks) { } @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.") - @JsonProperty("quicklinks") public List getQuicklinks() { return quicklinks; } @@ -312,7 +328,6 @@ public Component state(ComponentState state) { } @ApiModelProperty(example = "null", value = "State of the component.") - @JsonProperty("state") public ComponentState getState() { return state; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java index 0a3713c6d0aa3..2bcf68b8dc425 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java @@ -25,7 +25,11 @@ import java.util.Map; import java.util.Objects; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlEnum; +import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlType; import com.fasterxml.jackson.annotation.JsonProperty; @@ -43,6 +47,8 @@ @InterfaceStability.Unstable @ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.") @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) public class ReadinessCheck implements Serializable { private static final long serialVersionUID = -3836839816887186801L; @@ -68,8 +74,14 @@ public String toString() { } } + @JsonProperty("type") + @XmlElement(name = "type") private TypeEnum type = null; + @JsonProperty("properties") + @XmlElement(name = "properties") private Map properties = new HashMap(); + @JsonProperty("artifact") + @XmlElement(name = "artifact") private Artifact artifact = null; /** @@ -82,7 +94,6 @@ public ReadinessCheck type(TypeEnum type) { } @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).") - @JsonProperty("type") public TypeEnum getType() { return type; } @@ -129,7 +140,6 @@ public ReadinessCheck artifact(Artifact artifact) { } @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.") - @JsonProperty("artifact") public Artifact getArtifact() { return artifact; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java index 77a2610fe6c37..80458227a7a79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java @@ -26,6 +26,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import java.util.ArrayList; @@ -43,6 +45,7 @@ @ApiModel(description = "An Service resource has the following attributes.") @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") @XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ "name", "state", "resource", "number_of_containers", "lifetime", "containers" }) @@ -53,9 +56,15 @@ public class Service extends BaseResource { private String id = null; private Artifact artifact = null; private Resource resource = null; + @JsonProperty("launch_time") + @XmlElement(name = "launch_time") private Date launchTime = null; + @JsonProperty("number_of_running_containers") + @XmlElement(name = "number_of_running_containers") private Long numberOfRunningContainers = null; private Long lifetime = null; + @JsonProperty("placement_policy") + @XmlElement(name = "placement_policy") private PlacementPolicy placementPolicy = null; private List components = new ArrayList<>(); private Configuration configuration = new Configuration(); @@ -148,12 +157,10 @@ public Service launchTime(Date launchTime) { } @ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.") - @JsonProperty("launch_time") public Date getLaunchTime() { return launchTime == null ? null : (Date) launchTime.clone(); } - @XmlElement(name = "launch_time") public void setLaunchTime(Date launchTime) { this.launchTime = launchTime == null ? null : (Date) launchTime.clone(); } @@ -171,12 +178,10 @@ public Service numberOfRunningContainers(Long numberOfRunningContainers) { } @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.") - @JsonProperty("number_of_running_containers") public Long getNumberOfRunningContainers() { return numberOfRunningContainers; } - @XmlElement(name = "number_of_running_containers") public void setNumberOfRunningContainers(Long numberOfRunningContainers) { this.numberOfRunningContainers = numberOfRunningContainers; } @@ -215,12 +220,10 @@ public Service placementPolicy(PlacementPolicy placementPolicy) { } @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.") - @JsonProperty("placement_policy") public PlacementPolicy getPlacementPolicy() { return placementPolicy; } - @XmlElement(name = "placement_policy") public void setPlacementPolicy(PlacementPolicy placementPolicy) { this.placementPolicy = placementPolicy; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java index 35e1980170295..243fc5269bc1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java @@ -20,7 +20,7 @@ public interface RestApiConstants { // Rest endpoints - String CONTEXT_ROOT = "/ws/v1"; + String CONTEXT_ROOT = "/v1"; String VERSION = "/services/version"; String SERVICE_ROOT_PATH = "/services"; String SERVICE_PATH = "/services/{service_name}"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index 4f1cacf3aa4eb..d3ad53eb81086 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -394,16 +394,22 @@ public WebApp start() { } public WebApp start(WebApp webapp) { - return start(webapp, null); + return start(webapp, null, null); } - public WebApp start(WebApp webapp, WebAppContext ui2Context) { + public WebApp start(WebApp webapp, WebAppContext ui2Context, + Map services) { WebApp webApp = build(webapp); HttpServer2 httpServer = webApp.httpServer(); if (ui2Context != null) { addFiltersForNewContext(ui2Context); httpServer.addHandlerAtFront(ui2Context); } + if (services!=null) { + String packageName = services.get("PackageName"); + String pathSpec = services.get("PathSpec"); + httpServer.addJerseyResourcePackage(packageName, pathSpec); + } try { httpServer.start(); LOG.info("Web app " + name + " started at " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 727bc521740a2..6f8a0a48edbeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -126,7 +126,9 @@ import java.security.PrivilegedExceptionAction; import java.security.SecureRandom; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -1062,7 +1064,7 @@ public static HttpServer2.Builder httpServerTemplateForRM(Configuration conf, } protected void startWepApp() { - + Map serviceConfig = null; Configuration conf = getConfig(); RMWebAppUtil.setupSecurityAndFilters(conf, @@ -1128,7 +1130,15 @@ protected void startWepApp() { } } - webApp = builder.start(new RMWebApp(this), uiWebAppContext); + if (getConfig().getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, + false)) { + serviceConfig = new HashMap(); + String apiPackages = "org.apache.hadoop.yarn.service.webapp;" + + "org.apache.hadoop.yarn.webapp"; + serviceConfig.put("PackageName", apiPackages); + serviceConfig.put("PathSpec", "/app/*"); + } + webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig); } private String getWebAppsPath(String appName) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index bee9354037ccc..b8d439008035f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -44,8 +44,6 @@ public class RMWebApp extends WebApp implements YarnWebParams { LogFactory.getLog(RMWebApp.class.getName()); private final ResourceManager rm; private boolean standby = false; - private final static String APISERVER = - "org.apache.hadoop.yarn.service.webapp.ApiServer"; public RMWebApp(ResourceManager rm) { this.rm = rm; @@ -59,19 +57,6 @@ public void setup() { bind(RMWebApp.class).toInstance(this); if (rm != null) { - boolean enableServiceApi = rm.getConfig() - .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false); - if (enableServiceApi) { - try { - // Use reflection here to load ApiServer class, - // this is done to avoid creating cyclic dependency - // between maven projects. - Class apiServer = Class.forName(APISERVER); - bind(apiServer); - } catch (ClassNotFoundException e) { - LOG.warn("ApiServer REST API is not activated."); - } - } bind(ResourceManager.class).toInstance(rm); } route("/", RmController.class);