Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions hadoop-client-modules/hadoop-client-minicluster/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,13 @@
<exclude>**/*.java</exclude>
</excludes>
</filter>
<!-- Some of our dependencies include html, so remove it. -->
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>**/*.html</exclude>
</excludes>
</filter>
<!-- We pull in several test jars; keep out the actual test classes -->
<filter>
<artifact>*:*</artifact>
Expand Down
5 changes: 5 additions & 0 deletions hadoop-project/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2146,6 +2146,11 @@
<artifactId>jersey-media-json-jettison</artifactId>
<version>${jersey2.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-moxy</artifactId>
<version>${jersey2.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,10 @@
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jettison</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-moxy</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@

package org.apache.hadoop.yarn.server.resourcemanager.webapp;

import org.eclipse.persistence.jaxb.JAXBContextFactory;
import org.eclipse.persistence.jaxb.MarshallerProperties;
import org.glassfish.jersey.jettison.JettisonConfig;
import org.glassfish.jersey.jettison.JettisonJaxbContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;

import javax.inject.Inject;
import javax.inject.Singleton;
Expand All @@ -35,138 +38,45 @@
import javax.xml.bind.JAXBContext;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UsersInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerHealthInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInformationsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueAclsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueAclInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ContainerLaunchContextInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.jsonprovider.ClassSerialisationConfig;

@Singleton
@Provider
public class JAXBContextResolver implements ContextResolver<JAXBContext> {

private static final Logger LOG = LoggerFactory.getLogger(JAXBContextResolver.class.getName());

private final Map<Class, JAXBContext> typesContextMap;
private final Map<Class, JAXBContext> typesContextMap = new HashMap<>();

public JAXBContextResolver() throws Exception {
this(new Configuration());
}

@Inject
public JAXBContextResolver(@javax.inject.Named("conf") Configuration conf) throws Exception {

JAXBContext context;
JAXBContext unWrappedRootContext;

// you have to specify all the dao classes here
final Class[] cTypes =
{ AppInfo.class, AppAttemptInfo.class, AppAttemptsInfo.class,
ClusterInfo.class, CapacitySchedulerQueueInfo.class,
FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class,
UserMetricsInfo.class, CapacitySchedulerInfo.class,
ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class,
NodesInfo.class, RemoteExceptionData.class,
CapacitySchedulerQueueInfoList.class, ResourceInfo.class,
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class, ResourceInformationsInfo.class,
ActivitiesInfo.class, AppActivitiesInfo.class,
QueueAclsInfo.class, QueueAclInfo.class,
BulkActivitiesInfo.class};

// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
ContainerLaunchContextInfo.class, LocalResourceInfo.class,
DelegationToken.class, AppQueue.class, AppPriority.class,
ResourceOptionInfo.class };

ArrayList<Class> finalcTypesList = new ArrayList<>();
ArrayList<Class> finalRootUnwrappedTypesList = new ArrayList<>();

Collections.addAll(finalcTypesList, cTypes);
Collections.addAll(finalRootUnwrappedTypesList, rootUnwrappedTypes);

// Add Custom DAO Classes
Class[] daoClasses = null;
Class[] unwrappedDaoClasses = null;
boolean loadCustom = true;
try {
daoClasses = conf
.getClasses(YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES);
unwrappedDaoClasses = conf.getClasses(
YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES);
} catch (Exception e) {
LOG.warn("Failed to load custom dao class: ", e);
loadCustom = false;
}

if (loadCustom) {
if (daoClasses != null) {
Collections.addAll(finalcTypesList, daoClasses);
LOG.debug("Added custom dao classes: {}.", Arrays.toString(daoClasses));
}
if (unwrappedDaoClasses != null) {
Collections.addAll(finalRootUnwrappedTypesList, unwrappedDaoClasses);
LOG.debug("Added custom Unwrapped dao classes: {}", Arrays.toString(unwrappedDaoClasses));
}
}

final Class[] finalcTypes = finalcTypesList
.toArray(new Class[finalcTypesList.size()]);
final Class[] finalRootUnwrappedTypes = finalRootUnwrappedTypesList
.toArray(new Class[finalRootUnwrappedTypesList.size()]);

this.typesContextMap = new HashMap<>();
context = new JettisonJaxbContext(finalcTypes);
unWrappedRootContext = new JettisonJaxbContext(finalRootUnwrappedTypes);
for (Class type : finalcTypes) {
typesContextMap.put(type, context);
}
for (Class type : finalRootUnwrappedTypes) {
typesContextMap.put(type, unWrappedRootContext);
}
ClassSerialisationConfig classSerialisationConfig = new ClassSerialisationConfig(conf);
Set<Class<?>> wrappedClasses = classSerialisationConfig.getWrappedClasses();
Set<Class<?>> unWrappedClasses = classSerialisationConfig.getUnWrappedClasses();

//WARNING: AFAIK these properties not respected by MOXyJsonProvider
//For details check MOXyJsonProvider#readFrom method
JAXBContext wrappedContext = JAXBContextFactory.createContext(
wrappedClasses.toArray(new Class[0]),
Collections.singletonMap(MarshallerProperties.JSON_INCLUDE_ROOT, true)
);
JAXBContext unWrappedContext = JAXBContextFactory.createContext(
unWrappedClasses.toArray(new Class[0]),
Collections.singletonMap(MarshallerProperties.JSON_INCLUDE_ROOT, false)
);

wrappedClasses.forEach(type -> typesContextMap.put(type, wrappedContext));
unWrappedClasses.forEach(type -> typesContextMap.put(type, unWrappedContext));
}

@Override
public JAXBContext getContext(Class<?> objectType) {
return typesContextMap.get(objectType);
JAXBContext jaxbContext = typesContextMap.get(objectType);
LOG.trace("Context for {} is {}", objectType, jaxbContext);
return jaxbContext;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@
import java.net.InetSocketAddress;

import org.apache.hadoop.conf.Configuration;

import org.glassfish.jersey.moxy.json.MoxyJsonConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.jsonprovider.JsonProviderFeature;
import org.apache.hadoop.yarn.util.RMHAUtils;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
Expand Down Expand Up @@ -60,7 +63,8 @@ public ResourceConfig resourceConfig() {
config.register(new JerseyBinder());
config.register(RMWebServices.class);
config.register(GenericExceptionHandler.class);
config.register(new JettisonFeature()).register(JAXBContextResolver.class);
config.register(JsonProviderFeature.class);
config.register(JAXBContextResolver.class);
return config;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,59 +32,63 @@
public class NodeLabelsInfo {

@XmlElement(name = "nodeLabelInfo")
private ArrayList<NodeLabelInfo> nodeLabelsInfo = new ArrayList<>();
private ArrayList<NodeLabelInfo> nodeLabelInfo = new ArrayList<>();

public NodeLabelsInfo() {
// JAXB needs this
}

public NodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabels) {
this.nodeLabelsInfo = nodeLabels;
this.nodeLabelInfo = nodeLabels;
}

public NodeLabelsInfo(List<NodeLabel> nodeLabels) {
this.nodeLabelsInfo = new ArrayList<>();
this.nodeLabelInfo = new ArrayList<>();
for (NodeLabel label : nodeLabels) {
this.nodeLabelsInfo.add(new NodeLabelInfo(label));
this.nodeLabelInfo.add(new NodeLabelInfo(label));
}
}

public NodeLabelsInfo(Set<String> nodeLabelsName) {
this.nodeLabelsInfo = new ArrayList<>();
this.nodeLabelInfo = new ArrayList<>();
for (String labelName : nodeLabelsName) {
this.nodeLabelsInfo.add(new NodeLabelInfo(labelName));
this.nodeLabelInfo.add(new NodeLabelInfo(labelName));
}
}

public NodeLabelsInfo(Collection<NodeLabel> nodeLabels) {
this.nodeLabelsInfo = new ArrayList<>();
this.nodeLabelInfo = new ArrayList<>();
nodeLabels.stream().forEach(nodeLabel -> {
this.nodeLabelsInfo.add(new NodeLabelInfo(nodeLabel));
this.nodeLabelInfo.add(new NodeLabelInfo(nodeLabel));
});
}

public ArrayList<NodeLabelInfo> getNodeLabelsInfo() {
return nodeLabelsInfo;
return nodeLabelInfo;
}

public Set<NodeLabel> getNodeLabels() {
Set<NodeLabel> nodeLabels = new HashSet<>();
for (NodeLabelInfo label : nodeLabelsInfo) {
for (NodeLabelInfo label : nodeLabelInfo) {
nodeLabels.add(NodeLabel.newInstance(label.getName(),
label.getExclusivity()));
}
return nodeLabels;
}

public List<String> getNodeLabelsName() {
ArrayList<String> nodeLabelsName = new ArrayList<>();
for (NodeLabelInfo label : nodeLabelsInfo) {
for (NodeLabelInfo label : nodeLabelInfo) {
nodeLabelsName.add(label.getName());
}
return nodeLabelsName;
}

public void setNodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabelInfo) {
this.nodeLabelsInfo = nodeLabelInfo;
this.nodeLabelInfo = nodeLabelInfo;
}

public void setNodeLabelInfo(ArrayList<NodeLabelInfo> nodeLabelsInfo) {
this.nodeLabelInfo = nodeLabelsInfo;
}
}
Loading