Skip to content

Commit

Permalink
YARN-5993. Allow native services quicklinks to be exported for each c…
Browse files Browse the repository at this point in the history 
…omponent. Contributed by Billie Rinaldi
  • Loading branch information
@gsaha @jian-he
gsaha authored and jian-he committed Nov 6, 2017
1 parent 7d685f2 commit a5e20f0
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 173 deletions.
8 changes: 4 additions & 4 deletions ...der/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -183,17 +183,17 @@ public static boolean isSet(String s) {
return !isUnset(s); return !isUnset(s);
} }


public static boolean isEmpty(List l) { public static boolean isEmpty(Collection l) {
return l == null || l.isEmpty(); return l == null || l.isEmpty();
} }


/** /**
* Probe for a list existing and not being empty * Probe for a collection existing and not being empty
* @param l list * @param l collection
* @return true if the reference is valid and it contains entries * @return true if the reference is valid and it contains entries
*/ */


public static boolean isNotEmpty(List l) { public static boolean isNotEmpty(Collection l) {
return l != null && !l.isEmpty(); return l != null && !l.isEmpty();
} }


Expand Down
20 changes: 20 additions & 0 deletions ...-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -106,6 +106,26 @@ public void setValidUntil(String validUntil) {
this.validUntil = validUntil; this.validUntil = validUntil;
} }


@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

ExportEntry that = (ExportEntry) o;

if (value != null ? !value.equals(that.value) : that.value != null)
return false;
return containerId != null ? containerId.equals(that.containerId) :
that.containerId == null;
}

@Override
public int hashCode() {
int result = value != null ? value.hashCode() : 0;
result = 31 * result + (containerId != null ? containerId.hashCode() : 0);
return result;
}

@Override @Override
public String toString() { public String toString() {
return new StringBuilder("ExportEntry{"). return new StringBuilder("ExportEntry{").
Expand Down
23 changes: 16 additions & 7 deletions ...-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;


/** /**
* JSON-serializable description of a published key-val configuration. * JSON-serializable description of a published key-val configuration.
Expand All @@ -41,7 +43,7 @@ public class PublishedExports {
public String description; public String description;
public long updated; public long updated;
public String updatedTime; public String updatedTime;
public Map<String, List<ExportEntry>> entries = new HashMap<>(); public Map<String, Set<ExportEntry>> entries = new HashMap<>();


public PublishedExports() { public PublishedExports() {
} }
Expand All @@ -62,7 +64,7 @@ public PublishedExports(String description) {
* @param entries entries to put * @param entries entries to put
*/ */
public PublishedExports(String description, public PublishedExports(String description,
Iterable<Map.Entry<String, List<ExportEntry>>> entries) { Iterable<Entry<String, Set<ExportEntry>>> entries) {
this.description = description; this.description = description;
putValues(entries); putValues(entries);
} }
Expand All @@ -87,15 +89,22 @@ public void setUpdated(long updated) {
this.updatedTime = new Date(updated).toString(); this.updatedTime = new Date(updated).toString();
} }



public Map<String, Set<ExportEntry>> sortedEntries() {
Map<String, Set<ExportEntry>> sortedEntries = new TreeMap<>();
sortedEntries.putAll(entries);
return sortedEntries;
}

/** /**
* Set the values from an iterable (this includes a Hadoop Configuration and Java properties * Set the values from an iterable (this includes a Hadoop Configuration and Java properties
* object). Any existing value set is discarded * object). Any existing value set is discarded
* *
* @param entries entries to put * @param values values to put
*/ */
public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) { public void putValues(Iterable<Map.Entry<String, Set<ExportEntry>>> values) {
this.entries = new HashMap<String, List<ExportEntry>>(); this.entries = new HashMap<>();
for (Map.Entry<String, List<ExportEntry>> entry : entries) { for (Map.Entry<String, Set<ExportEntry>> entry : values) {
this.entries.put(entry.getKey(), entry.getValue()); this.entries.put(entry.getKey(), entry.getValue());
} }
} }
Expand Down
133 changes: 31 additions & 102 deletions ...ider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern; import java.util.regex.Pattern;


/** /**
Expand Down Expand Up @@ -1053,32 +1054,6 @@ public Iterable<String> getHostsList(Collection<ClusterNode> values,
return hosts; return hosts;
} }


/**
* Return a list of hostnames based on current ClusterNodes.
* @param values cluster nodes
* @return list of hosts
*/
public Iterable<String> getHostNamesList(Collection<ClusterNode> values) {
List<String> hosts = new ArrayList<>();
for (ClusterNode cn : values) {
hosts.add(cn.hostname);
}
return hosts;
}

/**
* Return a list of IPs based on current ClusterNodes.
* @param values cluster nodes
* @return list of hosts
*/
public Iterable<String> getIPsList(Collection<ClusterNode> values) {
List<String> hosts = new ArrayList<>();
for (ClusterNode cn : values) {
hosts.add(cn.ip);
}
return hosts;
}

/** /**
* Update ServiceRecord in Registry with IP and hostname. * Update ServiceRecord in Registry with IP and hostname.
* @param amState access to AM state * @param amState access to AM state
Expand Down Expand Up @@ -1148,27 +1123,30 @@ public void publishApplicationInstanceData(String name, String description,
* Publish an export group. * Publish an export group.
* @param exportGroup export groups * @param exportGroup export groups
* @param amState access to AM state * @param amState access to AM state
* @param roleGroup component group * @param groupName export group name
*/ */
public void publishExportGroup(Map<String, List<ExportEntry>> exportGroup, public void publishExportGroup(
StateAccessForProviders amState, String roleGroup) { Map<String, Set<ExportEntry>> exportGroup,
StateAccessForProviders amState, String groupName) {
// Publish in old format for the time being // Publish in old format for the time being
Map<String, String> simpleEntries = new HashMap<>(); Map<String, String> simpleEntries = new HashMap<>();
for (Entry<String, List<ExportEntry>> entry : exportGroup.entrySet()) { for (Entry<String, Set<ExportEntry>> entry : exportGroup.entrySet()) {
List<ExportEntry> exports = entry.getValue(); Set<ExportEntry> exports = entry.getValue();
if (SliderUtils.isNotEmpty(exports)) { if (SliderUtils.isNotEmpty(exports)) {
// there is no support for multiple exports per name, so extract only Set<String> values = new TreeSet<>();
// the first one for (ExportEntry export : exports) {
simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue()); values.add(export.getValue());
}
simpleEntries.put(entry.getKey(), StringUtils.join(",", values));
} }
} }
publishApplicationInstanceData(roleGroup, roleGroup, publishApplicationInstanceData(groupName, groupName,
simpleEntries.entrySet(), amState); simpleEntries.entrySet(), amState);


PublishedExports exports = new PublishedExports(roleGroup); PublishedExports exports = new PublishedExports(groupName);
exports.setUpdated(new Date().getTime()); exports.setUpdated(new Date().getTime());
exports.putValues(exportGroup.entrySet()); exports.putValues(exportGroup.entrySet());
amState.getPublishedExportsSet().put(roleGroup, exports); amState.getPublishedExportsSet().put(groupName, exports);
} }


public Map<String, String> getExports(ConfTreeOperations appConf, public Map<String, String> getExports(ConfTreeOperations appConf,
Expand All @@ -1179,75 +1157,26 @@ public Map<String, String> getExports(ConfTreeOperations appConf,
return exports; return exports;
} }


private static final String COMPONENT_TAG = "component"; public String getGroupKey(String roleGroup, ConfTreeOperations appConf) {
private static final String HOST_FOLDER_FORMAT = "%s:%s"; String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
private static final String CONTAINER_LOGS_TAG = "container_log_dirs"; return getNameOrGroupKey(rolePrefix, roleGroup);
private static final String CONTAINER_PWDS_TAG = "container_work_dirs"; }

/**
* Format the folder locations and publish in the registry service.
* @param folders folder information
* @param containerId container ID
* @param hostFqdn host FQDN
* @param componentName component name
*/
public void publishFolderPaths(Map<String, String> folders,
String containerId, String componentName, String hostFqdn,
StateAccessForProviders amState,
Map<String, ExportEntry> logFolderExports,
Map<String, ExportEntry> workFolderExports) {
Date now = new Date();
for (Map.Entry<String, String> entry : folders.entrySet()) {
ExportEntry exportEntry = new ExportEntry();
exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn,
entry.getValue()));
exportEntry.setContainerId(containerId);
exportEntry.setLevel(COMPONENT_TAG);
exportEntry.setTag(componentName);
exportEntry.setUpdatedTime(now.toString());
if (entry.getKey().equals("AGENT_LOG_ROOT") ||
entry.getKey().equals("LOG_DIR")) {
synchronized (logFolderExports) {
logFolderExports.put(containerId, exportEntry);
}
} else {
synchronized (workFolderExports) {
workFolderExports.put(containerId, exportEntry);
}
}
log.info("Updating log and pwd folders for container {}", containerId);
}

PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG);
exports.setUpdated(now.getTime());
synchronized (logFolderExports) {
updateExportsFromList(exports, logFolderExports);
}
amState.getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports);


exports = new PublishedExports(CONTAINER_PWDS_TAG); public String getNameKey(String roleName, String roleGroup,
exports.setUpdated(now.getTime()); ConfTreeOperations appConf) {
synchronized (workFolderExports) { String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
updateExportsFromList(exports, workFolderExports); return getNameOrGroupKey(rolePrefix, roleName);
}
amState.getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports);
} }


/** public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) {
* Update the export data from the map. if (!rolePrefix.isEmpty()) {
* @param exports published exports if (!roleNameOrGroup.startsWith(rolePrefix)) {
* @param folderExports folder exports log.warn("Something went wrong, {} doesn't start with {}",
*/ roleNameOrGroup, rolePrefix);
private void updateExportsFromList(PublishedExports exports, return null;
Map<String, ExportEntry> folderExports) {
Map<String, List<ExportEntry>> perComponentList = new HashMap<>();
for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet()) {
String componentName = logEntry.getValue().getTag();
if (!perComponentList.containsKey(componentName)) {
perComponentList.put(componentName, new ArrayList<ExportEntry>());
} }
perComponentList.get(componentName).add(logEntry.getValue()); roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length());
} }
exports.putValues(perComponentList.entrySet()); return roleNameOrGroup.toUpperCase(Locale.ENGLISH);
} }
} }
Loading

0 comments on commit a5e20f0

Please sign in to comment.