Skip to content
This repository has been archived by the owner on Apr 4, 2021. It is now read-only.

Commit

Permalink
FALCON-2260 Enhance extension List api to support for user extensions
Browse files Browse the repository at this point in the history
  • Loading branch information
@sandeepSamudrala
sandeepSamudrala committed Jan 18, 2017
1 parent d317d3d commit c949059
Show file tree
Hide file tree
Showing 7 changed files with 26 additions and 86 deletions.
5 changes: 1 addition & 4 deletions cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,10 +152,7 @@ void extensionCommand(CommandLine commandLine, FalconClient client) throws IOExc
} else if (optionsList.contains(FalconCLIConstants.LIST_OPT)) {
validateRequiredParameter(extensionName, EXTENSION_NAME_OPT);
ExtensionJobList jobs = client.listExtensionJob(extensionName, doAsUser,
commandLine.getOptionValue(FalconCLIConstants.SORT_ORDER_OPT),
commandLine.getOptionValue(FalconCLIConstants.OFFSET_OPT),
commandLine.getOptionValue(FalconCLIConstants.NUM_RESULTS_OPT),
commandLine.getOptionValue(FalconCLIConstants.FIELDS_OPT));
commandLine.getOptionValue(FalconCLIConstants.SORT_ORDER_OPT));
result = jobs != null ? jobs.toString() : "No extension job (" + extensionName + ") found.";
} else if (optionsList.contains(INSTANCES_OPT)) {
validateRequiredParameter(jobName, JOB_NAME_OPT);
Expand Down
6 changes: 1 addition & 5 deletions client/src/main/java/org/apache/falcon/client/FalconClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1247,15 +1247,11 @@ public APIResult deleteExtensionJob(final String jobName, final String doAsUser)
}

public ExtensionJobList listExtensionJob(final String extensionName, final String doAsUser,
final String sortOrder, final String offset,
final String numResults, final String fields) {
final String sortOrder) {
ClientResponse clientResponse = new ResourceBuilder()
.path(ExtensionOperations.LIST.path, extensionName)
.addQueryParam(DO_AS_OPT, doAsUser)
.addQueryParam(FIELDS, fields)
.addQueryParam(SORT_ORDER, sortOrder)
.addQueryParam(OFFSET, offset)
.addQueryParam(NUM_RESULTS, numResults)
.call(ExtensionOperations.LIST);
return getResponse(ExtensionJobList.class, clientResponse);
}
Expand Down
53 changes: 10 additions & 43 deletions common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@
public class ExtensionJobList {

@XmlElement
public int numJobs;
int numJobs;

@XmlElementWrapper(name = "jobs")
public List<JobElement> job;
public List<String> job;

public ExtensionJobList() {
numJobs = 0;
Expand All @@ -45,54 +45,21 @@ public ExtensionJobList() {

public ExtensionJobList(int numJobs) {
this.numJobs = numJobs;
job = new ArrayList<JobElement>();
job = new ArrayList<>();
}

public ExtensionJobList(int numJobs, List<JobElement> elements) {
public ExtensionJobList(int numJobs, List<String> extensionJobNames) {
this.numJobs = numJobs;
this.job = elements;
}

public void addJob(JobElement element) {
job.add(element);
this.job = extensionJobNames;
}

@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
buffer.append(numJobs + "\n\n");
for (JobElement element : job) {
buffer.append(element.toString());
}
return buffer.toString();
}

/**
* Element for a job.
*/
public static class JobElement {
@XmlElement
public String jobName;

@XmlElement
public EntityList jobEntities;

public JobElement() {
jobName = null;
jobEntities = null;
}

public JobElement(String name, EntityList entities) {
jobName = name;
jobEntities = entities;
}

@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
buffer.append("Job: " + jobName + ", #. entities: ");
buffer.append(jobEntities.toString() + "\n");
return buffer.toString();
StringBuilder builder = new StringBuilder();
builder.append(numJobs).append("\n");
for (String extensionJobNames : job) {
builder.append(extensionJobNames);
}
return builder.toString();
}
}
10 changes: 10 additions & 0 deletions prism/src/main/java/org/apache/falcon/resource/AbstractExtensionManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,16 @@ protected static void checkIfExtensionIsEnabled(String extensionName) {
}
}

protected static void checkIfExtensionExists(String extensionName) {
ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
ExtensionBean extensionBean = metaStore.getDetail(extensionName);
if (extensionBean == null) {
LOG.error("Extension not found: " + extensionName);
throw FalconWebException.newAPIException("Extension Job not found:" + extensionName,
Response.Status.NOT_FOUND);
}
}

protected static void checkIfExtensionJobNameExists(String jobName, String extensionName) {
ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
ExtensionJobsBean extensionJobsBean = metaStore.getExtensionJobDetails(jobName);
Expand Down
33 changes: 4 additions & 29 deletions prism/src/main/java/org/apache/falcon/resource/proxy/ExtensionManagerProxy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import org.apache.falcon.resource.InstancesResult;
import org.apache.falcon.resource.APIResult;
import org.apache.falcon.resource.AbstractExtensionManager;
import org.apache.falcon.resource.EntityList;
import org.apache.falcon.resource.ExtensionInstanceList;
import org.apache.falcon.resource.ExtensionJobList;
import org.apache.falcon.security.CurrentUser;
Expand Down Expand Up @@ -103,45 +102,21 @@ public class ExtensionManagerProxy extends AbstractExtensionManager {
@Produces({MediaType.TEXT_XML, MediaType.APPLICATION_JSON})
public ExtensionJobList getExtensionJobs(
@PathParam("extension-name") String extensionName,
@DefaultValue("") @QueryParam("fields") String fields,
@DefaultValue(ASCENDING_SORT_ORDER) @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
@QueryParam("numResults") Integer resultsPerPage,
@DefaultValue("") @QueryParam("doAs") String doAsUser) {
checkIfExtensionServiceIsEnabled();
resultsPerPage = resultsPerPage == null ? getDefaultResultsPerPage() : resultsPerPage;
checkIfExtensionExists(extensionName);
try {
// get filtered entities
List<Entity> entities = getEntityList("", "", "", TAG_PREFIX_EXTENSION_NAME + extensionName, "", doAsUser);
if (entities.isEmpty()) {
return new ExtensionJobList(0);
}

// group entities by extension job name
Map<String, List<Entity>> groupedEntities = groupEntitiesByJob(entities);

// sort by extension job name
List<String> jobNames = new ArrayList<>(groupedEntities.keySet());
List<String> jobNames = ExtensionStore.get().getJobsForAnExtension(extensionName);
switch (sortOrder.toLowerCase()) {
case DESCENDING_SORT_ORDER:
Collections.sort(jobNames, Collections.reverseOrder(String.CASE_INSENSITIVE_ORDER));
break;
default:
Collections.sort(jobNames, String.CASE_INSENSITIVE_ORDER);
}

// pagination and format output
int pageCount = getRequiredNumberOfResults(jobNames.size(), offset, resultsPerPage);
HashSet<String> fieldSet = new HashSet<>(Arrays.asList(fields.toUpperCase().split(",")));
ExtensionJobList jobList = new ExtensionJobList(pageCount);
for (int i = offset; i < offset + pageCount; i++) {
String jobName = jobNames.get(i);
List<Entity> jobEntities = groupedEntities.get(jobName);
EntityList entityList = new EntityList(buildEntityElements(fieldSet, jobEntities), jobEntities.size());
jobList.addJob(new ExtensionJobList.JobElement(jobName, entityList));
}
return jobList;
} catch (FalconException | IOException e) {
return new ExtensionJobList(jobNames.size(), jobNames);
} catch (FalconException e) {
LOG.error("Failed to get extension job list of " + extensionName + ": ", e);
throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
Expand Down
2 changes: 0 additions & 2 deletions webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,6 @@ public void testTrustedExtensionJob() throws Exception {
// validate job list results
ExtensionJobList jobs = context.getExtensionJobs("hdfs-mirroring", null, null, null, null, null);
Assert.assertEquals(jobs.numJobs, 2);
Assert.assertEquals(jobs.job.get(0).jobName, JOB_NAME_1);
Assert.assertEquals(jobs.job.get(1).jobName, JOB_NAME_2);

// list extension job instances
System.out.println("extension -instances -jobName " + JOB_NAME_1 + " -fields status,clusters,tags");
Expand Down
3 changes: 0 additions & 3 deletions webapp/src/test/java/org/apache/falcon/resource/TestContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,9 +294,6 @@ public ExtensionJobList getExtensionJobs(String extensionName, String fields, St
String resultsPerPage, String doAsUser) {
WebResource resource = this.service.path("api/extension/list/" + extensionName);
resource = addQueryParam(resource, "doAs", doAsUser);
resource = addQueryParam(resource, "fields", fields);
resource = addQueryParam(resource, "sortOrder", sortOrder);
resource = addQueryParam(resource, "offset", offset);
resource = addQueryParam(resource, "numResults", resultsPerPage);
ClientResponse response = resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken)
.accept(MediaType.APPLICATION_JSON).type(MediaType.TEXT_XML)
Expand Down

0 comments on commit c949059

Please sign in to comment.