go -b FALCON-2263Merge branch 'master' of https://github.com/apache/f…

…alcon

cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java

@@ -152,10 +152,7 @@ void extensionCommand(CommandLine commandLine, FalconClient client) throws IOExc
         } else if (optionsList.contains(FalconCLIConstants.LIST_OPT)) {
             validateRequiredParameter(extensionName, EXTENSION_NAME_OPT);
             ExtensionJobList jobs = client.listExtensionJob(extensionName, doAsUser,
-                    commandLine.getOptionValue(FalconCLIConstants.SORT_ORDER_OPT),
-                    commandLine.getOptionValue(FalconCLIConstants.OFFSET_OPT),
-                    commandLine.getOptionValue(FalconCLIConstants.NUM_RESULTS_OPT),
-                    commandLine.getOptionValue(FalconCLIConstants.FIELDS_OPT));
+                    commandLine.getOptionValue(FalconCLIConstants.SORT_ORDER_OPT));
             result = jobs != null ? jobs.toString() : "No extension job (" + extensionName + ") found.";
         } else if (optionsList.contains(INSTANCES_OPT)) {
             validateRequiredParameter(jobName, JOB_NAME_OPT);

client/src/main/java/org/apache/falcon/ExtensionHandler.java

@@ -24,7 +24,7 @@
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.EntityType;
 import org.apache.falcon.extensions.ExtensionBuilder;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.falcon.hadoop.HadoopClientFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.codehaus.jettison.json.JSONException;
@@ -38,6 +38,8 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.util.ArrayList;
@@ -95,11 +97,10 @@ private List<Entity> getEntities(ClassLoader extensionClassloader, String extens
     }
 
     public static List<Entity> loadAndPrepare(String extensionName, String jobName, InputStream configStream,
-                                              String extensionBuildLocation) throws IOException, FalconException {
-        Configuration conf = new Configuration();
-        FileSystem fs = FileSystem.get(conf);
+                                              String extensionBuildLocation)
+        throws IOException, FalconException, URISyntaxException {
         String stagePath = createStagePath(extensionName, jobName);
-        List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation, fs, stagePath);
+        List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation, stagePath);
 
         List<Entity> entities = prepare(extensionName, jobName, configStream, urls);
         ExtensionHandler.stageEntities(entities, stagePath);
@@ -152,9 +153,10 @@ private static String createStagePath(String extensionName, String jobName) {
         return stagePath;
     }
 
-    private static List<URL> copyExtensionPackage(String extensionBuildUrl, FileSystem fs, String stagePath)
-        throws IOException {
+    private static List<URL> copyExtensionPackage(String extensionBuildUrl, String stagePath)
+        throws IOException, FalconException, URISyntaxException {
 
+        FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(new URI(extensionBuildUrl));
         Path libsPath = new Path(extensionBuildUrl, FalconExtensionConstants.LIBS);
         Path buildLibsPath = new Path(libsPath, FalconExtensionConstants.BUILD);
         Path localStagePath = new Path(stagePath);

client/src/main/java/org/apache/falcon/client/FalconClient.java

@@ -1247,15 +1247,11 @@ public APIResult deleteExtensionJob(final String jobName, final String doAsUser)
     }
 
     public ExtensionJobList listExtensionJob(final String extensionName, final String doAsUser,
-                                             final String sortOrder, final String offset,
-                                             final String numResults, final String fields)  {
+                                             final String sortOrder)  {
         ClientResponse clientResponse = new ResourceBuilder()
                 .path(ExtensionOperations.LIST.path, extensionName)
                 .addQueryParam(DO_AS_OPT, doAsUser)
-                .addQueryParam(FIELDS, fields)
                 .addQueryParam(SORT_ORDER, sortOrder)
-                .addQueryParam(OFFSET, offset)
-                .addQueryParam(NUM_RESULTS, numResults)
                 .call(ExtensionOperations.LIST);
         return getResponse(ExtensionJobList.class, clientResponse);
     }

common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java

@@ -33,10 +33,10 @@
 public class ExtensionJobList {
 
     @XmlElement
-    public int numJobs;
+    int numJobs;
 
     @XmlElementWrapper(name = "jobs")
-    public List<JobElement> job;
+    public List<String> job;
 
     public ExtensionJobList() {
         numJobs = 0;
@@ -45,54 +45,21 @@ public ExtensionJobList() {
 
     public ExtensionJobList(int numJobs) {
         this.numJobs = numJobs;
-        job = new ArrayList<JobElement>();
+        job = new ArrayList<>();
     }
 
-    public ExtensionJobList(int numJobs, List<JobElement> elements) {
+    public ExtensionJobList(int numJobs, List<String> extensionJobNames) {
         this.numJobs = numJobs;
-        this.job = elements;
-    }
-
-    public void addJob(JobElement element) {
-        job.add(element);
+        this.job = extensionJobNames;
     }
 
     @Override
     public String toString() {
-        StringBuilder buffer = new StringBuilder();
-        buffer.append(numJobs + "\n\n");
-        for (JobElement element : job) {
-            buffer.append(element.toString());
-        }
-        return buffer.toString();
-    }
-
-    /**
-     * Element for a job.
-     */
-    public static class JobElement {
-        @XmlElement
-        public String jobName;
-
-        @XmlElement
-        public EntityList jobEntities;
-
-        public JobElement() {
-            jobName = null;
-            jobEntities = null;
-        }
-
-        public JobElement(String name, EntityList entities) {
-            jobName = name;
-            jobEntities = entities;
-        }
-
-        @Override
-        public String toString() {
-            StringBuilder buffer = new StringBuilder();
-            buffer.append("Job: " + jobName + ", #. entities: ");
-            buffer.append(jobEntities.toString() + "\n");
-            return buffer.toString();
+        StringBuilder builder = new StringBuilder();
+        builder.append(numJobs).append("\n");
+        for (String extensionJobNames : job) {
+            builder.append(extensionJobNames);
         }
+        return builder.toString();
     }
 }

common/src/main/java/org/apache/falcon/persistence/ExtensionJobsBean.java

@@ -45,7 +45,7 @@
         @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSION_JOBS, query = "select OBJECT(a) from ExtensionJobsBean a "),
         @NamedQuery(name = PersistenceConstants.DELETE_EXTENSION_JOB, query = "delete from ExtensionJobsBean a where a.jobName = :jobName "),
         @NamedQuery(name = PersistenceConstants.GET_EXTENSION_JOB, query = "select OBJECT(a) from ExtensionJobsBean a where a.jobName = :jobName"),
-        @NamedQuery(name = PersistenceConstants.GET_JOBS_FOR_AN_EXTENSION, query = "select OBJECT(a) from ExtensionJobsBean a where a.extensionName = :extensionName")
+        @NamedQuery(name = PersistenceConstants.GET_JOBS_FOR_AN_EXTENSION, query = "select a.jobName from ExtensionJobsBean a where a.extensionName = :extensionName")
 })
 //RESUME CHECKSTYLE CHECK  LineLengthCheck
 public class ExtensionJobsBean {

docs/src/site/twiki/falconcli/ExtensionList.twiki

@@ -8,9 +8,9 @@ Usage:
 $FALCON_HOME/bin/falcon extension -list -extensionName <<extension-name>>
 
 Optional Args : -doAs <<user-name>>
--sortOrder <<sortOrder>> -offset <<offset-number>> -numResults <<number-of-results>> -fields <<field1,field2>>
+-sortOrder <<sortOrder>>
 
 <a href="../restapi/ExtensionList.html">Parameters and examples described here.</a>
 
 Example:
-$FALCON_HOME/bin/falcon extension -list -extensionName hdfs-mirroring -offset 0 -numResults 10
+$FALCON_HOME/bin/falcon extension -list -extensionName hdfs-mirroring

docs/src/site/twiki/restapi/ExtensionList.twiki

@@ -10,61 +10,24 @@ List jobs generated from an extension.
 ---++ Parameters
    * :extension-name Name of the extension.
    * sortOrder <optional> Sort order by job name. Valid options: "asc" (default) and "desc".
-   * offset <optional> Show results from the offset. Default is 0.
-   * numResults <optional> Number of results to show per request. Default is 10.
-   * fields <optional> Output fields separated by commas. Valid options: STATUS, TAGS, PIPELINES, CLUSTERS.
    * doAs <optional> Impersonate the user.
 
 ---++ Results
-Total number of results and a list of jobs generated from the extension, followed by the associated entities.
+Total number of results and a list of jobs implementing the given extension.
 
 ---++ Examples
 ---+++ Rest Call
 <verbatim>
-GET http://localhost:15000/api/extensions/list/billCollection?fields=status,clusters,tags
+GET http://localhost:15000/api/extensions/list/billCollection
 </verbatim>
 ---+++ Result
 <verbatim>
 {
   "numJobs":"2",
   "jobs":{
     "job": [
-      {
-        "jobName": "daily-health-bill",
-        "jobEntities: {
-          "totalResults": "2",
-          "entity": [
-            {
-              "type":"FEED",
-              "name":"SampleUSHealthBill",
-              "status":"SUBMITTED”,
-              "tags":{"tag":["related=ushealthcare","department=billingDepartment","_falcon_extension_name=billCoollection","_falcon_extension_job=daily-health-bill"]},
-              "clusters": {"cluster":["SampleCluster1","primaryCluster”]}
-            },
-            {
-              "type":"PROCESS”,
-              "name":"SampleBillPay”,
-              "status":"RUNNING”,
-              "tags":{"tag":["related=healthcare","department=billingDepartment","_falcon_extension_name=billCoollection","_falcon_extension_job=daily-health-bill"]},
-              "clusters":{"cluster":"primaryCluster”}
-            }
-          ]
-        }
-      },
-      {
-        "jobName": "fsa-bill",
-        "jobEntities": {
-          "totalResults": "1",
-          "entity":
-              {
-                "type":"PROCESS”,
-                "name":"FSAPay”,
-                "status":"RUNNING”,
-                "tags”:{"tag":["related=healthcare","department=billingDepartment","_falcon_extension_name=billCollection","_falcon_extension_job=fsa-bill"]},
-                "clusters":{"cluster":"primaryCluster”}
-              }
-        }
-      }
+      {"daily-health-bill"},
+      {"fsa-bill"}
     ]
   }
 }

extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java

@@ -27,6 +27,7 @@
 
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
@@ -74,10 +75,7 @@ public Boolean checkIfExtensionExists(String extensionName) {
         } finally {
             commitAndCloseTransaction(entityManager);
         }
-        if (resultSize > 0){
-            return true;
-        }
-        return false;
+        return resultSize > 0;
     }
 
     public Boolean checkIfExtensionJobExists(String jobName) {
@@ -91,18 +89,15 @@ public Boolean checkIfExtensionJobExists(String jobName) {
         } finally {
             commitAndCloseTransaction(entityManager);
         }
-        if (resultSize > 0){
-            return true;
-        }
-        return false;
+        return resultSize > 0;
     }
 
     public List<ExtensionBean> getAllExtensions() {
         EntityManager entityManager = getEntityManager();
         beginTransaction(entityManager);
         Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSIONS);
         try {
-            return (List<ExtensionBean>)q.getResultList();
+            return (List<ExtensionBean>) q.getResultList();
         } finally {
             commitAndCloseTransaction(entityManager);
         }
@@ -113,7 +108,7 @@ public void deleteExtensionsOfType(ExtensionType extensionType) {
         beginTransaction(entityManager);
         Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE);
         q.setParameter(EXTENSION_TYPE, extensionType);
-        try{
+        try {
             q.executeUpdate();
         } finally {
             commitAndCloseTransaction(entityManager);
@@ -128,7 +123,7 @@ public ExtensionBean getDetail(String extensionName) {
         try {
             List resultList = q.getResultList();
             if (!resultList.isEmpty()) {
-                return (ExtensionBean)resultList.get(0);
+                return (ExtensionBean) resultList.get(0);
             } else {
                 return null;
             }
@@ -137,24 +132,26 @@ public ExtensionBean getDetail(String extensionName) {
         }
     }
 
-    public List<ExtensionJobsBean> getJobsForAnExtension(String extensionName) {
+    public List<String> getJobsForAnExtension(String extensionName) {
         EntityManager entityManager = getEntityManager();
         beginTransaction(entityManager);
         Query query = entityManager.createNamedQuery(PersistenceConstants.GET_JOBS_FOR_AN_EXTENSION);
         query.setParameter(EXTENSION_NAME, extensionName);
+        List<String> jobNames = new ArrayList<>();
         try {
-            return (List<ExtensionJobsBean>)query.getResultList();
+            jobNames.addAll((List<String>) query.getResultList());
         } finally {
             commitAndCloseTransaction(entityManager);
         }
+        return jobNames;
     }
 
-    public void deleteExtension(String extensionName){
+    public void deleteExtension(String extensionName) {
         EntityManager entityManager = getEntityManager();
         beginTransaction(entityManager);
         Query q = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSION);
         q.setParameter(EXTENSION_NAME, extensionName);
-        try{
+        try {
             q.executeUpdate();
         } finally {
             commitAndCloseTransaction(entityManager);
@@ -165,7 +162,7 @@ public void storeExtensionJob(String jobName, String extensionName, List<String>
                                   byte[] config) {
         ExtensionMetaStore metaStore = ExtensionStore.getMetaStore();
         boolean alreadySubmitted = false;
-        if (metaStore.getExtensionJobDetails(jobName) != null){
+        if (metaStore.getExtensionJobDetails(jobName) != null) {
             alreadySubmitted = true;
         }
         ExtensionJobsBean extensionJobsBean = new ExtensionJobsBean();
@@ -195,7 +192,7 @@ public void deleteExtensionJob(String jobName) {
         beginTransaction(entityManager);
         Query query = entityManager.createNamedQuery(PersistenceConstants.DELETE_EXTENSION_JOB);
         query.setParameter(JOB_NAME, jobName);
-        try{
+        try {
             query.executeUpdate();
         } finally {
             commitAndCloseTransaction(entityManager);
@@ -237,7 +234,7 @@ public ExtensionJobsBean getExtensionJobDetails(String jobName) {
         }
     }
 
-    public List<ExtensionJobsBean> getAllExtensionJobs() {
+    List<ExtensionJobsBean> getAllExtensionJobs() {
         EntityManager entityManager = getEntityManager();
         beginTransaction(entityManager);
         Query q = entityManager.createNamedQuery(PersistenceConstants.GET_ALL_EXTENSION_JOBS);