Merge branch 'master' of https://github.com/apache/falcon

pallavi-rao · Nov 21, 2016 · bbca081 · bbca081
2 parents 48f6afa + 3d61e96
commit bbca081
Show file tree

Hide file tree

Showing 16 changed files with 71 additions and 19 deletions.
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -100,7 +100,8 @@ public synchronized int run(final String[] args) throws Exception {
                 metadataCLI.createMetadataOptions(), true);
         parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "",
                 "Extension operations like enumerate, definition, describe, list, instances, "
-                        + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister",
+                        + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister"
+                        + ",detail",
                 extensionCLI.createExtensionOptions(), true);
         parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false);
 

diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
@@ -50,6 +50,7 @@ public class FalconExtensionCLI {
     public static final String DESCRIBE_OPT = "describe";
     public static final String INSTANCES_OPT = "instances";
     public static final String UNREGISTER_OPT = "unregister";
+    public static final String DETAIL_OPT = "detail";
 
     // Input parameters
     public static final String ENTENSION_NAME_OPT = "extensionName";
@@ -83,6 +84,9 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) {
         } else if (optionsList.contains(UNREGISTER_OPT)) {
             validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
             result = client.unregisterExtension(extensionName);
+        }else if (optionsList.contains(DETAIL_OPT)) {
+            validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
+            result = client.getExtensionDetail(extensionName);
         } else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) {
             validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
             validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
@@ -157,7 +161,9 @@ public Options createExtensionOptions() {
         Option suspend = new Option(FalconCLIConstants.SUSPEND_OPT, false, "Suspend an extension job");
         Option resume = new Option(FalconCLIConstants.RESUME_OPT, false, "Resume an extension job");
         Option delete = new Option(FalconCLIConstants.DELETE_OPT, false, "Delete an extension job");
-        Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Delete metadata of extension job");
+        Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Un-register an extension. This will make"
+                + " the extension unavailable for instantiation");
+        Option detail = new Option(FalconCLIConstants.DETAIL, false, "Show details of a given extension");
 
         OptionGroup group = new OptionGroup();
         group.addOption(enumerate);
@@ -174,6 +180,7 @@ public Options createExtensionOptions() {
         group.addOption(resume);
         group.addOption(delete);
         group.addOption(unregister);
+        group.addOption(detail);
         extensionOptions.addOptionGroup(group);
 
         Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL");

diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
@@ -218,4 +218,5 @@ private FalconCLIConstants(){
     public static final String DEBUG_OPTION_DESCRIPTION = "Use debug mode to see debugging statements on stdout";
     public static final String DO_AS_DESCRIPTION = "doAs user";
     public static final String UREGISTER = "unregister";
+    public static final String DETAIL = "detail";
 }
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -347,7 +347,8 @@ protected static enum ExtensionOperations {
         SUSPEND("api/extension/suspend", HttpMethod.POST, MediaType.TEXT_XML),
         RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML),
         DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML),
-        UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN);
+        UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN),
+        DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON);;
 
         private String path;
         private String method;
@@ -1029,6 +1030,12 @@ public String unregisterExtension(final String extensionName) {
         return getResponse(String.class, clientResponse);
     }
 
+    public String getExtensionDetail(final String extensionName) {
+        ClientResponse clientResponse = new ResourceBuilder().path(ExtensionOperations.DETAIL.path, extensionName)
+                .call(ExtensionOperations.DETAIL);
+        return getResponse(String.class, clientResponse);
+    }
+
     public String getExtensionDefinition(final String extensionName)  {
         ClientResponse clientResponse = new ResourceBuilder()
                 .path(ExtensionOperations.DEFINITION.path, extensionName)

diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java
@@ -37,7 +37,6 @@
 @Entity
 @NamedQueries({
         @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a) from ExtensionMetadataBean a "),
-        @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location from ExtensionMetadataBean a where a.extensionName = :extensionName"),
         @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType "),
         @NamedQuery(name = PersistenceConstants.DELETE_EXTENSION, query = "delete from ExtensionMetadataBean a where a.extensionName = :extensionName "),
         @NamedQuery(name = PersistenceConstants.GET_EXTENSION, query = "select OBJECT(a) from ExtensionMetadataBean a where a.extensionName = :extensionName")

diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java
@@ -68,7 +68,6 @@ private PersistenceConstants(){
     public static final String GET_ALL_BACKLOG_INSTANCES = "GET_ALL_BACKLOG_INSTANCES";
     public static final String DELETE_ALL_BACKLOG_ENTITY_INSTANCES ="DELETE_ALL_BACKLOG_ENTITY_INSTANCES";
     public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS";
-    public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION";
     public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE";
     public static final String DELETE_EXTENSION = "DELETE_EXTENSION";
     public static final String GET_EXTENSION = "GET_EXTENSION";

diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
@@ -34,6 +34,7 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
+                        org.apache.falcon.service.FalconJPAService,\
                         org.apache.falcon.extensions.ExtensionService,\
                         org.apache.falcon.service.EntitySLAMonitoringService,\
                         org.apache.falcon.service.LifecyclePolicyMap,\
@@ -43,8 +44,7 @@
                         org.apache.falcon.metadata.MetadataMappingService,\
                         org.apache.falcon.service.LogCleanupService,\
                         org.apache.falcon.service.GroupsService,\
-                        org.apache.falcon.service.ProxyUserService,\
-                        org.apache.falcon.service.FalconJPAService
+                        org.apache.falcon.service.ProxyUserService
 ##Add if you want to send data to graphite
 #                        org.apache.falcon.metrics.MetricNotificationService\
 ## Add if you want to use Falcon Azure integration ##

diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java
@@ -87,13 +87,13 @@ public void deleteExtensionsOfType(ExtensionType extensionType){
         }
     }
 
-    public String getLocation(String extensionName){
+    public ExtensionMetadataBean getDetail(String extensionName){
         EntityManager entityManager = getEntityManager();
         beginTransaction(entityManager);
-        Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION_LOCATION);
+        Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION);
         q.setParameter("extensionName", extensionName);
         try {
-            return (String)q.getSingleResult();
+            return (ExtensionMetadataBean)q.getSingleResult();
         } finally {
             commitAndCloseTransaction(entityManager);
         }

diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -250,7 +250,6 @@ public List<String> getExtensions() throws StoreAccessException {
         }
         return extesnionList;
     }
-
     public String deleteExtensionMetadata(final String extensionName) throws ValidationException{
         ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED
                 : ExtensionType.CUSTOM;

diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java
@@ -20,6 +20,7 @@
 import org.apache.falcon.cluster.util.EmbeddedCluster;
 import org.apache.falcon.extensions.ExtensionType;
 import org.apache.falcon.extensions.store.AbstractTestExtensionStore;
+import org.apache.falcon.persistence.ExtensionMetadataBean;
 import org.apache.falcon.service.FalconJPAService;
 
 import org.apache.hadoop.conf.Configuration;
@@ -62,7 +63,8 @@ public void dbOpertaions(){
 
         Assert.assertEquals(stateStore.getAllExtensions().size(), 1);
         //check data
-        Assert.assertEquals(stateStore.getLocation("test1"), "test_location");
+        ExtensionMetadataBean bean = stateStore.getDetail("test1");
+        Assert.assertEquals(bean.getLocation(), "test_location");
         //delete
         stateStore.deleteExtensionsOfType(ExtensionType.TRUSTED);
         Assert.assertEquals(stateStore.getAllExtensions().size(), 0);

diff --git a/extensions/src/test/resources/startup.properties b/extensions/src/test/resources/startup.properties
@@ -33,15 +33,15 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
+                        org.apache.falcon.service.FalconJPAService,\
                         org.apache.falcon.entity.store.ConfigurationStore,\
                         org.apache.falcon.rerun.service.RetryService,\
                         org.apache.falcon.rerun.service.LateRunService,\
                         org.apache.falcon.notification.service.impl.JobCompletionService,\
                         org.apache.falcon.notification.service.impl.SchedulerService,\
                         org.apache.falcon.notification.service.impl.AlarmService,\
                         org.apache.falcon.notification.service.impl.DataAvailabilityService,\
-                        org.apache.falcon.execution.FalconExecutionService,\
-                        org.apache.falcon.service.FalconJPAService
+                        org.apache.falcon.execution.FalconExecutionService
 
 ##### Falcon Configuration Store Change listeners #####
 *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\

diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
@@ -21,6 +21,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.FalconWebException;
+import org.apache.falcon.entity.parser.ValidationException;
 import org.apache.falcon.entity.store.StoreAccessException;
 import org.apache.falcon.entity.v0.Entity;
 import org.apache.falcon.entity.v0.cluster.Cluster;
@@ -420,6 +421,20 @@ public String getExtensionDescription(
         }
     }
 
+    @GET
+    @Path("detail/{extension-name}")
+    @Produces({MediaType.APPLICATION_JSON})
+    public Response getDetail(@PathParam("extension-name") String extensionName){
+        checkIfExtensionServiceIsEnabled();
+        validateExtensionName(extensionName);
+        try {
+            return Response.ok(buildDetailResult(extensionName)).build();
+        } catch (Throwable e) {
+            throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
+        }
+    }
+
+
     @POST
     @Path("unregister/{extension-name}")
     @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
@@ -524,6 +539,27 @@ private void setEntityTags(Entity entity, String tags) {
         }
     }
 
+    private  JSONObject buildDetailResult(final String extensionName) throws FalconException {
+        ExtensionMetaStore metaStore = ExtensionStore.get().getMetaStore();
+
+        if (!metaStore.checkIfExtensionExists(extensionName)){
+            throw new ValidationException("No extension resources found for " + extensionName);
+        }
+
+        ExtensionMetadataBean bean = metaStore.getDetail(extensionName);
+        JSONObject resultObject = new JSONObject();
+        try {
+            resultObject.put(EXTENSION_NAME, bean.getExtensionName());
+            resultObject.put(EXTENSION_TYPE, bean.getExtensionType());
+            resultObject.put(EXTENSION_DESC, bean.getDescription());
+            resultObject.put(EXTENSION_LOCATION, bean.getLocation());
+        } catch (JSONException e) {
+            LOG.error("Exception in buildDetailResults:", e);
+            throw new FalconException(e);
+        }
+        return resultObject;
+    }
+
     private Map<String, List<Entity>> groupEntitiesByJob(List<Entity> entities) {
         Map<String, List<Entity>> groupedEntities = new HashMap<>();
         for (Entity entity : entities) {

diff --git a/prism/src/test/resources/startup.properties b/prism/src/test/resources/startup.properties
@@ -34,6 +34,7 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
+                        org.apache.falcon.service.FalconJPAService,\
                         org.apache.falcon.extensions.ExtensionService,\
                         org.apache.falcon.service.EntitySLAMonitoringService,\
                         org.apache.falcon.service.LifecyclePolicyMap,\
@@ -43,8 +44,7 @@
                         org.apache.falcon.metadata.MetadataMappingService,\
                         org.apache.falcon.service.LogCleanupService,\
                         org.apache.falcon.service.GroupsService,\
-                        org.apache.falcon.service.ProxyUserService,\
-                        org.apache.falcon.service.FalconJPAService
+                        org.apache.falcon.service.ProxyUserService
 ##Add if you want to send data to graphite
 #                        org.apache.falcon.metrics.MetricNotificationService\
 ## Add if you want to use Falcon Azure integration ##

diff --git a/scheduler/src/test/resources/startup.properties b/scheduler/src/test/resources/startup.properties
@@ -33,15 +33,15 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
+                        org.apache.falcon.service.FalconJPAService,\
                         org.apache.falcon.entity.store.ConfigurationStore,\
                         org.apache.falcon.rerun.service.RetryService,\
                         org.apache.falcon.rerun.service.LateRunService,\
                         org.apache.falcon.notification.service.impl.JobCompletionService,\
                         org.apache.falcon.notification.service.impl.SchedulerService,\
                         org.apache.falcon.notification.service.impl.AlarmService,\
                         org.apache.falcon.notification.service.impl.DataAvailabilityService,\
-                        org.apache.falcon.execution.FalconExecutionService,\
-                        org.apache.falcon.service.FalconJPAService
+                        org.apache.falcon.execution.FalconExecutionService
 
 ##### Falcon Configuration Store Change listeners #####
 *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\

diff --git a/src/conf/startup.properties b/src/conf/startup.properties
@@ -42,6 +42,7 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
+                        org.apache.falcon.service.FalconJPAService,\
                         org.apache.falcon.extensions.ExtensionService,\
                         org.apache.falcon.service.LifecyclePolicyMap,\
                         org.apache.falcon.entity.store.ConfigurationStore,\

diff --git a/webapp/src/test/resources/startup.properties b/webapp/src/test/resources/startup.properties
@@ -32,8 +32,8 @@
 *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
                         org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
                         org.apache.falcon.service.ProcessSubscriberService,\
-                        org.apache.falcon.extensions.ExtensionService,\
                         org.apache.falcon.service.FalconJPAService,\
+                        org.apache.falcon.extensions.ExtensionService,\
                         org.apache.falcon.entity.store.ConfigurationStore,\
                         org.apache.falcon.rerun.service.RetryService,\
                         org.apache.falcon.rerun.service.LateRunService,\