diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java index 5b5f9ff20..a071d7ac1 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java @@ -100,7 +100,8 @@ public synchronized int run(final String[] args) throws Exception { metadataCLI.createMetadataOptions(), true); parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "", "Extension operations like enumerate, definition, describe, list, instances, " - + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister", + + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister" + + ",detail", extensionCLI.createExtensionOptions(), true); parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false); diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java index 1bb27a03b..4d023a673 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java @@ -50,6 +50,7 @@ public class FalconExtensionCLI { public static final String DESCRIBE_OPT = "describe"; public static final String INSTANCES_OPT = "instances"; public static final String UNREGISTER_OPT = "unregister"; + public static final String DETAIL_OPT = "detail"; // Input parameters public static final String ENTENSION_NAME_OPT = "extensionName"; @@ -83,6 +84,9 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) { } else if (optionsList.contains(UNREGISTER_OPT)) { validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); result = client.unregisterExtension(extensionName); + }else if (optionsList.contains(DETAIL_OPT)) { + validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); + result = client.getExtensionDetail(extensionName); } else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) { validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT); @@ -157,7 +161,9 @@ public Options createExtensionOptions() { Option suspend = new Option(FalconCLIConstants.SUSPEND_OPT, false, "Suspend an extension job"); Option resume = new Option(FalconCLIConstants.RESUME_OPT, false, "Resume an extension job"); Option delete = new Option(FalconCLIConstants.DELETE_OPT, false, "Delete an extension job"); - Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Delete metadata of extension job"); + Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Un-register an extension. This will make" + + " the extension unavailable for instantiation"); + Option detail = new Option(FalconCLIConstants.DETAIL, false, "Show details of a given extension"); OptionGroup group = new OptionGroup(); group.addOption(enumerate); @@ -174,6 +180,7 @@ public Options createExtensionOptions() { group.addOption(resume); group.addOption(delete); group.addOption(unregister); + group.addOption(detail); extensionOptions.addOptionGroup(group); Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL"); diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java index 8b15f8667..3775771e8 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java +++ b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java @@ -218,4 +218,5 @@ private FalconCLIConstants(){ public static final String DEBUG_OPTION_DESCRIPTION = "Use debug mode to see debugging statements on stdout"; public static final String DO_AS_DESCRIPTION = "doAs user"; public static final String UREGISTER = "unregister"; + public static final String DETAIL = "detail"; } diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index ef9df7f54..3db249ae4 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -347,7 +347,8 @@ protected static enum ExtensionOperations { SUSPEND("api/extension/suspend", HttpMethod.POST, MediaType.TEXT_XML), RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML), DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML), - UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN); + UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN), + DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON);; private String path; private String method; @@ -1029,6 +1030,12 @@ public String unregisterExtension(final String extensionName) { return getResponse(String.class, clientResponse); } + public String getExtensionDetail(final String extensionName) { + ClientResponse clientResponse = new ResourceBuilder().path(ExtensionOperations.DETAIL.path, extensionName) + .call(ExtensionOperations.DETAIL); + return getResponse(String.class, clientResponse); + } + public String getExtensionDefinition(final String extensionName) { ClientResponse clientResponse = new ResourceBuilder() .path(ExtensionOperations.DEFINITION.path, extensionName) diff --git a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java index f4208addf..9f4cf728b 100644 --- a/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java +++ b/common/src/main/java/org/apache/falcon/persistence/ExtensionMetadataBean.java @@ -37,7 +37,6 @@ @Entity @NamedQueries({ @NamedQuery(name = PersistenceConstants.GET_ALL_EXTENSIONS, query = "select OBJECT(a) from ExtensionMetadataBean a "), - @NamedQuery(name = PersistenceConstants.GET_EXTENSION_LOCATION, query = "select a.location from ExtensionMetadataBean a where a.extensionName = :extensionName"), @NamedQuery(name = PersistenceConstants.DELETE_EXTENSIONS_OF_TYPE, query = "delete from ExtensionMetadataBean a where a.extensionType = :extensionType "), @NamedQuery(name = PersistenceConstants.DELETE_EXTENSION, query = "delete from ExtensionMetadataBean a where a.extensionName = :extensionName "), @NamedQuery(name = PersistenceConstants.GET_EXTENSION, query = "select OBJECT(a) from ExtensionMetadataBean a where a.extensionName = :extensionName") diff --git a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java index d0e8e9c60..94eb32eae 100644 --- a/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java +++ b/common/src/main/java/org/apache/falcon/persistence/PersistenceConstants.java @@ -68,7 +68,6 @@ private PersistenceConstants(){ public static final String GET_ALL_BACKLOG_INSTANCES = "GET_ALL_BACKLOG_INSTANCES"; public static final String DELETE_ALL_BACKLOG_ENTITY_INSTANCES ="DELETE_ALL_BACKLOG_ENTITY_INSTANCES"; public static final String GET_ALL_EXTENSIONS = "GET_ALL_EXTENSIONS"; - public static final String GET_EXTENSION_LOCATION = "GET_EXTENSION_LOCATION"; public static final String DELETE_EXTENSIONS_OF_TYPE = "DELETE_EXTENSIONS_OF_TYPE"; public static final String DELETE_EXTENSION = "DELETE_EXTENSION"; public static final String GET_EXTENSION = "GET_EXTENSION"; diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index 9fb1c0a19..5d5da5a8e 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -34,6 +34,7 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.extensions.ExtensionService,\ org.apache.falcon.service.EntitySLAMonitoringService,\ org.apache.falcon.service.LifecyclePolicyMap,\ @@ -43,8 +44,7 @@ org.apache.falcon.metadata.MetadataMappingService,\ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ - org.apache.falcon.service.ProxyUserService,\ - org.apache.falcon.service.FalconJPAService + org.apache.falcon.service.ProxyUserService ##Add if you want to send data to graphite # org.apache.falcon.metrics.MetricNotificationService\ ## Add if you want to use Falcon Azure integration ## diff --git a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java index 2c6003df0..0a1a0e748 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStore.java @@ -87,13 +87,13 @@ public void deleteExtensionsOfType(ExtensionType extensionType){ } } - public String getLocation(String extensionName){ + public ExtensionMetadataBean getDetail(String extensionName){ EntityManager entityManager = getEntityManager(); beginTransaction(entityManager); - Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION_LOCATION); + Query q = entityManager.createNamedQuery(PersistenceConstants.GET_EXTENSION); q.setParameter("extensionName", extensionName); try { - return (String)q.getSingleResult(); + return (ExtensionMetadataBean)q.getSingleResult(); } finally { commitAndCloseTransaction(entityManager); } diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index ffb434115..02f7e6218 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -250,7 +250,6 @@ public List getExtensions() throws StoreAccessException { } return extesnionList; } - public String deleteExtensionMetadata(final String extensionName) throws ValidationException{ ExtensionType extensionType = AbstractExtension.isExtensionTrusted(extensionName) ? ExtensionType.TRUSTED : ExtensionType.CUSTOM; diff --git a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java index 7e0db0d29..d0f1c0cbe 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/jdbc/ExtensionMetaStoreTest.java @@ -20,6 +20,7 @@ import org.apache.falcon.cluster.util.EmbeddedCluster; import org.apache.falcon.extensions.ExtensionType; import org.apache.falcon.extensions.store.AbstractTestExtensionStore; +import org.apache.falcon.persistence.ExtensionMetadataBean; import org.apache.falcon.service.FalconJPAService; import org.apache.hadoop.conf.Configuration; @@ -62,7 +63,8 @@ public void dbOpertaions(){ Assert.assertEquals(stateStore.getAllExtensions().size(), 1); //check data - Assert.assertEquals(stateStore.getLocation("test1"), "test_location"); + ExtensionMetadataBean bean = stateStore.getDetail("test1"); + Assert.assertEquals(bean.getLocation(), "test_location"); //delete stateStore.deleteExtensionsOfType(ExtensionType.TRUSTED); Assert.assertEquals(stateStore.getAllExtensions().size(), 0); diff --git a/extensions/src/test/resources/startup.properties b/extensions/src/test/resources/startup.properties index 58daf3263..7a726c3dd 100644 --- a/extensions/src/test/resources/startup.properties +++ b/extensions/src/test/resources/startup.properties @@ -33,6 +33,7 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ @@ -40,8 +41,7 @@ org.apache.falcon.notification.service.impl.SchedulerService,\ org.apache.falcon.notification.service.impl.AlarmService,\ org.apache.falcon.notification.service.impl.DataAvailabilityService,\ - org.apache.falcon.execution.FalconExecutionService,\ - org.apache.falcon.service.FalconJPAService + org.apache.falcon.execution.FalconExecutionService ##### Falcon Configuration Store Change listeners ##### *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java index 7fd4de5a7..b70b3a714 100644 --- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.FalconWebException; +import org.apache.falcon.entity.parser.ValidationException; import org.apache.falcon.entity.store.StoreAccessException; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.cluster.Cluster; @@ -420,6 +421,20 @@ public String getExtensionDescription( } } + @GET + @Path("detail/{extension-name}") + @Produces({MediaType.APPLICATION_JSON}) + public Response getDetail(@PathParam("extension-name") String extensionName){ + checkIfExtensionServiceIsEnabled(); + validateExtensionName(extensionName); + try { + return Response.ok(buildDetailResult(extensionName)).build(); + } catch (Throwable e) { + throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR); + } + } + + @POST @Path("unregister/{extension-name}") @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN}) @@ -524,6 +539,27 @@ private void setEntityTags(Entity entity, String tags) { } } + private JSONObject buildDetailResult(final String extensionName) throws FalconException { + ExtensionMetaStore metaStore = ExtensionStore.get().getMetaStore(); + + if (!metaStore.checkIfExtensionExists(extensionName)){ + throw new ValidationException("No extension resources found for " + extensionName); + } + + ExtensionMetadataBean bean = metaStore.getDetail(extensionName); + JSONObject resultObject = new JSONObject(); + try { + resultObject.put(EXTENSION_NAME, bean.getExtensionName()); + resultObject.put(EXTENSION_TYPE, bean.getExtensionType()); + resultObject.put(EXTENSION_DESC, bean.getDescription()); + resultObject.put(EXTENSION_LOCATION, bean.getLocation()); + } catch (JSONException e) { + LOG.error("Exception in buildDetailResults:", e); + throw new FalconException(e); + } + return resultObject; + } + private Map> groupEntitiesByJob(List entities) { Map> groupedEntities = new HashMap<>(); for (Entity entity : entities) { diff --git a/prism/src/test/resources/startup.properties b/prism/src/test/resources/startup.properties index 5258b96a4..63fcd3b70 100644 --- a/prism/src/test/resources/startup.properties +++ b/prism/src/test/resources/startup.properties @@ -34,6 +34,7 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.extensions.ExtensionService,\ org.apache.falcon.service.EntitySLAMonitoringService,\ org.apache.falcon.service.LifecyclePolicyMap,\ @@ -43,8 +44,7 @@ org.apache.falcon.metadata.MetadataMappingService,\ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ - org.apache.falcon.service.ProxyUserService,\ - org.apache.falcon.service.FalconJPAService + org.apache.falcon.service.ProxyUserService ##Add if you want to send data to graphite # org.apache.falcon.metrics.MetricNotificationService\ ## Add if you want to use Falcon Azure integration ## diff --git a/scheduler/src/test/resources/startup.properties b/scheduler/src/test/resources/startup.properties index 46031e33f..2da3e2ebc 100644 --- a/scheduler/src/test/resources/startup.properties +++ b/scheduler/src/test/resources/startup.properties @@ -33,6 +33,7 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ @@ -40,8 +41,7 @@ org.apache.falcon.notification.service.impl.SchedulerService,\ org.apache.falcon.notification.service.impl.AlarmService,\ org.apache.falcon.notification.service.impl.DataAvailabilityService,\ - org.apache.falcon.execution.FalconExecutionService,\ - org.apache.falcon.service.FalconJPAService + org.apache.falcon.execution.FalconExecutionService ##### Falcon Configuration Store Change listeners ##### *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ diff --git a/src/conf/startup.properties b/src/conf/startup.properties index 8eb58b999..6a95cce24 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -42,6 +42,7 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.extensions.ExtensionService,\ org.apache.falcon.service.LifecyclePolicyMap,\ org.apache.falcon.entity.store.ConfigurationStore,\ diff --git a/webapp/src/test/resources/startup.properties b/webapp/src/test/resources/startup.properties index 1cfebabcc..5751d9c38 100644 --- a/webapp/src/test/resources/startup.properties +++ b/webapp/src/test/resources/startup.properties @@ -32,8 +32,8 @@ *.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ org.apache.falcon.service.ProcessSubscriberService,\ - org.apache.falcon.extensions.ExtensionService,\ org.apache.falcon.service.FalconJPAService,\ + org.apache.falcon.extensions.ExtensionService,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\