diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java index a071d7ac1..2e0978f9d 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java @@ -101,7 +101,7 @@ public synchronized int run(final String[] args) throws Exception { parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "", "Extension operations like enumerate, definition, describe, list, instances, " + "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister" - + ",detail", + + ",detail,register", extensionCLI.createExtensionOptions(), true); parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false); diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java index 4d023a673..83b550fd0 100644 --- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java +++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java @@ -51,10 +51,13 @@ public class FalconExtensionCLI { public static final String INSTANCES_OPT = "instances"; public static final String UNREGISTER_OPT = "unregister"; public static final String DETAIL_OPT = "detail"; + public static final String REGISTER_OPT = "register"; // Input parameters public static final String ENTENSION_NAME_OPT = "extensionName"; public static final String JOB_NAME_OPT = "jobName"; + public static final String DESCRIPTION = "description"; + public static final String PATH = "path"; public FalconExtensionCLI() { } @@ -70,6 +73,8 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) { String jobName = commandLine.getOptionValue(JOB_NAME_OPT); String filePath = commandLine.getOptionValue(FalconCLIConstants.FILE_PATH_OPT); String doAsUser = commandLine.getOptionValue(FalconCLIConstants.DO_AS_OPT); + String path = commandLine.getOptionValue(FalconCLIConstants.PATH); + String description = commandLine.getOptionValue(FalconCLIConstants.DESCRIPTION); if (optionsList.contains(ENUMERATE_OPT)) { result = client.enumerateExtensions(); @@ -91,6 +96,14 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) { validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT); result = client.submitExtensionJob(extensionName, filePath, doAsUser).getMessage(); + } else if (optionsList.contains(REGISTER_OPT)) { + validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); + validateRequiredParameter(path, PATH); + result = client.registerExtension(extensionName, path, description); + }else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) { + validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); + validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT); + result = client.submitExtensionJob(extensionName, filePath, doAsUser).getMessage(); } else if (optionsList.contains(FalconCLIConstants.SUBMIT_AND_SCHEDULE_OPT)) { validateRequiredParameter(extensionName, ENTENSION_NAME_OPT); validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT); @@ -164,6 +177,8 @@ public Options createExtensionOptions() { Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Un-register an extension. This will make" + " the extension unavailable for instantiation"); Option detail = new Option(FalconCLIConstants.DETAIL, false, "Show details of a given extension"); + Option register = new Option(FalconCLIConstants.REGISTER, false, "Register an extension with Falcon. This will " + + "make the extension available for instantiation for all users."); OptionGroup group = new OptionGroup(); group.addOption(enumerate); @@ -181,6 +196,7 @@ public Options createExtensionOptions() { group.addOption(delete); group.addOption(unregister); group.addOption(detail); + group.addOption(register); extensionOptions.addOptionGroup(group); Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL"); @@ -200,6 +216,8 @@ public Options createExtensionOptions() { Option status = new Option(FalconCLIConstants.STATUS_OPT, true, "Filter returned instances by status"); Option orderBy = new Option(FalconCLIConstants.ORDER_BY_OPT, true, "Order returned instances by this field"); Option filePath = new Option(FalconCLIConstants.FILE_PATH_OPT, true, "File path of extension parameters"); + Option path = new Option(FalconCLIConstants.PATH, true, "Path of hdfs location for extension"); + Option description = new Option(FalconCLIConstants.DESCRIPTION, true, "Short Description for extension"); extensionOptions.addOption(url); extensionOptions.addOption(doAs); @@ -216,6 +234,8 @@ public Options createExtensionOptions() { extensionOptions.addOption(status); extensionOptions.addOption(orderBy); extensionOptions.addOption(filePath); + extensionOptions.addOption(path); + extensionOptions.addOption(description); return extensionOptions; } diff --git a/client/pom.xml b/client/pom.xml index 6df9d6d1a..9daa998e0 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -40,6 +40,12 @@ hadoop-auth + + org.apache.falcon + falcon-common-types + ${project.version} + + org.apache.hadoop hadoop-client @@ -124,33 +130,6 @@ - - org.jvnet.jaxb2.maven2 - maven-jaxb2-plugin - 0.8.0 - - - 1 - generate-sources - - generate - - - - - - -extension - -Xinheritance - - - - org.jvnet.jaxb2_commons - jaxb2-basics - 0.6.3 - - - - org.apache.maven.plugins maven-compiler-plugin diff --git a/client/src/main/java/org/apache/falcon/ValidationUtil.java b/client/src/main/java/org/apache/falcon/ValidationUtil.java index cdf1f0594..3db521ff5 100644 --- a/client/src/main/java/org/apache/falcon/ValidationUtil.java +++ b/client/src/main/java/org/apache/falcon/ValidationUtil.java @@ -108,7 +108,7 @@ public static void validateEntityTypeForSummary(String type) { public static List getLifeCycle(String lifeCycleValue) { if (lifeCycleValue != null) { String[] lifeCycleValues = lifeCycleValue.split(","); - List lifeCycles = new ArrayList(); + List lifeCycles = new ArrayList<>(); try { for (String lifeCycle : lifeCycleValues) { lifeCycles.add(LifeCycle.valueOf(lifeCycle.toUpperCase().trim())); diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java index 3775771e8..26e893744 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java +++ b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java @@ -219,4 +219,7 @@ private FalconCLIConstants(){ public static final String DO_AS_DESCRIPTION = "doAs user"; public static final String UREGISTER = "unregister"; public static final String DETAIL = "detail"; + public static final String REGISTER = "register"; + public static final String PATH = "path"; + public static final String DESCRIPTION = "description"; } diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 3db249ae4..dabed3f72 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -348,7 +348,8 @@ protected static enum ExtensionOperations { RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML), DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML), UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN), - DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON);; + DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON), + REGISTER("api/extension/register/", HttpMethod.POST, MediaType.TEXT_PLAIN); private String path; private String method; @@ -1036,6 +1037,14 @@ public String getExtensionDetail(final String extensionName) { return getResponse(String.class, clientResponse); } + public String registerExtension(final String extensionName, final String path, final String description) { + ClientResponse clientResponse = new ResourceBuilder() + .path(ExtensionOperations.REGISTER.path, extensionName).addQueryParam(PATH, path) + .addQueryParam(FalconCLIConstants.DESCRIPTION, description) + .call(ExtensionOperations.REGISTER); + return getResponse(String.class, clientResponse); + } + public String getExtensionDefinition(final String extensionName) { ClientResponse clientResponse = new ResourceBuilder() .path(ExtensionOperations.DEFINITION.path, extensionName) diff --git a/common-types/pom.xml b/common-types/pom.xml new file mode 100644 index 000000000..714b166fc --- /dev/null +++ b/common-types/pom.xml @@ -0,0 +1,164 @@ + + + + + + + 4.0.0 + + org.apache.falcon + falcon-main + 0.11-SNAPSHOT + + falcon-common-types + Apache Falcon Java common types + Apache Falcon Java common types + jar + + + + + + org.apache.hadoop + hadoop-auth + + + + org.apache.hadoop + hadoop-client + + + + commons-cli + commons-cli + + + + commons-net + commons-net + + + + commons-codec + commons-codec + + + + org.apache.commons + commons-lang3 + + + + com.sun.jersey + jersey-client + + + + com.sun.jersey + jersey-json + + + + org.slf4j + slf4j-api + provided + + + + log4j + log4j + + + + commons-io + commons-io + + + + com.github.stephenc.findbugs + findbugs-annotations + + + + org.testng + testng + + + + org.apache.hive.hcatalog + hive-webhcat-java-client + ${hive.version} + + + + + + + maven-dependency-plugin + + + package + + copy-dependencies + + + ${project.build.directory}/dependency + + + + + + org.jvnet.jaxb2.maven2 + maven-jaxb2-plugin + 0.8.0 + + + 1 + generate-sources + + generate + + + + + + -extension + -Xinheritance + + + + org.jvnet.jaxb2_commons + jaxb2-basics + 0.6.3 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + + + + + diff --git a/client/src/main/java/org/apache/falcon/LifeCycle.java b/common-types/src/main/java/org/apache/falcon/LifeCycle.java similarity index 100% rename from client/src/main/java/org/apache/falcon/LifeCycle.java rename to common-types/src/main/java/org/apache/falcon/LifeCycle.java diff --git a/client/src/main/java/org/apache/falcon/Pair.java b/common-types/src/main/java/org/apache/falcon/Pair.java similarity index 100% rename from client/src/main/java/org/apache/falcon/Pair.java rename to common-types/src/main/java/org/apache/falcon/Pair.java diff --git a/client/src/main/java/org/apache/falcon/Tag.java b/common-types/src/main/java/org/apache/falcon/Tag.java similarity index 85% rename from client/src/main/java/org/apache/falcon/Tag.java rename to common-types/src/main/java/org/apache/falcon/Tag.java index a8d60b69d..3c2f25e20 100644 --- a/client/src/main/java/org/apache/falcon/Tag.java +++ b/common-types/src/main/java/org/apache/falcon/Tag.java @@ -24,12 +24,15 @@ * Tag to include in the entity type. */ public enum Tag { - DEFAULT(EntityType.PROCESS), RETENTION(EntityType.FEED), REPLICATION(EntityType.FEED), - IMPORT(EntityType.FEED), EXPORT(EntityType.FEED); + DEFAULT(EntityType.PROCESS), + RETENTION(EntityType.FEED), + REPLICATION(EntityType.FEED), + IMPORT(EntityType.FEED), + EXPORT(EntityType.FEED); private final EntityType entityType; - private Tag(EntityType entityType) { + Tag(EntityType entityType) { this.entityType = entityType; } diff --git a/client/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java b/common-types/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java diff --git a/client/src/main/java/org/apache/falcon/entity/v0/DateValidator.java b/common-types/src/main/java/org/apache/falcon/entity/v0/DateValidator.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/DateValidator.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/DateValidator.java diff --git a/client/src/main/java/org/apache/falcon/entity/v0/Entity.java b/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/Entity.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java diff --git a/client/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java b/common-types/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java diff --git a/client/src/main/java/org/apache/falcon/entity/v0/EntityType.java b/common-types/src/main/java/org/apache/falcon/entity/v0/EntityType.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/EntityType.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/EntityType.java diff --git a/client/src/main/java/org/apache/falcon/entity/v0/Frequency.java b/common-types/src/main/java/org/apache/falcon/entity/v0/Frequency.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/Frequency.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/Frequency.java diff --git a/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java b/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java similarity index 100% rename from client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java rename to common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java b/common-types/src/main/java/org/apache/falcon/extensions/ExtensionType.java similarity index 100% rename from extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java rename to common-types/src/main/java/org/apache/falcon/extensions/ExtensionType.java diff --git a/client/src/main/java/org/apache/falcon/metadata/RelationshipType.java b/common-types/src/main/java/org/apache/falcon/metadata/RelationshipType.java similarity index 100% rename from client/src/main/java/org/apache/falcon/metadata/RelationshipType.java rename to common-types/src/main/java/org/apache/falcon/metadata/RelationshipType.java diff --git a/client/src/main/java/org/apache/falcon/resource/APIResult.java b/common-types/src/main/java/org/apache/falcon/resource/APIResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/APIResult.java rename to common-types/src/main/java/org/apache/falcon/resource/APIResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/EntityList.java b/common-types/src/main/java/org/apache/falcon/resource/EntityList.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/EntityList.java rename to common-types/src/main/java/org/apache/falcon/resource/EntityList.java diff --git a/client/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java b/common-types/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java rename to common-types/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java b/common-types/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java rename to common-types/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java b/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java rename to common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java diff --git a/client/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java b/common-types/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java rename to common-types/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/FeedLookupResult.java b/common-types/src/main/java/org/apache/falcon/resource/FeedLookupResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/FeedLookupResult.java rename to common-types/src/main/java/org/apache/falcon/resource/FeedLookupResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java b/common-types/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java rename to common-types/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/InstancesResult.java b/common-types/src/main/java/org/apache/falcon/resource/InstancesResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/InstancesResult.java rename to common-types/src/main/java/org/apache/falcon/resource/InstancesResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java b/common-types/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java rename to common-types/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/LineageGraphResult.java b/common-types/src/main/java/org/apache/falcon/resource/LineageGraphResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/LineageGraphResult.java rename to common-types/src/main/java/org/apache/falcon/resource/LineageGraphResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java b/common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java rename to common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java diff --git a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java b/common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java rename to common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java diff --git a/client/src/main/java/org/apache/falcon/resource/TriageResult.java b/common-types/src/main/java/org/apache/falcon/resource/TriageResult.java similarity index 100% rename from client/src/main/java/org/apache/falcon/resource/TriageResult.java rename to common-types/src/main/java/org/apache/falcon/resource/TriageResult.java diff --git a/client/src/main/resources/cluster-0.1.xsd b/common-types/src/main/resources/cluster-0.1.xsd similarity index 100% rename from client/src/main/resources/cluster-0.1.xsd rename to common-types/src/main/resources/cluster-0.1.xsd diff --git a/client/src/main/resources/datasource-0.1.xsd b/common-types/src/main/resources/datasource-0.1.xsd similarity index 100% rename from client/src/main/resources/datasource-0.1.xsd rename to common-types/src/main/resources/datasource-0.1.xsd diff --git a/client/src/main/resources/feed-0.1.xsd b/common-types/src/main/resources/feed-0.1.xsd similarity index 100% rename from client/src/main/resources/feed-0.1.xsd rename to common-types/src/main/resources/feed-0.1.xsd diff --git a/client/src/main/resources/jaxb-binding.xjb b/common-types/src/main/resources/jaxb-binding.xjb similarity index 100% rename from client/src/main/resources/jaxb-binding.xjb rename to common-types/src/main/resources/jaxb-binding.xjb diff --git a/client/src/main/resources/mysql_database.xml b/common-types/src/main/resources/mysql_database.xml similarity index 100% rename from client/src/main/resources/mysql_database.xml rename to common-types/src/main/resources/mysql_database.xml diff --git a/client/src/main/resources/process-0.1.xsd b/common-types/src/main/resources/process-0.1.xsd similarity index 100% rename from client/src/main/resources/process-0.1.xsd rename to common-types/src/main/resources/process-0.1.xsd diff --git a/client/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java b/common-types/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java similarity index 100% rename from client/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java rename to common-types/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java diff --git a/client/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java b/common-types/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java similarity index 100% rename from client/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java rename to common-types/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java diff --git a/common/pom.xml b/common/pom.xml index 96cb7f571..34822aa56 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -92,7 +92,8 @@ org.apache.falcon - falcon-client + falcon-common-types + ${project.version} diff --git a/extensions/pom.xml b/extensions/pom.xml index 77821cac3..df8f35f0f 100644 --- a/extensions/pom.xml +++ b/extensions/pom.xml @@ -71,7 +71,8 @@ org.apache.falcon - falcon-client + falcon-common-types + ${project.version} diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index 02f7e6218..e15919f39 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -25,12 +25,14 @@ import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.falcon.entity.parser.ValidationException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.io.IOUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -39,6 +41,8 @@ import org.apache.falcon.util.StartupProperties; import org.apache.falcon.entity.store.StoreAccessException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -264,6 +268,54 @@ public String deleteExtensionMetadata(final String extensionName) throws Validat } } + public String registerExtensionMetadata(final String extensionName, final String path, final String description) + throws URISyntaxException, FalconException { + Configuration conf = new Configuration(); + URI uri = new URI(path); + conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority()); + FileSystem fileSystem = HadoopClientFactory.get().createFalconFileSystem(uri); + try { + fileSystem.listStatus(new Path(uri.getPath() + "/README")); + } catch (IOException e){ + LOG.error("Exception in registerExtensionMetadata:", e); + throw new ValidationException("README file is not present in the " + path); + } + PathFilter filter=new PathFilter(){ + public boolean accept(Path file){ + return file.getName().endsWith(".jar"); + } + }; + FileStatus[] jarStatus; + try { + jarStatus = fileSystem.listStatus(new Path(uri.getPath() + "/libs/build"), filter); + if (jarStatus.length <=0) { + throw new ValidationException("Jars are not present in the " + uri.getPath() + "libs/build."); + } + } catch (IOException e){ + LOG.error("Exception in registerExtensionMetadata:", e); + throw new ValidationException("Jars are not present in the " + uri.getPath() + "libs/build."); + } + FileStatus[] propStatus; + try{ + propStatus = fileSystem.listStatus(new Path(uri.getPath() + "/META")); + if (propStatus.length <=0){ + throw new ValidationException("No properties file is not present in the " + uri.getPath() + "/META" + + " structure."); + } + } catch (IOException e){ + LOG.error("Exception in registerExtensionMetadata:", e); + throw new ValidationException("Directory is not present in the " + uri.getPath() + "/META" + + " structure."); + } + + if (!metaStore.checkIfExtensionExists(extensionName)){ + metaStore.storeExtensionMetadataBean(extensionName, path, ExtensionType.CUSTOM, description); + }else{ + throw new ValidationException(extensionName + " already exsists."); + } + return "Extension :" + extensionName + " registered succesfully."; + } + public String getResource(final String extensionName, final String resourceName) throws StoreAccessException { Map resources = getExtensionArtifacts(extensionName); if (resources.isEmpty()) { diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java index bc04d8834..27bea5355 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java @@ -19,19 +19,37 @@ package org.apache.falcon.extensions.store; import com.google.common.collect.ImmutableMap; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.URISyntaxException; +import java.util.Map; +import javax.persistence.EntityManager; +import javax.persistence.Query; +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.parser.ValidationException; import org.apache.falcon.entity.store.StoreAccessException; +import org.apache.falcon.extensions.jdbc.ExtensionMetaStore; import org.apache.falcon.extensions.mirroring.hdfs.HdfsMirroringExtension; +import org.apache.falcon.hadoop.JailedFileSystem; +import org.apache.falcon.service.FalconJPAService; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; import org.testng.Assert; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.Map; - /** * Tests for extension store. */ public class ExtensionStoreTest extends AbstractTestExtensionStore { private static Map resourcesMap; + private static JailedFileSystem fs; + protected static final String EXTENSION_PATH = "/projects/falcon/extension"; + private static final String STORAGE_URL = "jail://global:00"; @BeforeClass public void init() throws Exception { @@ -46,6 +64,8 @@ public void init() throws Exception { "hdfs-snapshot-mirroring-workflow.xml", extensionStorePath + "/hdfs-mirroring/resources/runtime/hdfs-snapshot-mirroring-workflow.xml" ); + fs = new JailedFileSystem(); + initFileSystem(); } @Test @@ -66,5 +86,96 @@ public void testGetExtensionLibPath() throws StoreAccessException { Assert.assertEquals(store.getExtensionLibPath(extensionName), libPath); } + private static void initFileSystem() throws IOException { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", STORAGE_URL); + fs.initialize(LocalFileSystem.getDefaultUri(conf), conf); + } + + @BeforeMethod + public void clean() { + clearDB(); + // Clean up FS + try { + fs.delete(new Path(extensionStorePath + "/hdfs-mirroring"), true); + } catch (IOException e) { + // Ignore + } + } + + + @Test + public void testRegisterExtensionMetadata() throws IOException, URISyntaxException, FalconException{ + createlibs(); + createReadmeAndJar(); + createMETA(); + store = ExtensionStore.get(); + store.registerExtensionMetadata("test", STORAGE_URL + EXTENSION_PATH, "test desc"); + ExtensionMetaStore metaStore = new ExtensionMetaStore(); + Assert.assertEquals(metaStore.getAllExtensions().size(), 1); + } + + @Test(expectedExceptions=ValidationException.class) + public void testFailureCaseRegisterExtensionMetadata() throws IOException, URISyntaxException, FalconException{ + store = ExtensionStore.get(); + createlibs(); + store.registerExtensionMetadata("test", STORAGE_URL + EXTENSION_PATH, "test desc"); + } + + private void createMETA() throws IOException{ + Path path = new Path(EXTENSION_PATH + "/META"); + if (fs.exists(path)){ + fs.delete(path, true); + } + fs.mkdirs(path); + path = new Path(EXTENSION_PATH + "/META/test.properties"); + OutputStream os = fs.create(path); + BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + br.write("Hello World"); + if (fs.exists(path)){ + fs.delete(path, true); + } + br.write("test properties"); + fs.create(path); + br.close(); + } + + private void createlibs() throws IOException{ + Path path = new Path(EXTENSION_PATH); + if (fs.exists(path)){ + fs.delete(path, true); + } + fs.mkdirs(path); + path = new Path(EXTENSION_PATH + "/libs//libs/build"); + fs.mkdirs(path); + } + + private void createReadmeAndJar() throws IOException{ + Path path = new Path(EXTENSION_PATH + "/README"); + if (fs.exists(path)){ + fs.delete(path, true); + } + fs.create(path); + path = new Path(EXTENSION_PATH + "/libs/build/test.jar"); + OutputStream os = fs.create(path); + BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); + br.write("Hello World"); + br.write("test jar"); + fs.create(path); + br.close(); + } + + private void clearDB() { + EntityManager em = FalconJPAService.get().getEntityManager(); + em.getTransaction().begin(); + try { + Query query = em.createNativeQuery("delete from EXTENSION_METADATA"); + query.executeUpdate(); + } finally { + em.getTransaction().commit(); + em.close(); + } + } + } diff --git a/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java b/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java index 27b5a9e32..d9ac756d3 100644 --- a/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java +++ b/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java @@ -89,6 +89,11 @@ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwr replication, blockSize, progress); } + @Override + public FSDataOutputStream create(Path f) throws IOException { + return localFS.create(toLocalPath(f)); + } + @Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { return localFS.append(toLocalPath(f), bufferSize, progress); diff --git a/metrics/pom.xml b/metrics/pom.xml index 8188e74a7..30aff8d1a 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -52,7 +52,8 @@ org.apache.falcon - falcon-client + falcon-common-types + ${project.version} diff --git a/pom.xml b/pom.xml index 9e4d19618..6a1598797 100644 --- a/pom.xml +++ b/pom.xml @@ -370,6 +370,7 @@ falcon-ui build-tools + common-types client shell cli diff --git a/prism/pom.xml b/prism/pom.xml index 038a3ba3d..11f3944e4 100644 --- a/prism/pom.xml +++ b/prism/pom.xml @@ -82,7 +82,14 @@ org.apache.falcon - falcon-client + falcon-common-types + ${project.version} + + + + org.apache.falcon + falcon-common-types + ${project.version} diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java index b70b3a714..266e63158 100644 --- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java @@ -450,6 +450,22 @@ public String deleteExtensionMetadata( } } + @POST + @Path("register/{extension-name}") + @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN}) + @Produces(MediaType.TEXT_PLAIN) + public String registerExtensionMetadata( + @PathParam("extension-name") String extensionName, + @QueryParam("path") String path, @QueryParam("description") String description){ + checkIfExtensionServiceIsEnabled(); + validateExtensionName(extensionName); + try { + return ExtensionStore.get().registerExtensionMetadata(extensionName, path, description); + } catch (Throwable e) { + throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR); + } + } + @GET @Path("definition/{extension-name}") @Produces({MediaType.APPLICATION_JSON}) diff --git a/test-util/pom.xml b/test-util/pom.xml index 2a0fd6fca..6e2303afd 100644 --- a/test-util/pom.xml +++ b/test-util/pom.xml @@ -68,7 +68,8 @@ org.apache.falcon - falcon-client + falcon-common-types + ${project.version}