diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
index a071d7ac1..2e0978f9d 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -101,7 +101,7 @@ public synchronized int run(final String[] args) throws Exception {
parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "",
"Extension operations like enumerate, definition, describe, list, instances, "
+ "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister"
- + ",detail",
+ + ",detail,register",
extensionCLI.createExtensionOptions(), true);
parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false);
diff --git a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
index 4d023a673..83b550fd0 100644
--- a/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
+++ b/cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
@@ -51,10 +51,13 @@ public class FalconExtensionCLI {
public static final String INSTANCES_OPT = "instances";
public static final String UNREGISTER_OPT = "unregister";
public static final String DETAIL_OPT = "detail";
+ public static final String REGISTER_OPT = "register";
// Input parameters
public static final String ENTENSION_NAME_OPT = "extensionName";
public static final String JOB_NAME_OPT = "jobName";
+ public static final String DESCRIPTION = "description";
+ public static final String PATH = "path";
public FalconExtensionCLI() {
}
@@ -70,6 +73,8 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) {
String jobName = commandLine.getOptionValue(JOB_NAME_OPT);
String filePath = commandLine.getOptionValue(FalconCLIConstants.FILE_PATH_OPT);
String doAsUser = commandLine.getOptionValue(FalconCLIConstants.DO_AS_OPT);
+ String path = commandLine.getOptionValue(FalconCLIConstants.PATH);
+ String description = commandLine.getOptionValue(FalconCLIConstants.DESCRIPTION);
if (optionsList.contains(ENUMERATE_OPT)) {
result = client.enumerateExtensions();
@@ -91,6 +96,14 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
result = client.submitExtensionJob(extensionName, filePath, doAsUser).getMessage();
+ } else if (optionsList.contains(REGISTER_OPT)) {
+ validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
+ validateRequiredParameter(path, PATH);
+ result = client.registerExtension(extensionName, path, description);
+ }else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) {
+ validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
+ validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
+ result = client.submitExtensionJob(extensionName, filePath, doAsUser).getMessage();
} else if (optionsList.contains(FalconCLIConstants.SUBMIT_AND_SCHEDULE_OPT)) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
@@ -164,6 +177,8 @@ public Options createExtensionOptions() {
Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Un-register an extension. This will make"
+ " the extension unavailable for instantiation");
Option detail = new Option(FalconCLIConstants.DETAIL, false, "Show details of a given extension");
+ Option register = new Option(FalconCLIConstants.REGISTER, false, "Register an extension with Falcon. This will "
+ + "make the extension available for instantiation for all users.");
OptionGroup group = new OptionGroup();
group.addOption(enumerate);
@@ -181,6 +196,7 @@ public Options createExtensionOptions() {
group.addOption(delete);
group.addOption(unregister);
group.addOption(detail);
+ group.addOption(register);
extensionOptions.addOptionGroup(group);
Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL");
@@ -200,6 +216,8 @@ public Options createExtensionOptions() {
Option status = new Option(FalconCLIConstants.STATUS_OPT, true, "Filter returned instances by status");
Option orderBy = new Option(FalconCLIConstants.ORDER_BY_OPT, true, "Order returned instances by this field");
Option filePath = new Option(FalconCLIConstants.FILE_PATH_OPT, true, "File path of extension parameters");
+ Option path = new Option(FalconCLIConstants.PATH, true, "Path of hdfs location for extension");
+ Option description = new Option(FalconCLIConstants.DESCRIPTION, true, "Short Description for extension");
extensionOptions.addOption(url);
extensionOptions.addOption(doAs);
@@ -216,6 +234,8 @@ public Options createExtensionOptions() {
extensionOptions.addOption(status);
extensionOptions.addOption(orderBy);
extensionOptions.addOption(filePath);
+ extensionOptions.addOption(path);
+ extensionOptions.addOption(description);
return extensionOptions;
}
diff --git a/client/pom.xml b/client/pom.xml
index 6df9d6d1a..9daa998e0 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -40,6 +40,12 @@
hadoop-auth
+
+ org.apache.falcon
+ falcon-common-types
+ ${project.version}
+
+
org.apache.hadoop
hadoop-client
@@ -124,33 +130,6 @@
-
- org.jvnet.jaxb2.maven2
- maven-jaxb2-plugin
- 0.8.0
-
-
- 1
- generate-sources
-
- generate
-
-
-
-
-
- -extension
- -Xinheritance
-
-
-
- org.jvnet.jaxb2_commons
- jaxb2-basics
- 0.6.3
-
-
-
-
org.apache.maven.plugins
maven-compiler-plugin
diff --git a/client/src/main/java/org/apache/falcon/ValidationUtil.java b/client/src/main/java/org/apache/falcon/ValidationUtil.java
index cdf1f0594..3db521ff5 100644
--- a/client/src/main/java/org/apache/falcon/ValidationUtil.java
+++ b/client/src/main/java/org/apache/falcon/ValidationUtil.java
@@ -108,7 +108,7 @@ public static void validateEntityTypeForSummary(String type) {
public static List getLifeCycle(String lifeCycleValue) {
if (lifeCycleValue != null) {
String[] lifeCycleValues = lifeCycleValue.split(",");
- List lifeCycles = new ArrayList();
+ List lifeCycles = new ArrayList<>();
try {
for (String lifeCycle : lifeCycleValues) {
lifeCycles.add(LifeCycle.valueOf(lifeCycle.toUpperCase().trim()));
diff --git a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
index 3775771e8..26e893744 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconCLIConstants.java
@@ -219,4 +219,7 @@ private FalconCLIConstants(){
public static final String DO_AS_DESCRIPTION = "doAs user";
public static final String UREGISTER = "unregister";
public static final String DETAIL = "detail";
+ public static final String REGISTER = "register";
+ public static final String PATH = "path";
+ public static final String DESCRIPTION = "description";
}
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 3db249ae4..dabed3f72 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -348,7 +348,8 @@ protected static enum ExtensionOperations {
RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML),
DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML),
UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN),
- DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON);;
+ DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON),
+ REGISTER("api/extension/register/", HttpMethod.POST, MediaType.TEXT_PLAIN);
private String path;
private String method;
@@ -1036,6 +1037,14 @@ public String getExtensionDetail(final String extensionName) {
return getResponse(String.class, clientResponse);
}
+ public String registerExtension(final String extensionName, final String path, final String description) {
+ ClientResponse clientResponse = new ResourceBuilder()
+ .path(ExtensionOperations.REGISTER.path, extensionName).addQueryParam(PATH, path)
+ .addQueryParam(FalconCLIConstants.DESCRIPTION, description)
+ .call(ExtensionOperations.REGISTER);
+ return getResponse(String.class, clientResponse);
+ }
+
public String getExtensionDefinition(final String extensionName) {
ClientResponse clientResponse = new ResourceBuilder()
.path(ExtensionOperations.DEFINITION.path, extensionName)
diff --git a/common-types/pom.xml b/common-types/pom.xml
new file mode 100644
index 000000000..714b166fc
--- /dev/null
+++ b/common-types/pom.xml
@@ -0,0 +1,164 @@
+
+
+
+
+
+
+ 4.0.0
+
+ org.apache.falcon
+ falcon-main
+ 0.11-SNAPSHOT
+
+ falcon-common-types
+ Apache Falcon Java common types
+ Apache Falcon Java common types
+ jar
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-auth
+
+
+
+ org.apache.hadoop
+ hadoop-client
+
+
+
+ commons-cli
+ commons-cli
+
+
+
+ commons-net
+ commons-net
+
+
+
+ commons-codec
+ commons-codec
+
+
+
+ org.apache.commons
+ commons-lang3
+
+
+
+ com.sun.jersey
+ jersey-client
+
+
+
+ com.sun.jersey
+ jersey-json
+
+
+
+ org.slf4j
+ slf4j-api
+ provided
+
+
+
+ log4j
+ log4j
+
+
+
+ commons-io
+ commons-io
+
+
+
+ com.github.stephenc.findbugs
+ findbugs-annotations
+
+
+
+ org.testng
+ testng
+
+
+
+ org.apache.hive.hcatalog
+ hive-webhcat-java-client
+ ${hive.version}
+
+
+
+
+
+
+ maven-dependency-plugin
+
+
+ package
+
+ copy-dependencies
+
+
+ ${project.build.directory}/dependency
+
+
+
+
+
+ org.jvnet.jaxb2.maven2
+ maven-jaxb2-plugin
+ 0.8.0
+
+
+ 1
+ generate-sources
+
+ generate
+
+
+
+
+
+ -extension
+ -Xinheritance
+
+
+
+ org.jvnet.jaxb2_commons
+ jaxb2-basics
+ 0.6.3
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+
+ 1.7
+
+
+
+
+
diff --git a/client/src/main/java/org/apache/falcon/LifeCycle.java b/common-types/src/main/java/org/apache/falcon/LifeCycle.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/LifeCycle.java
rename to common-types/src/main/java/org/apache/falcon/LifeCycle.java
diff --git a/client/src/main/java/org/apache/falcon/Pair.java b/common-types/src/main/java/org/apache/falcon/Pair.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/Pair.java
rename to common-types/src/main/java/org/apache/falcon/Pair.java
diff --git a/client/src/main/java/org/apache/falcon/Tag.java b/common-types/src/main/java/org/apache/falcon/Tag.java
similarity index 85%
rename from client/src/main/java/org/apache/falcon/Tag.java
rename to common-types/src/main/java/org/apache/falcon/Tag.java
index a8d60b69d..3c2f25e20 100644
--- a/client/src/main/java/org/apache/falcon/Tag.java
+++ b/common-types/src/main/java/org/apache/falcon/Tag.java
@@ -24,12 +24,15 @@
* Tag to include in the entity type.
*/
public enum Tag {
- DEFAULT(EntityType.PROCESS), RETENTION(EntityType.FEED), REPLICATION(EntityType.FEED),
- IMPORT(EntityType.FEED), EXPORT(EntityType.FEED);
+ DEFAULT(EntityType.PROCESS),
+ RETENTION(EntityType.FEED),
+ REPLICATION(EntityType.FEED),
+ IMPORT(EntityType.FEED),
+ EXPORT(EntityType.FEED);
private final EntityType entityType;
- private Tag(EntityType entityType) {
+ Tag(EntityType entityType) {
this.entityType = entityType;
}
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java b/common-types/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/AccessControlList.java
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/DateValidator.java b/common-types/src/main/java/org/apache/falcon/entity/v0/DateValidator.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/DateValidator.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/DateValidator.java
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/Entity.java b/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/Entity.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java b/common-types/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/EntityNotification.java
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/EntityType.java b/common-types/src/main/java/org/apache/falcon/entity/v0/EntityType.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/EntityType.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/EntityType.java
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/Frequency.java b/common-types/src/main/java/org/apache/falcon/entity/v0/Frequency.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/Frequency.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/Frequency.java
diff --git a/client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java b/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
rename to common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java b/common-types/src/main/java/org/apache/falcon/extensions/ExtensionType.java
similarity index 100%
rename from extensions/src/main/java/org/apache/falcon/extensions/ExtensionType.java
rename to common-types/src/main/java/org/apache/falcon/extensions/ExtensionType.java
diff --git a/client/src/main/java/org/apache/falcon/metadata/RelationshipType.java b/common-types/src/main/java/org/apache/falcon/metadata/RelationshipType.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/metadata/RelationshipType.java
rename to common-types/src/main/java/org/apache/falcon/metadata/RelationshipType.java
diff --git a/client/src/main/java/org/apache/falcon/resource/APIResult.java b/common-types/src/main/java/org/apache/falcon/resource/APIResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/APIResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/APIResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/EntityList.java b/common-types/src/main/java/org/apache/falcon/resource/EntityList.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/EntityList.java
rename to common-types/src/main/java/org/apache/falcon/resource/EntityList.java
diff --git a/client/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java b/common-types/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/EntitySummaryResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java b/common-types/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
rename to common-types/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java
diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java b/common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
rename to common-types/src/main/java/org/apache/falcon/resource/ExtensionJobList.java
diff --git a/client/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java b/common-types/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/FeedInstanceResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/FeedLookupResult.java b/common-types/src/main/java/org/apache/falcon/resource/FeedLookupResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/FeedLookupResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/FeedLookupResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java b/common-types/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/InstanceDependencyResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/InstancesResult.java b/common-types/src/main/java/org/apache/falcon/resource/InstancesResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/InstancesResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/InstancesResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java b/common-types/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/InstancesSummaryResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/LineageGraphResult.java b/common-types/src/main/java/org/apache/falcon/resource/LineageGraphResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/LineageGraphResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/LineageGraphResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java b/common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java
rename to common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java
diff --git a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java b/common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/SchedulableEntityInstanceResult.java
diff --git a/client/src/main/java/org/apache/falcon/resource/TriageResult.java b/common-types/src/main/java/org/apache/falcon/resource/TriageResult.java
similarity index 100%
rename from client/src/main/java/org/apache/falcon/resource/TriageResult.java
rename to common-types/src/main/java/org/apache/falcon/resource/TriageResult.java
diff --git a/client/src/main/resources/cluster-0.1.xsd b/common-types/src/main/resources/cluster-0.1.xsd
similarity index 100%
rename from client/src/main/resources/cluster-0.1.xsd
rename to common-types/src/main/resources/cluster-0.1.xsd
diff --git a/client/src/main/resources/datasource-0.1.xsd b/common-types/src/main/resources/datasource-0.1.xsd
similarity index 100%
rename from client/src/main/resources/datasource-0.1.xsd
rename to common-types/src/main/resources/datasource-0.1.xsd
diff --git a/client/src/main/resources/feed-0.1.xsd b/common-types/src/main/resources/feed-0.1.xsd
similarity index 100%
rename from client/src/main/resources/feed-0.1.xsd
rename to common-types/src/main/resources/feed-0.1.xsd
diff --git a/client/src/main/resources/jaxb-binding.xjb b/common-types/src/main/resources/jaxb-binding.xjb
similarity index 100%
rename from client/src/main/resources/jaxb-binding.xjb
rename to common-types/src/main/resources/jaxb-binding.xjb
diff --git a/client/src/main/resources/mysql_database.xml b/common-types/src/main/resources/mysql_database.xml
similarity index 100%
rename from client/src/main/resources/mysql_database.xml
rename to common-types/src/main/resources/mysql_database.xml
diff --git a/client/src/main/resources/process-0.1.xsd b/common-types/src/main/resources/process-0.1.xsd
similarity index 100%
rename from client/src/main/resources/process-0.1.xsd
rename to common-types/src/main/resources/process-0.1.xsd
diff --git a/client/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java b/common-types/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java
similarity index 100%
rename from client/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java
rename to common-types/src/test/java/org/apache/falcon/entity/v0/DateValidatorTest.java
diff --git a/client/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java b/common-types/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java
similarity index 100%
rename from client/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java
rename to common-types/src/test/java/org/apache/falcon/entity/v0/TestFrequency.java
diff --git a/common/pom.xml b/common/pom.xml
index 96cb7f571..34822aa56 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -92,7 +92,8 @@
org.apache.falcon
- falcon-client
+ falcon-common-types
+ ${project.version}
diff --git a/extensions/pom.xml b/extensions/pom.xml
index 77821cac3..df8f35f0f 100644
--- a/extensions/pom.xml
+++ b/extensions/pom.xml
@@ -71,7 +71,8 @@
org.apache.falcon
- falcon-client
+ falcon-common-types
+ ${project.version}
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index 02f7e6218..e15919f39 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -25,12 +25,14 @@
import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.entity.parser.ValidationException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.io.IOUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -39,6 +41,8 @@
import org.apache.falcon.util.StartupProperties;
import org.apache.falcon.entity.store.StoreAccessException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -264,6 +268,54 @@ public String deleteExtensionMetadata(final String extensionName) throws Validat
}
}
+ public String registerExtensionMetadata(final String extensionName, final String path, final String description)
+ throws URISyntaxException, FalconException {
+ Configuration conf = new Configuration();
+ URI uri = new URI(path);
+ conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority());
+ FileSystem fileSystem = HadoopClientFactory.get().createFalconFileSystem(uri);
+ try {
+ fileSystem.listStatus(new Path(uri.getPath() + "/README"));
+ } catch (IOException e){
+ LOG.error("Exception in registerExtensionMetadata:", e);
+ throw new ValidationException("README file is not present in the " + path);
+ }
+ PathFilter filter=new PathFilter(){
+ public boolean accept(Path file){
+ return file.getName().endsWith(".jar");
+ }
+ };
+ FileStatus[] jarStatus;
+ try {
+ jarStatus = fileSystem.listStatus(new Path(uri.getPath() + "/libs/build"), filter);
+ if (jarStatus.length <=0) {
+ throw new ValidationException("Jars are not present in the " + uri.getPath() + "libs/build.");
+ }
+ } catch (IOException e){
+ LOG.error("Exception in registerExtensionMetadata:", e);
+ throw new ValidationException("Jars are not present in the " + uri.getPath() + "libs/build.");
+ }
+ FileStatus[] propStatus;
+ try{
+ propStatus = fileSystem.listStatus(new Path(uri.getPath() + "/META"));
+ if (propStatus.length <=0){
+ throw new ValidationException("No properties file is not present in the " + uri.getPath() + "/META"
+ + " structure.");
+ }
+ } catch (IOException e){
+ LOG.error("Exception in registerExtensionMetadata:", e);
+ throw new ValidationException("Directory is not present in the " + uri.getPath() + "/META"
+ + " structure.");
+ }
+
+ if (!metaStore.checkIfExtensionExists(extensionName)){
+ metaStore.storeExtensionMetadataBean(extensionName, path, ExtensionType.CUSTOM, description);
+ }else{
+ throw new ValidationException(extensionName + " already exsists.");
+ }
+ return "Extension :" + extensionName + " registered succesfully.";
+ }
+
public String getResource(final String extensionName, final String resourceName) throws StoreAccessException {
Map resources = getExtensionArtifacts(extensionName);
if (resources.isEmpty()) {
diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
index bc04d8834..27bea5355 100644
--- a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
+++ b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java
@@ -19,19 +19,37 @@
package org.apache.falcon.extensions.store;
import com.google.common.collect.ImmutableMap;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.net.URISyntaxException;
+import java.util.Map;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.entity.parser.ValidationException;
import org.apache.falcon.entity.store.StoreAccessException;
+import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.extensions.mirroring.hdfs.HdfsMirroringExtension;
+import org.apache.falcon.hadoop.JailedFileSystem;
+import org.apache.falcon.service.FalconJPAService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-import java.util.Map;
-
/**
* Tests for extension store.
*/
public class ExtensionStoreTest extends AbstractTestExtensionStore {
private static Map resourcesMap;
+ private static JailedFileSystem fs;
+ protected static final String EXTENSION_PATH = "/projects/falcon/extension";
+ private static final String STORAGE_URL = "jail://global:00";
@BeforeClass
public void init() throws Exception {
@@ -46,6 +64,8 @@ public void init() throws Exception {
"hdfs-snapshot-mirroring-workflow.xml", extensionStorePath
+ "/hdfs-mirroring/resources/runtime/hdfs-snapshot-mirroring-workflow.xml"
);
+ fs = new JailedFileSystem();
+ initFileSystem();
}
@Test
@@ -66,5 +86,96 @@ public void testGetExtensionLibPath() throws StoreAccessException {
Assert.assertEquals(store.getExtensionLibPath(extensionName), libPath);
}
+ private static void initFileSystem() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", STORAGE_URL);
+ fs.initialize(LocalFileSystem.getDefaultUri(conf), conf);
+ }
+
+ @BeforeMethod
+ public void clean() {
+ clearDB();
+ // Clean up FS
+ try {
+ fs.delete(new Path(extensionStorePath + "/hdfs-mirroring"), true);
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+
+
+ @Test
+ public void testRegisterExtensionMetadata() throws IOException, URISyntaxException, FalconException{
+ createlibs();
+ createReadmeAndJar();
+ createMETA();
+ store = ExtensionStore.get();
+ store.registerExtensionMetadata("test", STORAGE_URL + EXTENSION_PATH, "test desc");
+ ExtensionMetaStore metaStore = new ExtensionMetaStore();
+ Assert.assertEquals(metaStore.getAllExtensions().size(), 1);
+ }
+
+ @Test(expectedExceptions=ValidationException.class)
+ public void testFailureCaseRegisterExtensionMetadata() throws IOException, URISyntaxException, FalconException{
+ store = ExtensionStore.get();
+ createlibs();
+ store.registerExtensionMetadata("test", STORAGE_URL + EXTENSION_PATH, "test desc");
+ }
+
+ private void createMETA() throws IOException{
+ Path path = new Path(EXTENSION_PATH + "/META");
+ if (fs.exists(path)){
+ fs.delete(path, true);
+ }
+ fs.mkdirs(path);
+ path = new Path(EXTENSION_PATH + "/META/test.properties");
+ OutputStream os = fs.create(path);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
+ br.write("Hello World");
+ if (fs.exists(path)){
+ fs.delete(path, true);
+ }
+ br.write("test properties");
+ fs.create(path);
+ br.close();
+ }
+
+ private void createlibs() throws IOException{
+ Path path = new Path(EXTENSION_PATH);
+ if (fs.exists(path)){
+ fs.delete(path, true);
+ }
+ fs.mkdirs(path);
+ path = new Path(EXTENSION_PATH + "/libs//libs/build");
+ fs.mkdirs(path);
+ }
+
+ private void createReadmeAndJar() throws IOException{
+ Path path = new Path(EXTENSION_PATH + "/README");
+ if (fs.exists(path)){
+ fs.delete(path, true);
+ }
+ fs.create(path);
+ path = new Path(EXTENSION_PATH + "/libs/build/test.jar");
+ OutputStream os = fs.create(path);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
+ br.write("Hello World");
+ br.write("test jar");
+ fs.create(path);
+ br.close();
+ }
+
+ private void clearDB() {
+ EntityManager em = FalconJPAService.get().getEntityManager();
+ em.getTransaction().begin();
+ try {
+ Query query = em.createNativeQuery("delete from EXTENSION_METADATA");
+ query.executeUpdate();
+ } finally {
+ em.getTransaction().commit();
+ em.close();
+ }
+ }
+
}
diff --git a/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java b/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java
index 27b5a9e32..d9ac756d3 100644
--- a/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java
+++ b/hadoop-dependencies/src/main/java/org/apache/falcon/hadoop/JailedFileSystem.java
@@ -89,6 +89,11 @@ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwr
replication, blockSize, progress);
}
+ @Override
+ public FSDataOutputStream create(Path f) throws IOException {
+ return localFS.create(toLocalPath(f));
+ }
+
@Override
public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {
return localFS.append(toLocalPath(f), bufferSize, progress);
diff --git a/metrics/pom.xml b/metrics/pom.xml
index 8188e74a7..30aff8d1a 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -52,7 +52,8 @@
org.apache.falcon
- falcon-client
+ falcon-common-types
+ ${project.version}
diff --git a/pom.xml b/pom.xml
index 9e4d19618..6a1598797 100644
--- a/pom.xml
+++ b/pom.xml
@@ -370,6 +370,7 @@
falcon-ui
build-tools
+ common-types
client
shell
cli
diff --git a/prism/pom.xml b/prism/pom.xml
index 038a3ba3d..11f3944e4 100644
--- a/prism/pom.xml
+++ b/prism/pom.xml
@@ -82,7 +82,14 @@
org.apache.falcon
- falcon-client
+ falcon-common-types
+ ${project.version}
+
+
+
+ org.apache.falcon
+ falcon-common-types
+ ${project.version}
diff --git a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
index b70b3a714..266e63158 100644
--- a/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/extensions/ExtensionManager.java
@@ -450,6 +450,22 @@ public String deleteExtensionMetadata(
}
}
+ @POST
+ @Path("register/{extension-name}")
+ @Consumes({MediaType.TEXT_XML, MediaType.TEXT_PLAIN})
+ @Produces(MediaType.TEXT_PLAIN)
+ public String registerExtensionMetadata(
+ @PathParam("extension-name") String extensionName,
+ @QueryParam("path") String path, @QueryParam("description") String description){
+ checkIfExtensionServiceIsEnabled();
+ validateExtensionName(extensionName);
+ try {
+ return ExtensionStore.get().registerExtensionMetadata(extensionName, path, description);
+ } catch (Throwable e) {
+ throw FalconWebException.newAPIException(e, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
@GET
@Path("definition/{extension-name}")
@Produces({MediaType.APPLICATION_JSON})
diff --git a/test-util/pom.xml b/test-util/pom.xml
index 2a0fd6fca..6e2303afd 100644
--- a/test-util/pom.xml
+++ b/test-util/pom.xml
@@ -68,7 +68,8 @@
org.apache.falcon
- falcon-client
+ falcon-common-types
+ ${project.version}