Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/apache/falcon
Browse files Browse the repository at this point in the history
  • Loading branch information
sandeepSamudrala committed Nov 25, 2016
2 parents bbca081 + 1b7708f commit 9cf36e9
Show file tree
Hide file tree
Showing 51 changed files with 416 additions and 42 deletions.
2 changes: 1 addition & 1 deletion cli/src/main/java/org/apache/falcon/cli/FalconCLI.java
Expand Up @@ -101,7 +101,7 @@ public synchronized int run(final String[] args) throws Exception {
parser.addCommand(FalconCLIConstants.EXTENSION_CMD, "",
"Extension operations like enumerate, definition, describe, list, instances, "
+ "submit, submitAndSchedule, schedule, suspend, resume, delete, update, validate,unregister"
+ ",detail",
+ ",detail,register",
extensionCLI.createExtensionOptions(), true);
parser.addCommand(FalconCLIConstants.VERSION_OPT, "", "show client version", new Options(), false);

Expand Down
20 changes: 20 additions & 0 deletions cli/src/main/java/org/apache/falcon/cli/FalconExtensionCLI.java
Expand Up @@ -51,10 +51,13 @@ public class FalconExtensionCLI {
public static final String INSTANCES_OPT = "instances";
public static final String UNREGISTER_OPT = "unregister";
public static final String DETAIL_OPT = "detail";
public static final String REGISTER_OPT = "register";

// Input parameters
public static final String ENTENSION_NAME_OPT = "extensionName";
public static final String JOB_NAME_OPT = "jobName";
public static final String DESCRIPTION = "description";
public static final String PATH = "path";

public FalconExtensionCLI() {
}
Expand All @@ -70,6 +73,8 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) {
String jobName = commandLine.getOptionValue(JOB_NAME_OPT);
String filePath = commandLine.getOptionValue(FalconCLIConstants.FILE_PATH_OPT);
String doAsUser = commandLine.getOptionValue(FalconCLIConstants.DO_AS_OPT);
String path = commandLine.getOptionValue(FalconCLIConstants.PATH);
String description = commandLine.getOptionValue(FalconCLIConstants.DESCRIPTION);

if (optionsList.contains(ENUMERATE_OPT)) {
result = client.enumerateExtensions();
Expand All @@ -91,6 +96,14 @@ public void extensionCommand(CommandLine commandLine, FalconClient client) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
result = client.submitExtensionJob(extensionName, filePath, doAsUser).getMessage();
} else if (optionsList.contains(REGISTER_OPT)) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(path, PATH);
result = client.registerExtension(extensionName, path, description);
}else if (optionsList.contains(FalconCLIConstants.SUBMIT_OPT)) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
result = client.submitExtensionJob(extensionName, filePath, doAsUser).getMessage();
} else if (optionsList.contains(FalconCLIConstants.SUBMIT_AND_SCHEDULE_OPT)) {
validateRequiredParameter(extensionName, ENTENSION_NAME_OPT);
validateRequiredParameter(filePath, FalconCLIConstants.FILE_PATH_OPT);
Expand Down Expand Up @@ -164,6 +177,8 @@ public Options createExtensionOptions() {
Option unregister = new Option(FalconCLIConstants.UREGISTER, false, "Un-register an extension. This will make"
+ " the extension unavailable for instantiation");
Option detail = new Option(FalconCLIConstants.DETAIL, false, "Show details of a given extension");
Option register = new Option(FalconCLIConstants.REGISTER, false, "Register an extension with Falcon. This will "
+ "make the extension available for instantiation for all users.");

OptionGroup group = new OptionGroup();
group.addOption(enumerate);
Expand All @@ -181,6 +196,7 @@ public Options createExtensionOptions() {
group.addOption(delete);
group.addOption(unregister);
group.addOption(detail);
group.addOption(register);
extensionOptions.addOptionGroup(group);

Option url = new Option(FalconCLIConstants.URL_OPTION, true, "Falcon URL");
Expand All @@ -200,6 +216,8 @@ public Options createExtensionOptions() {
Option status = new Option(FalconCLIConstants.STATUS_OPT, true, "Filter returned instances by status");
Option orderBy = new Option(FalconCLIConstants.ORDER_BY_OPT, true, "Order returned instances by this field");
Option filePath = new Option(FalconCLIConstants.FILE_PATH_OPT, true, "File path of extension parameters");
Option path = new Option(FalconCLIConstants.PATH, true, "Path of hdfs location for extension");
Option description = new Option(FalconCLIConstants.DESCRIPTION, true, "Short Description for extension");

extensionOptions.addOption(url);
extensionOptions.addOption(doAs);
Expand All @@ -216,6 +234,8 @@ public Options createExtensionOptions() {
extensionOptions.addOption(status);
extensionOptions.addOption(orderBy);
extensionOptions.addOption(filePath);
extensionOptions.addOption(path);
extensionOptions.addOption(description);

return extensionOptions;
}
Expand Down
33 changes: 6 additions & 27 deletions client/pom.xml
Expand Up @@ -40,6 +40,12 @@
<artifactId>hadoop-auth</artifactId>
</dependency>

<dependency>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-common-types</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
Expand Down Expand Up @@ -124,33 +130,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jvnet.jaxb2.maven2</groupId>
<artifactId>maven-jaxb2-plugin</artifactId>
<version>0.8.0</version>
<executions>
<execution>
<id>1</id>
<phase>generate-sources</phase>
<goals>
<goal>generate</goal>
</goals>
</execution>
</executions>
<configuration>
<args>
<arg>-extension</arg>
<arg>-Xinheritance</arg>
</args>
<plugins>
<plugin>
<groupId>org.jvnet.jaxb2_commons</groupId>
<artifactId>jaxb2-basics</artifactId>
<version>0.6.3</version>
</plugin>
</plugins>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion client/src/main/java/org/apache/falcon/ValidationUtil.java
Expand Up @@ -108,7 +108,7 @@ public static void validateEntityTypeForSummary(String type) {
public static List<LifeCycle> getLifeCycle(String lifeCycleValue) {
if (lifeCycleValue != null) {
String[] lifeCycleValues = lifeCycleValue.split(",");
List<LifeCycle> lifeCycles = new ArrayList<LifeCycle>();
List<LifeCycle> lifeCycles = new ArrayList<>();
try {
for (String lifeCycle : lifeCycleValues) {
lifeCycles.add(LifeCycle.valueOf(lifeCycle.toUpperCase().trim()));
Expand Down
Expand Up @@ -219,4 +219,7 @@ private FalconCLIConstants(){
public static final String DO_AS_DESCRIPTION = "doAs user";
public static final String UREGISTER = "unregister";
public static final String DETAIL = "detail";
public static final String REGISTER = "register";
public static final String PATH = "path";
public static final String DESCRIPTION = "description";
}
11 changes: 10 additions & 1 deletion client/src/main/java/org/apache/falcon/client/FalconClient.java
Expand Up @@ -348,7 +348,8 @@ protected static enum ExtensionOperations {
RESUME("api/extension/resume", HttpMethod.POST, MediaType.TEXT_XML),
DELETE("api/extension/delete", HttpMethod.POST, MediaType.TEXT_XML),
UNREGISTER("api/extension/unregister/", HttpMethod.POST, MediaType.TEXT_PLAIN),
DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON);;
DETAIL("api/extension/detail/", HttpMethod.GET, MediaType.APPLICATION_JSON),
REGISTER("api/extension/register/", HttpMethod.POST, MediaType.TEXT_PLAIN);

private String path;
private String method;
Expand Down Expand Up @@ -1036,6 +1037,14 @@ public String getExtensionDetail(final String extensionName) {
return getResponse(String.class, clientResponse);
}

public String registerExtension(final String extensionName, final String path, final String description) {
ClientResponse clientResponse = new ResourceBuilder()
.path(ExtensionOperations.REGISTER.path, extensionName).addQueryParam(PATH, path)
.addQueryParam(FalconCLIConstants.DESCRIPTION, description)
.call(ExtensionOperations.REGISTER);
return getResponse(String.class, clientResponse);
}

public String getExtensionDefinition(final String extensionName) {
ClientResponse clientResponse = new ResourceBuilder()
.path(ExtensionOperations.DEFINITION.path, extensionName)
Expand Down
164 changes: 164 additions & 0 deletions common-types/pom.xml
@@ -0,0 +1,164 @@
<?xml version="1.0" encoding="UTF-8"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-main</artifactId>
<version>0.11-SNAPSHOT</version>
</parent>
<artifactId>falcon-common-types</artifactId>
<description>Apache Falcon Java common types</description>
<name>Apache Falcon Java common types</name>
<packaging>jar</packaging>


<dependencies>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>

<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>

<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
</dependency>

<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>

<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</dependency>

<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>

<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>

<dependency>
<groupId>com.github.stephenc.findbugs</groupId>
<artifactId>findbugs-annotations</artifactId>
</dependency>

<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-webhcat-java-client</artifactId>
<version>${hive.version}</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jvnet.jaxb2.maven2</groupId>
<artifactId>maven-jaxb2-plugin</artifactId>
<version>0.8.0</version>
<executions>
<execution>
<id>1</id>
<phase>generate-sources</phase>
<goals>
<goal>generate</goal>
</goals>
</execution>
</executions>
<configuration>
<args>
<arg>-extension</arg>
<arg>-Xinheritance</arg>
</args>
<plugins>
<plugin>
<groupId>org.jvnet.jaxb2_commons</groupId>
<artifactId>jaxb2-basics</artifactId>
<version>0.6.3</version>
</plugin>
</plugins>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
Expand Up @@ -24,12 +24,15 @@
* Tag to include in the entity type.
*/
public enum Tag {
DEFAULT(EntityType.PROCESS), RETENTION(EntityType.FEED), REPLICATION(EntityType.FEED),
IMPORT(EntityType.FEED), EXPORT(EntityType.FEED);
DEFAULT(EntityType.PROCESS),
RETENTION(EntityType.FEED),
REPLICATION(EntityType.FEED),
IMPORT(EntityType.FEED),
EXPORT(EntityType.FEED);

private final EntityType entityType;

private Tag(EntityType entityType) {
Tag(EntityType entityType) {
this.entityType = entityType;
}

Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 2 additions & 1 deletion common/pom.xml
Expand Up @@ -92,7 +92,8 @@

<dependency>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-client</artifactId>
<artifactId>falcon-common-types</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
Expand Down
3 changes: 2 additions & 1 deletion extensions/pom.xml
Expand Up @@ -71,7 +71,8 @@

<dependency>
<groupId>org.apache.falcon</groupId>
<artifactId>falcon-client</artifactId>
<artifactId>falcon-common-types</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
Expand Down

0 comments on commit 9cf36e9

Please sign in to comment.