Skip to content

Commit

Permalink
create module for test dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@jetoile
jetoile committed Feb 9, 2016
1 parent 5107cf0 commit c5deff4
Show file tree
Hide file tree
Showing 34 changed files with 1,133 additions and 299 deletions.
111 changes: 111 additions & 0 deletions hadoop-bootstrap-client/hadoop-bootstrap-client-hdfs/pom.xml
View file
@@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>hadoop-bootstrap-client</artifactId>
<groupId>fr.jetoile.sample</groupId>
<version>1.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>hadoop-bootstrap-client-hdfs</artifactId>

<properties>

<hive.version>1.2.1</hive.version>

</properties>


<dependencies>

<dependency>
<groupId>fr.jetoile.sample</groupId>
<artifactId>hadoop-bootstrap-commons</artifactId>
</dependency>


<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<exclusions>
<exclusion>
<artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
</exclusions>
</dependency>
<!--<dependency>-->
<!--<groupId>org.apache.hadoop</groupId>-->
<!--<artifactId>hadoop-hdfs</artifactId>-->
<!--<version>2.7.1.2.3.2.0-2950</version>-->
<!--</dependency>-->

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>

<!-- Hive JDBC -->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
</dependency>

<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>

<!--<dependency>-->
<!--<groupId>commons-collections</groupId>-->
<!--<artifactId>commons-collections</artifactId>-->
<!--</dependency>-->

<!--<dependency>-->
<!--<groupId>commons-io</groupId>-->
<!--<artifactId>commons-io</artifactId>-->
<!--</dependency>-->

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>

<dependency>
<groupId>com.ninja-squad</groupId>
<artifactId>DbSetup</artifactId>
<version>${dbSetup.version}</version>
</dependency>

<!-- Dependances autres - test -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easytesting</groupId>
<artifactId>fest-assert</artifactId>
<version>1.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.8.5</version>
<scope>test</scope>
</dependency>



</dependencies>


</project>
61 changes: 61 additions & 0 deletions ...ent/hadoop-bootstrap-client-hdfs/src/main/java/fr/jetoile/hadoop/test/hdfs/HdfsUtils.java
View file
@@ -0,0 +1,61 @@
package fr.jetoile.hadoop.test.hdfs;


import fr.jetoile.sample.Config;
import fr.jetoile.sample.exception.ConfigException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;

public enum HdfsUtils {
INSTANCE;

private final Logger LOGGER = LoggerFactory.getLogger(HdfsUtils.class);

private FileSystem fileSystem = null;

private Configuration configuration;
private int port;
private int httpPort;

HdfsUtils() {
try {
loadConfig();
} catch (ConfigException e) {
System.exit(-1);
}
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
conf.set("fs.default.name", "hdfs://127.0.0.1:" + configuration.getInt(Config.HDFS_NAMENODE_PORT_KEY));

URI uri = URI.create("hdfs://127.0.0.1:" + configuration.getInt(Config.HDFS_NAMENODE_PORT_KEY));

try {
fileSystem = FileSystem.get(uri, conf);
} catch (IOException e) {
LOGGER.error("unable to create FileSystem", e);
}
}

private void loadConfig() throws ConfigException {
try {
configuration = new PropertiesConfiguration("default.properties");
} catch (ConfigurationException e) {
throw new ConfigException("bad config", e);
}

port = configuration.getInt(Config.HDFS_NAMENODE_PORT_KEY);
httpPort = configuration.getInt(Config.HDFS_NAMENODE_HTTP_PORT_KEY);
}


public FileSystem getFileSystem() {
return fileSystem;
}

}
108 changes: 108 additions & 0 deletions hadoop-bootstrap-client/hadoop-bootstrap-client-hive/pom.xml
View file
@@ -0,0 +1,108 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>hadoop-bootstrap-client</artifactId>
<groupId>fr.jetoile.sample</groupId>
<version>1.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>hadoop-bootstrap-client-hive</artifactId>

<properties>

<hive.version>1.2.1</hive.version>

<solr.version>5.2.1</solr.version>
<oozie.version>4.2.0.2.3.2.0-2950</oozie.version>

</properties>


<dependencies>

<dependency>
<groupId>fr.jetoile.sample</groupId>
<artifactId>hadoop-bootstrap-commons</artifactId>
</dependency>

<!--<dependency>-->
<!--<groupId>org.apache.hadoop</groupId>-->
<!--<artifactId>hadoop-hdfs</artifactId>-->
<!--<version>2.7.1.2.3.2.0-2950</version>-->
<!--</dependency>-->

<!--<dependency>-->
<!--<groupId>org.apache.hadoop</groupId>-->
<!--<artifactId>hadoop-common</artifactId>-->
<!--</dependency>-->

<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>

<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>

<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>

<dependency>
<groupId>com.ninja-squad</groupId>
<artifactId>DbSetup</artifactId>
<version>${dbSetup.version}</version>
</dependency>

<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>

<!-- Dependances autres - test -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easytesting</groupId>
<artifactId>fest-assert</artifactId>
<version>1.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.8.5</version>
<scope>test</scope>
</dependency>

<!-- Hive JDBC -->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.version}</version>
</dependency>

</dependencies>


</project>
66 changes: 66 additions & 0 deletions ...-bootstrap-client-hive/src/main/java/fr/jetoile/hadoop/test/hive/HiveConnectionUtils.java
View file
@@ -0,0 +1,66 @@
package fr.jetoile.hadoop.test.hive;

import com.ninja_squad.dbsetup.destination.Destination;
import com.ninja_squad.dbsetup.destination.DriverManagerDestination;
import fr.jetoile.sample.Config;
import fr.jetoile.sample.exception.ConfigException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.SQLException;

/**
* Utility class when using a default.properties file which contains connection's parameters
*/
public enum HiveConnectionUtils {
INSTANCE;

final private Logger LOGGER = LoggerFactory.getLogger(HiveConnectionUtils.class);

private DriverManagerDestination driverManagerDestination;
private Configuration configuration;
private String databaseName;
private String host;
private int port;

HiveConnectionUtils() {
try {
loadConfig();
} catch (ConfigException e) {
System.exit(-1);
}
driverManagerDestination = new DriverManagerDestination(
"jdbc:hive2://" + host + ":" + port + "/" + databaseName,
"user",
"pass");
}

private void loadConfig() throws ConfigException {
try {
configuration = new PropertiesConfiguration("default.properties");
} catch (ConfigurationException e) {
throw new ConfigException("bad config", e);
}

port = configuration.getInt(Config.HIVE_SERVER2_PORT_KEY);
host = configuration.getString(Config.HIVE_SERVER2_HOSTNAME_KEY);
databaseName = configuration.getString(Config.HIVE_TEST_DATABASE_NAME_KEY);
}

public Destination getDestination() {
return driverManagerDestination;
}

public Connection getConnection() {
try {
return driverManagerDestination.getConnection();
} catch (SQLException e) {
LOGGER.error("unable to create hive connection", e);
return null;
}
}
}

0 comments on commit c5deff4

Please sign in to comment.