Skip to content

Commit

Permalink
hbase-maven-plugin writes seperate core-site.xml file.
Browse files Browse the repository at this point in the history
Previously, the hbase-maven-plugin, after startup, would write an hbase-site.xml file containing
all configuration used to start the HDFS, MapReduce, and HBase clusters to the test-classes
folder of the build directory of the project. There's a disadvantage to this, in that client test
code must add HBase resources to a Configuratin object in order to get HDFS and MapReduce
configuration, even if they don't intend to use HBase.

This commit changes the hbase-maven-plugin so that it writes both an hbase-site.xml and
a core-site.xml file.  core-site.xml contains MapReduce/HDFS configuration, and hbase-site.xml
contains HBase-specific configuration. This has been tested by using the plugin in a sample
project and verifying that creating a new configuration adds the configuration found in
core-site.xml.
  • Loading branch information
kiyan committed Jun 26, 2012
1 parent f7ae6e7 commit 21312c2
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 6 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Expand Up @@ -7,6 +7,8 @@
/.settings/
/prj.el
/.jde/
.idea
*.iml

# Build artifacts.
/target/
96 changes: 90 additions & 6 deletions src/main/java/com/wibidata/maven/plugins/hbase/StartMojo.java
Expand Up @@ -77,6 +77,15 @@ public class StartMojo extends AbstractMojo {
*/
private File mHBaseSiteFile;

/**
* The file that will store the configuration required to connect to the started mini HDFS and
* MapReduce clusters. This file will be generated by the goal.
*
* @parameter property="coreSiteFile" expression="${core.site.file}" default-value="${project.build.testOutputDirectory}/core-site.xml"
* @required
*/
private File mCoreSiteFile;

/**
* If true, this goal should write an index file that provides the paths to the HBase
* configuration files written by this goal.
Expand Down Expand Up @@ -148,6 +157,15 @@ public void setHbaseSiteFile(File hbaseSiteFile) {
mHBaseSiteFile = hbaseSiteFile;
}

/**
* Sets the file that we should write the MapReduce/HDFS cluster configuration to.
*
* @param coreSiteFile The file we should write to.
*/
public void setCoreSiteFile(File coreSiteFile) {
mCoreSiteFile = coreSiteFile;
}

/**
* Sets whether this goal should write a configuration index file.
*
Expand Down Expand Up @@ -225,6 +243,8 @@ public void execute() throws MojoExecutionException {

// Write the HBase configuration file.
writeHBaseSiteFile(conf);
// Write the MapReduce/HDFS configuration file.
writeCoreSiteFile(conf);
// Write the configuration index.
if (mWriteConfIndex) {
writeConfigurationIndex();
Expand Down Expand Up @@ -297,27 +317,91 @@ private String getClassPath() throws MojoExecutionException {
}

/**
* Writes the contents of the specified configuration to the HBase site file.
* Writes the HBase-specific contents of the specified configuration to the HBase site file.
*
* @param conf The configuration to write.
* @throws MojoExecutionException If there is an error writing the file.
*/
private void writeHBaseSiteFile(Configuration conf) throws MojoExecutionException {
// Create the parent directory for the hbase conf file if it does not already exist.
createFileParentDir(mHBaseSiteFile);
writeSiteFile(getHBaseOnlyConfiguration(conf), mHBaseSiteFile);
}

/**
* Writes the MapReduce/HDFS-specific contents of the specified configuration to the core
* site file.
*
* @param conf The configuration to write.
* @throws MojoExecutionException If there is an error writing the file.
*/
private void writeCoreSiteFile(Configuration conf) throws MojoExecutionException {
writeSiteFile(getMapReduceOnlyConfiguration(conf), mCoreSiteFile);
}

/**
* Writes the specified configuration to the specified file.
*
* @param conf The configuration to write.
* @param siteFile The file to write the configuration to.
* @throws MojoExecutionException If there is an error writing the file.
*/
private void writeSiteFile(Configuration conf, File siteFile) throws MojoExecutionException {
// Create the parent directory for the site file if it does not already exist.
createFileParentDir(siteFile);

// Write the file.
FileOutputStream fileOutputStream = null;
try {
fileOutputStream = new FileOutputStream(mHBaseSiteFile);
fileOutputStream = new FileOutputStream(siteFile);
conf.writeXml(fileOutputStream);
} catch (IOException e) {
throw new MojoExecutionException(
"Unable to write to hbase conf file: " + mHBaseSiteFile.getPath(), e);
"Unable to write to site file: " + siteFile.getPath(), e);
} finally {
closeFileOutputStream(fileOutputStream);
}
getLog().info("Wrote " + mHBaseSiteFile.getPath() + ".");
getLog().info("Wrote " + siteFile.getPath() + ".");
}

/**
* Gets a new configuration created from the specified configuration, including only HBase
* configuration variables.
*
* @param conf The configuration to filter.
* @return A new configuration containing copies of the appropriate configuration variables.
*/
private Configuration getHBaseOnlyConfiguration(Configuration conf) {
return getFilteredConfiguration(conf, true);
}

/**
* Gets a new configuration created from the specified configuration, including only
* MapReduce/HDFS configuration variables.
*
* @param conf The configuration to filter.
* @return A new configuration containing copies of the appropriate configuration variables.
*/
private Configuration getMapReduceOnlyConfiguration(Configuration conf) {
return getFilteredConfiguration(conf, false);
}

/**
* Gets a new configuration created from the specified configuration,
* including only MapReduce/HDFS configuration variables or HBase only configuration variables.
*
* @param conf The configuration to filter.
* @param hBaseOnly <code>true</code> if only HBase configuration variables should be included,
* <code>false</code> if only MapReduce/HDFS configuration variables should be included.
* @return A new configuration with copies of the appropriate configuration variables.
*/
private Configuration getFilteredConfiguration(Configuration conf, boolean hBaseOnly) {
Configuration filteredConf = new Configuration(false);
for (Map.Entry<String, String> entry: conf) {
boolean startsWithHBase = entry.getKey().startsWith("hbase");
if ((startsWithHBase && hBaseOnly) || (!startsWithHBase && !hBaseOnly)) {
filteredConf.set(entry.getKey(), entry.getValue());
}
}
return filteredConf;
}

/**
Expand Down

0 comments on commit 21312c2

Please sign in to comment.