removed CLI args (they may come back at some point) and improved prop…

…erty file support

src/main/config/example.conf

@@ -1,61 +1,79 @@
 #############################################################################
 #
 #  A configuration file which can be used with the
-#  "--config-file" option.  Command-line arguments
-#  will override values in this file.
-#
-#  For running in any of the two daemon modes, they
-#  must be specified on the command-line.
+#  "__config_file" option.  
 #
 #############################################################################
 
 # A name used for the PID file, as well as the log filename, to support
 # multiple Slurpers instances working from the same installation directory.
 #
-DATASOURCE-NAME = alex
+DATASOURCE_NAME = alex
 
-# The source directory.  This must be a fully-qualified URI.
+# The source directory.  This must be a fully_qualified URI.
 #
-SRC-DIR = file:/tmp/slurper/in
+SRC_DIR = file:/tmp/slurper/in
 
-# The work directory.  This must be a fully-qualified URI, and must be on the
-# same file system as SRC-DIR.
+# The work directory.  This must be a fully_qualified URI, and must be on the
+# same file system as SRC_DIR.
 #
-WORK-DIR = file:/tmp/slurper/work
+WORK_DIR = file:/tmp/slurper/work
 
-# The completed directory.  This must be a fully-qualified URI, and must be on the
-# same file system as SRC-DIR.
+# The completed directory.  This must be a fully_qualified URI, and must be on the
+# same file system as SRC_DIR.
 #
-COMPLETE-DIR = file:/tmp/slurper/complete
+COMPLETE_DIR = file:/tmp/slurper/complete
 
 # Whether the file should be removed after the copy has completed.
-# Either this or COMPLETE-DIR must be specified.
+# Either this or COMPLETE_DIR must be specified.
 #
-REMOVE-AFTER-COPY = false
+REMOVE_AFTER_COPY = false
 
-# The error directory.  This must be a fully-qualified URI, and must be on the
-# same file system as SRC-DIR.
+# The error directory.  This must be a fully_qualified URI, and must be on the
+# same file system as SRC_DIR.
 #
-ERROR-DIR = file:/tmp/slurper/error
+ERROR_DIR = file:/tmp/slurper/error
 
-# The destination staging directory.  This must be a fully-qualified URI, and must be on the
-# same file system as DEST-DIR.
+# The destination staging directory.  This must be a fully_qualified URI, and must be on the
+# same file system as DEST_DIR.
 #
-DEST-STAGING-DIR = hdfs:/tmp/slurper/stage
+DEST_STAGING_DIR = hdfs:/tmp/slurper/stage
 
-# The destination directory.  This must be a fully-qualified URI.
+# The destination directory.  This must be a fully_qualified URI.
 #
-DEST-DIR = hdfs:/tmp/slurper/dest
+DEST_DIR = hdfs:/tmp/slurper/dest
 
 # The compression codec which should be used to compress the output.
 #
-# COMPRESS = com.hadoop.compression.lzo.LzopCodec
+COMPRESSION_CODEC = com.hadoop.compression.lzo.LzopCodec
 
 # If the destination file is LZOP, this option will create an index file.
 #
-# CREATE-LZO-INDEX = true
+CREATE_LZO_INDEX = true
 
 # Reads the destination file after the copy has completed and verifies
 # its integrity.
 #
-# VERIFY = true
+# VERIFY = true
+
+# A script which can be called to dynamically determine the destination path.
+# The standard input will contain a single line with the fully qualified URI
+# of the source file, and the script must put the destination fully qualified URI
+# on standard out.
+#
+# This and the DEST_DIR configuration setting are mutually exclusive; only one can be set.
+#
+# SCRIPT = /tmp/sample-python.py
+
+# A script is called prior to the file being copied.
+# The standard input will contain a single line with the fully qualified URI
+# of the source file in the source work directory (see WORK_DIR above).
+# The script must put the modified (or unchanged) fully qualified URI of the
+# file in the WORK_DIR.  This script can perform pre-processing manipulations
+# such as decompressing the file, or injecting a date/time into the filename.
+#
+# WORK_SCRIPT = /tmp/sample-stage-python.py
+
+# The number of threads to be used for slurping
+#
+THREADS = 1

src/main/config/slurper-env.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+##########################################################
+#
+# This file is sourced by slurper shell scripts to load
+# properties which will override any settings in the
+# environment.
+#
+##########################################################
+
+
+# Set the directory for the local Java installation
+#
+# export JAVA_HOME=/usr/java/latest
+
+
+# Set the directory for the local Hadoop installation
+#
+# export HADOOP_HOME=/usr/local/hadoop
+

src/main/java/com/alexholmes/hdfsslurper/Config.java

@@ -0,0 +1,207 @@
+/*
+ * Copyright 2011 Alex Holmes
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alexholmes.hdfsslurper;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CompressionCodec;
+
+public class Config {
+
+  private String datasource;
+  private CompressionCodec codec;
+  private boolean createLzopIndex;
+  private Path srcDir;
+  private Path workDir;
+  private Path completeDir;
+  private Path errorDir;
+  private Path destDir;
+  private Path destStagingDir;
+  private String script;
+  private String workScript;
+  private boolean remove;
+  private boolean verify;
+  private int numThreads;
+  private long pollSleepPeriodMillis;
+  FileSystem srcFs;
+  FileSystem destFs;
+  Configuration config = new Configuration();
+
+
+  public String getDatasource() {
+    return datasource;
+  }
+
+  public Config setDatasource(String datasource) {
+    this.datasource = datasource;
+    return this;
+  }
+
+  public CompressionCodec getCodec() {
+    return codec;
+  }
+
+  public Config setCodec(CompressionCodec codec) {
+    this.codec = codec;
+    return this;
+  }
+
+  public boolean isCreateLzopIndex() {
+    return createLzopIndex;
+  }
+
+  public Config setCreateLzopIndex(boolean createLzopIndex) {
+    this.createLzopIndex = createLzopIndex;
+    return this;
+  }
+
+  public Path getSrcDir() {
+    return srcDir;
+  }
+
+  public Config setSrcDir(Path srcDir) {
+    this.srcDir = srcDir;
+    return this;
+  }
+
+  public Path getWorkDir() {
+    return workDir;
+  }
+
+  public Config setWorkDir(Path workDir) {
+    this.workDir = workDir;
+    return this;
+  }
+
+  public Path getCompleteDir() {
+    return completeDir;
+  }
+
+  public Config setCompleteDir(Path completeDir) {
+    this.completeDir = completeDir;
+    return this;
+  }
+
+  public Path getErrorDir() {
+    return errorDir;
+  }
+
+  public Config setErrorDir(Path errorDir) {
+    this.errorDir = errorDir;
+    return this;
+  }
+
+  public Path getDestDir() {
+    return destDir;
+  }
+
+  public Config setDestDir(Path destDir) {
+    this.destDir = destDir;
+    return this;
+  }
+
+  public Path getDestStagingDir() {
+    return destStagingDir;
+  }
+
+  public Config setDestStagingDir(Path destStagingDir) {
+    this.destStagingDir = destStagingDir;
+    return this;
+  }
+
+  public String getScript() {
+    return script;
+  }
+
+  public Config setScript(String script) {
+    this.script = script;
+    return this;
+  }
+
+  public String getWorkScript() {
+    return workScript;
+  }
+
+  public Config setWorkScript(String workScript) {
+    this.workScript = workScript;
+    return this;
+  }
+
+  public boolean isRemove() {
+    return remove;
+  }
+
+  public Config setRemove(boolean remove) {
+    this.remove = remove;
+    return this;
+  }
+
+  public boolean isVerify() {
+    return verify;
+  }
+
+  public Config setVerify(boolean verify) {
+    this.verify = verify;
+    return this;
+  }
+
+  public int getNumThreads() {
+    return numThreads;
+  }
+
+  public Config setNumThreads(int numThreads) {
+    this.numThreads = numThreads;
+    return this;
+  }
+
+  public long getPollSleepPeriodMillis() {
+    return pollSleepPeriodMillis;
+  }
+
+  public Config setPollSleepPeriodMillis(long pollSleepPeriodMillis) {
+    this.pollSleepPeriodMillis = pollSleepPeriodMillis;
+    return this;
+  }
+
+  public FileSystem getSrcFs() {
+    return srcFs;
+  }
+
+  public Config setSrcFs(FileSystem srcFs) {
+    this.srcFs = srcFs;
+    return this;
+  }
+
+  public FileSystem getDestFs() {
+    return destFs;
+  }
+
+  public Config setDestFs(FileSystem destFs) {
+    this.destFs = destFs;
+    return this;
+  }
+
+  public Configuration getConfig() {
+    return config;
+  }
+
+  public Config setConfig(Configuration config) {
+    this.config = config;
+    return this;
+  }
+}