TACHYON-255: Support for OpenStack Swift object store

conf/core-site.xml.template

@@ -0,0 +1,121 @@
+<configuration>
+<property>
+   <name>fs.swift.impl</name>
+   <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
+</property>
+
+<!--SoftLayer Dallas - identified via "dal05" 
+Use this via swift://<swift container>.dal05/ -->
+
+<property>
+    <name>fs.swift.service.dal05.auth.url</name>
+    <value>https://dal05.objectstorage.softlayer.net/auth/v1.0</value>
+</property>
+<property>
+    <name>fs.swift.service.dal05.http.port</name>
+    <value>8080</value>
+</property>
+<property>
+    <name>fs.swift.service.dal05.public</name>
+    <value>true</value>
+</property>
+<property>
+   <name>fs.swift.service.dal05.location-aware</name>
+   <value>false</value>
+</property>
+<property>
+    <name>fs.swift.service.ibm.dal05.endpoint.prefix</name>
+    <value>endpoints</value>
+</property>
+<property>
+    <name>fs.swift.service.dal05.apikey</name>
+    <value>API_KEY</value>
+</property>
+<property>
+    <name>fs.swift.service.dal05.username</name>
+    <value>ACCOUNT:USER NAME</value>
+</property>
+<property>
+    <name>fs.swift.service.dal05.use.get.auth</name>
+    <value>true</value>
+</property>
+
+<!-- Local Swift cluster using Keystone- identified via "swift1"
+Use this via swift://<swift container>.swift1 -->
+
+<property>
+   <name>fs.swift.service.swift.location-aware</name>
+   <value>false</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.auth.url</name>
+    <value>http://127.0.0.1:5000/v2.0/tokens</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.http.port</name>
+    <value>8080</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.region</name>
+    <value>RegionOne</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.public</name>
+    <value>true</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.auth.endpoint.prefix</name>
+    <value>endpoints</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.tenant</name>
+    <value>TENANT</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.password</name>
+    <value>PASSWORD</value>
+</property>
+<property>
+    <name>fs.swift.service.swift1.username</name>
+    <value>USER NAME</value>
+</property>
+
+<!--Local Swift using Temp Auth - identified via "swift2"
+Use this via swift://<swift container>.swift2 -->
+
+
+<property>
+    <name>fs.swift.service.swift2.auth.url</name>
+    <value>http://127.0.0.1:8080/auth/v1.0</value>
+</property>
+<property>
+    <name>fs.swift.service.swift2.http.port</name>
+    <value>8080</value>
+</property>
+<property>
+    <name>fs.swift.service.swift2.public</name>
+    <value>true</value>
+</property>
+<property>
+   <name>fs.swift.service.swift2.location-aware</name>
+   <value>false</value>
+</property>
+<property>
+    <name>fs.swift.service.ibm.swift2.endpoint.prefix</name>
+    <value>endpoints</value>
+</property>
+<property>
+    <name>fs.swift.service.swift2.apikey</name>
+    <value>testing</value>
+</property>
+<property>
+    <name>fs.swift.service.swift2.username</name>
+    <value>test:tester</value>
+</property>
+<property>
+    <name>fs.swift.service.swift2.use.get.auth</name>
+    <value>true</value>
+</property>
+</configuration>
+
+

conf/tachyon-env.sh.swift

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+# This file contains environment variables required to run Tachyon. Copy it as tachyon-env.sh and
+# edit that to configure Tachyon for your site. At a minimum,
+# the following variables should be set:
+#
+# - JAVA_HOME, to point to your JAVA installation
+# - TACHYON_MASTER_ADDRESS, to bind the master to a different IP address or hostname
+# - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address.
+# - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker
+# - TACHYON_RAM_FOLDER, to set where worker stores in memory data
+# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem,
+#   org.apache.hadoop.hdfs.DistributedFileSystem)
+
+# The following gives an example:
+
+# Uncomment this section to add a local installation of Hadoop to Tachyon's CLASSPATH.
+# The hadoop command must be in the path to automatically populate the Hadoop classpath.
+#
+# if type "hadoop" > /dev/null 2>&1; then
+#  export HADOOP_TACHYON_CLASSPATH=`hadoop classpath`
+# fi
+# export TACHYON_CLASSPATH=$HADOOP_TACHYON_CLASSPATH
+
+if [[ `uname -a` == Darwin* ]]; then
+  # Assuming Mac OS X
+  export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home)}
+  export TACHYON_RAM_FOLDER=/Volumes/ramdisk
+  export TACHYON_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc="
+else
+  # Assuming Linux
+  if [ -z "$JAVA_HOME" ]; then
+    if [ -d /usr/lib/jvm/java-7-oracle ]; then
+      export JAVA_HOME=/usr/lib/jvm/java-7-oracle
+    else
+      # openjdk will set this
+      if [ -d /usr/lib/jvm/jre-1.7.0 ]; then
+        export JAVA_HOME=/usr/lib/jvm/jre-1.7.0
+      fi
+    fi
+  fi
+  export TACHYON_RAM_FOLDER=/mnt/ramdisk
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+  export JAVA_HOME="$(dirname $(which java))/.."
+fi
+
+export JAVA="$JAVA_HOME/bin/java"
+export TACHYON_MASTER_ADDRESS=localhost
+export TACHYON_UNDERFS_ADDRESS=swift://testcont.tactest
+export TACHYON_WORKER_MEMORY_SIZE=1GB
+export TACHYON_UNDERFS_HDFS_IMPL=org.apache.hadoop.hdfs.DistributedFileSystem
+export TACHYON_WORKER_MAX_WORKER_THREADS=2048
+export TACHYON_MASTER_MAX_WORKER_THREADS=2048
+
+export TACHYON_SSH_FOREGROUND="yes"
+export TACHYON_WORKER_SLEEP="0.02"
+
+# Prepend Tachyon classes before classes specified by TACHYON_CLASSPATH
+# in the Java classpath.  May be necessary if there are jar conflicts
+#export TACHYON_PREPEND_TACHYON_CLASSES="yes"
+
+CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export TACHYON_JAVA_OPTS+="
+  -Dlog4j.configuration=file:$CONF_DIR/log4j.properties
+  -Dtachyon.debug=false
+  -Dtachyon.worker.hierarchystore.level.max=1
+  -Dtachyon.worker.hierarchystore.level0.alias=MEM
+  -Dtachyon.worker.hierarchystore.level0.dirs.path=$TACHYON_RAM_FOLDER
+  -Dtachyon.worker.hierarchystore.level0.dirs.quota=$TACHYON_WORKER_MEMORY_SIZE
+  -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS
+  -Dtachyon.underfs.hdfs.impl=$TACHYON_UNDERFS_HDFS_IMPL
+  -Dtachyon.data.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/data
+  -Dtachyon.worker.max.worker.threads=$TACHYON_WORKER_MAX_WORKER_THREADS
+  -Dtachyon.workers.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/workers
+  -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE
+  -Dtachyon.worker.data.folder=/tachyonworker/
+  -Dtachyon.master.max.worker.threads=$TACHYON_MASTER_MAX_WORKER_THREADS
+  -Dtachyon.master.worker.timeout.ms=60000
+  -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS
+  -Dtachyon.master.journal.folder=$TACHYON_UNDERFS_ADDRESS/journal/
+  -Dorg.apache.jasper.compiler.disablejsr199=true
+  -Djava.net.preferIPv4Stack=true
+"
+
+# Master specific parameters. Default to TACHYON_JAVA_OPTS.
+export TACHYON_MASTER_JAVA_OPTS="$TACHYON_JAVA_OPTS"
+
+# Worker specific parameters that will be shared to all workers. Default to TACHYON_JAVA_OPTS.
+export TACHYON_WORKER_JAVA_OPTS="$TACHYON_JAVA_OPTS"

core/pom.xml

@@ -166,8 +166,13 @@
     <dependency>
       <groupId>commons-httpclient</groupId>
       <artifactId>commons-httpclient</artifactId>
-      <version>3.0.1</version>
-      <scope>test</scope>
+      <version>3.1</version>
+      <!--scope>test</scope-->
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-openstack</artifactId>
+      <version>${hadoop.version}</version>
     </dependency>
   </dependencies>
 

core/src/main/java/tachyon/conf/TachyonConf.java

@@ -228,6 +228,10 @@ public String get(String key, final String defaultValue) {
     return updated;
   }
 
+  public boolean containsKey(String key) {
+    return mProperties.containsKey(key);
+  }
+
   public int getInt(String key, final int defaultValue) {
     if (mProperties.containsKey(key)) {
       String rawValue = mProperties.getProperty(key);

core/src/main/java/tachyon/hadoop/Utils.java

@@ -27,6 +27,7 @@
 
 import tachyon.Constants;
 import tachyon.TachyonURI;
+import tachyon.conf.TachyonConf;
 
 public final class Utils {
   private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
@@ -45,6 +46,19 @@ public static void addS3Credentials(Configuration conf) {
       conf.set(secretKeyConf, System.getProperty(secretKeyConf));
     }
   }
+
+  /**
+   * Replace default key with user provided key 
+   * @param conf
+   * @param key
+   */
+  public static void addKey(Configuration conf, TachyonConf tachyonConf, String key) {
+    if (System.getProperty(key) != null && conf.get(key) == null) {
+      conf.set(key, System.getProperty(key));
+    } else if (tachyonConf.containsKey(key)) {
+      conf.set(key, tachyonConf.get(key, null));
+    }
+  }
 
   public static Path getHDFSPath(TachyonURI path, String ufsAddress) {
     if (path.isPathAbsolute()) {

core/src/main/resources/tachyon-default.properties

@@ -23,7 +23,8 @@ tachyon.debug=false
 tachyon.web.resources=${tachyon.home}/core/src/main/webapp
 tachyon.web.threads=1
 tachyon.logs.dir=${tachyon.home}/logs
-tachyon.underfs.hadoop.prefixes=hdfs://,s3://,s3n://,glusterfs:///
+tachyon.underfs.hadoop.prefixes=hdfs://,s3://,s3n://,glusterfs:///,swift://
+tachyon.underfs.hadoop.configuration=${tachyon.home}/conf/core-site.xml
 tachyon.underfs.address=${tachyon.home}/underFSStorage
 tachyon.underfs.hdfs.impl=org.apache.hadoop.hdfs.DistributedFileSystem
 tachyon.underfs.glusterfs.impl=org.apache.hadoop.fs.glusterfs.GlusterFileSystem
@@ -34,16 +35,17 @@ tachyon.zookeeper.election.path=/election
 tachyon.zookeeper.leader.path=/leader
 tachyon.async.enabled=false
 tachyon.max.columns=1000
-tachyon.test.mode=false
+tachyon.test.mode=falseZZ
 tachyon.max.table.metadata.byte=5MB
 tachyon.host.resolution.timeout.ms=5000
 
 # Master properties
-tachyon.master.journal.folder=${tachyon.home}/journal/
+#tachyon.master.journal.folder=${tachyon.home}/journal/
+tachyon.master.journal.folder=${tachyon.underfs.address}/journal/
 tachyon.master.format.file_prefix=_format_
 tachyon.master.port=19998
 tachyon.master.web.port=19999
-tachyon.master.temporary.folder=/tmp
+tachyon.master.temporary.folder=${tachyon.underfs.address}/tmp
 tachyon.master.heartbeat.interval.ms=1000
 tachyon.master.max.worker.threads=2048
 tachyon.master.worker.timeout.ms=10000

pom.xml

@@ -124,7 +124,7 @@
     <test.output.redirect>true</test.output.redirect>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <java.version>1.6</java.version>
-    <hadoop.version>1.0.4</hadoop.version>
+    <hadoop.version>2.6.0</hadoop.version>
     <glusterfs-hadoop.version>2.3.13</glusterfs-hadoop.version>
     <libthrift.version>0.9.1</libthrift.version>
     <cxf.version>2.7.0</cxf.version>

underfs/hdfs/src/main/java/tachyon/underfs/hdfs/HdfsUnderFileSystem.java

@@ -66,9 +66,10 @@ public HdfsUnderFileSystem(String fsDefaultName, TachyonConf tachyonConf, Object
     } else {
       tConf = new Configuration();
     }
-
     prepareConfiguration(fsDefaultName, tachyonConf, tConf);
-
+    Utils.addKey(tConf, tachyonConf, "tachyon.underfs.hadoop.configuration");
+    LOG.debug("Adding" + tConf.get("tachyon.underfs.hadoop.configuration"));
+    tConf.addResource(new Path(tConf.get("tachyon.underfs.hadoop.configuration")));
     Utils.addS3Credentials(tConf);
 
     Path path = new Path(mUfsPrefix);