forked from DmitryMezhensky/Hadoop-and-Swift-integration
-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
dmezhenskiy
committed
Nov 29, 2012
1 parent
c09b34b
commit 21e2423
Showing
37 changed files
with
3,787 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
.classpath | ||
.project | ||
target | ||
.settings | ||
.DS_Store | ||
.faces-config.xml.jsfdia | ||
MANIFEST.MF | ||
.idea | ||
*.iml | ||
*.ipr | ||
*.iws | ||
|
||
/rest-api/index | ||
/rest-api/rest/index |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> | ||
<modelVersion>4.0.0</modelVersion> | ||
<groupId>com.mirantis</groupId> | ||
<artifactId>swift-file-system-locality-test</artifactId> | ||
<version>1.0-SNAPSHOT</version> | ||
<name>SwiftFileSystem Locality Test</name> | ||
<url>http://maven.apache.org</url> | ||
|
||
<repositories> | ||
<repository> | ||
<id>Cloudera</id> | ||
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url> | ||
</repository> | ||
</repositories> | ||
|
||
<properties> | ||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> | ||
</properties> | ||
|
||
<build> | ||
<plugins> | ||
<plugin> | ||
<groupId>org.apache.maven.plugins</groupId> | ||
<artifactId>maven-compiler-plugin</artifactId> | ||
<version>2.5.1</version> | ||
<configuration> | ||
<source>1.6</source> | ||
<target>1.6</target> | ||
</configuration> | ||
</plugin> | ||
</plugins> | ||
</build> | ||
|
||
<dependencies> | ||
<dependency> | ||
<groupId>org.apache.hadoop</groupId> | ||
<artifactId>hadoop-core</artifactId> | ||
<version>0.20.2-cdh3u1</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>commons-lang</groupId> | ||
<artifactId>commons-lang</artifactId> | ||
<version>2.4</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.mirantis</groupId> | ||
<artifactId>swift-file-system</artifactId> | ||
<version>1.0-SNAPSHOT</version> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>junit</groupId> | ||
<artifactId>junit</artifactId> | ||
<version>4.10</version> | ||
<scope>test</scope> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.google.guava</groupId> | ||
<artifactId>guava</artifactId> | ||
<version>13.0.1</version> | ||
<scope>test</scope> | ||
</dependency> | ||
</dependencies> | ||
|
||
</project> |
22 changes: 22 additions & 0 deletions
22
swift-file-system-locality-test/src/main/java/com/mirantis/swift/fs/App.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
package com.mirantis.swift.fs; | ||
|
||
import java.io.FileNotFoundException; | ||
import java.io.PrintWriter; | ||
|
||
/** | ||
* @author dmezhensky | ||
*/ | ||
public class App { | ||
public static void main(String[] args) throws FileNotFoundException { | ||
String filename = "object"; | ||
|
||
for (int i = 0; i < 7; i++) { | ||
final PrintWriter printWriter = new PrintWriter(filename + String.valueOf(i + 4)); | ||
for (int j = 1; j < 5000000; ++j) { | ||
printWriter.write("file".concat(String.valueOf(i + 4)) + " line".concat(String.valueOf(j))); | ||
printWriter.write("\n"); | ||
} | ||
printWriter.close(); | ||
} | ||
} | ||
} |
76 changes: 76 additions & 0 deletions
76
swift-file-system-locality-test/src/main/java/com/mirantis/swift/fs/TestJob.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
package com.mirantis.swift.fs; | ||
|
||
import org.apache.commons.logging.Log; | ||
import org.apache.commons.logging.LogFactory; | ||
import org.apache.hadoop.conf.Configuration; | ||
import org.apache.hadoop.conf.Configured; | ||
import org.apache.hadoop.fs.Path; | ||
import org.apache.hadoop.io.LongWritable; | ||
import org.apache.hadoop.io.NullWritable; | ||
import org.apache.hadoop.io.Text; | ||
import org.apache.hadoop.mapreduce.Job; | ||
import org.apache.hadoop.mapreduce.Mapper; | ||
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; | ||
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; | ||
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; | ||
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; | ||
import org.apache.hadoop.util.Tool; | ||
import org.apache.hadoop.util.ToolRunner; | ||
|
||
import java.io.IOException; | ||
import java.net.InetAddress; | ||
import java.net.UnknownHostException; | ||
|
||
/** | ||
* @author vsorokin | ||
*/ | ||
public class TestJob extends Configured implements Tool { | ||
public static class TestMapper extends Mapper<LongWritable, Text, NullWritable, Text> { | ||
private static final InetAddress LOCAL; | ||
static { | ||
try { | ||
LOCAL = InetAddress.getLocalHost(); | ||
} catch (UnknownHostException e) { | ||
throw new IllegalStateException(e); | ||
} | ||
} | ||
|
||
private static final Log LOGGER = LogFactory.getLog(TestMapper.class); | ||
|
||
@Override | ||
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { | ||
LOGGER.info(String.format("[%s/%s] Processing value: %s", LOCAL.getHostName(), LOCAL.getHostAddress(), value)); | ||
context.write(NullWritable.get(), value); | ||
} | ||
} | ||
|
||
@Override | ||
public int run(String[] args) throws Exception { | ||
final Configuration conf = getConf(); | ||
conf.set("io.sort.mb", "32"); | ||
|
||
conf.set("fs.swift.impl", "org.apache.hadoop.swift.fs.snative.SwiftFileSystem"); // TODO remove | ||
|
||
final Job job = new Job(conf, "Test Job"); | ||
job.setJarByClass(TestJob.class); | ||
|
||
FileInputFormat.setInputPaths(job, args[0]); | ||
FileOutputFormat.setOutputPath(job, new Path(args[1])); | ||
|
||
job.setInputFormatClass(TextInputFormat.class); | ||
job.setOutputFormatClass(TextOutputFormat.class); | ||
|
||
job.setMapOutputKeyClass(NullWritable.class); | ||
job.setMapOutputValueClass(Text.class); | ||
|
||
job.setMapperClass(TestMapper.class); | ||
|
||
|
||
|
||
return job.waitForCompletion(true) ? 0 : 1; | ||
} | ||
|
||
public static void main(String[] args) throws Exception { | ||
ToolRunner.run(new Configuration(), new TestJob(), args); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> | ||
<modelVersion>4.0.0</modelVersion> | ||
<groupId>com.mirantis</groupId> | ||
<artifactId>swift-file-system</artifactId> | ||
<version>1.0-SNAPSHOT</version> | ||
<name>SwiftFileSystem</name> | ||
<url>http://maven.apache.org</url> | ||
|
||
<repositories> | ||
<repository> | ||
<id>Cloudera</id> | ||
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url> | ||
</repository> | ||
<repository> | ||
<id>woorea-releases</id> | ||
<url>https://github.com/woorea/maven/raw/master/releases</url> | ||
</repository> | ||
</repositories> | ||
|
||
<properties> | ||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> | ||
<commons.lang.version>2.6</commons.lang.version> | ||
<commons.io.version>2.3</commons.io.version> | ||
<httpcomponents.version>4.2.1</httpcomponents.version> | ||
<jackson.version>1.9.7</jackson.version> | ||
<junit.version>4.10</junit.version> | ||
<mockito.version>1.9.0</mockito.version> | ||
<!-- Reporting --> | ||
<maven.cobertura.version>2.5.2</maven.cobertura.version> | ||
<maven.javadoc.version>2.8</maven.javadoc.version> | ||
<maven.project.version>2.4</maven.project.version> | ||
<maven.site.plugin.version>3.1</maven.site.plugin.version> | ||
</properties> | ||
|
||
<build> | ||
<plugins> | ||
<plugin> | ||
<groupId>org.apache.maven.plugins</groupId> | ||
<artifactId>maven-compiler-plugin</artifactId> | ||
<version>2.5.1</version> | ||
<configuration> | ||
<source>1.6</source> | ||
<target>1.6</target> | ||
</configuration> | ||
</plugin> | ||
</plugins> | ||
</build> | ||
|
||
<dependencies> | ||
<dependency> | ||
<groupId>org.apache.hadoop</groupId> | ||
<artifactId>hadoop-core</artifactId> | ||
<version>0.20.2-cdh3u1</version> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.codehaus.jackson</groupId> | ||
<artifactId>jackson-mapper-asl</artifactId> | ||
<version>${jackson.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.codehaus.jackson</groupId> | ||
<artifactId>jackson-core-asl</artifactId> | ||
<version>${jackson.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.apache.httpcomponents</groupId> | ||
<artifactId>httpclient</artifactId> | ||
<version>${httpcomponents.version}</version> | ||
</dependency> | ||
|
||
<!-- Used for loading test resources and converting a File to byte[] --> | ||
<dependency> | ||
<groupId>commons-io</groupId> | ||
<artifactId>commons-io</artifactId> | ||
<version>${commons.io.version}</version> | ||
</dependency> | ||
|
||
<!-- Used for unit testing --> | ||
<dependency> | ||
<groupId>junit</groupId> | ||
<artifactId>junit-dep</artifactId> | ||
<version>${junit.version}</version> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<!-- Used for mocking dependencies --> | ||
<dependency> | ||
<groupId>org.mockito</groupId> | ||
<artifactId>mockito-core</artifactId> | ||
<version>${mockito.version}</version> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>junit</groupId> | ||
<artifactId>junit</artifactId> | ||
<version>4.10</version> | ||
<scope>test</scope> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.google.guava</groupId> | ||
<artifactId>guava</artifactId> | ||
<version>13.0.1</version> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>nl.tweeenveertig.openstack</groupId> | ||
<artifactId>joss</artifactId> | ||
<version>0.5.0</version> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
</dependencies> | ||
|
||
</project> |
Binary file not shown.
71 changes: 71 additions & 0 deletions
71
swift-file-system/src/main/java/org/apache/hadoop/swift/fs/SwiftObjectPath.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
package org.apache.hadoop.swift.fs; | ||
|
||
import org.apache.hadoop.fs.Path; | ||
|
||
import java.util.regex.Pattern; | ||
|
||
/** | ||
* Swift hierarchy mapping | ||
*/ | ||
public class SwiftObjectPath { | ||
private static final Pattern PATH_PART_PATTERN = Pattern.compile(".*/AUTH_\\w*/"); | ||
|
||
|
||
//TODO: !!!!!!!!!!!!!!! currently container name is hardcoded. Not clear how to make it modifiable | ||
/** | ||
* Swift container | ||
*/ | ||
private final String container = "/data"; | ||
|
||
/** | ||
* swift object | ||
*/ | ||
private final String object; | ||
|
||
public SwiftObjectPath(String object) { | ||
this.object = object; | ||
} | ||
|
||
public String getContainer() { | ||
return container; | ||
} | ||
|
||
public String getObject() { | ||
return object; | ||
} | ||
|
||
@Override | ||
public boolean equals(Object o) { | ||
if (this == o) return true; | ||
if (!(o instanceof SwiftObjectPath)) return false; | ||
final SwiftObjectPath that = (SwiftObjectPath) o; | ||
return this.toUriPath().equals(that.toUriPath()); | ||
} | ||
|
||
@Override | ||
public int hashCode() { | ||
int result = container.hashCode(); | ||
result = 31 * result + object.hashCode(); | ||
return result; | ||
} | ||
|
||
public String toUriPath() { | ||
if (container.endsWith("/")) | ||
return container + object; | ||
else if (object.startsWith("/")) | ||
return container + object; | ||
else | ||
return container + "/" + object; | ||
} | ||
|
||
@Override | ||
public String toString() { | ||
return toUriPath(); | ||
} | ||
|
||
public static SwiftObjectPath fromPath(Path path) { | ||
final String url = path.toUri().getPath().replaceAll(PATH_PART_PATTERN.pattern(), ""); | ||
|
||
return new SwiftObjectPath(url); | ||
} | ||
} |
Oops, something went wrong.