Permalink
Browse files

Basic WebDAV server, using key-value store and blobs on cloud files

  • Loading branch information...
justinsb committed Dec 20, 2013
1 parent 25aff52 commit f6680430ff8ee5027f8da73b69806cb552ed2154
Showing with 5,976 additions and 47 deletions.
  1. +116 −0 cloudata-files/pom.xml
  2. +27 −0 cloudata-files/src/main/assembly/assembly.xml
  3. +67 −0 cloudata-files/src/main/java/com/cloudata/files/FileServer.java
  4. +32 −0 cloudata-files/src/main/java/com/cloudata/files/FileServerModule.java
  5. +1,952 −0 cloudata-files/src/main/java/com/cloudata/files/FilesModel.java
  6. +91 −0 cloudata-files/src/main/java/com/cloudata/files/blobs/BlobCache.java
  7. +14 −0 cloudata-files/src/main/java/com/cloudata/files/blobs/BlobStore.java
  8. +56 −0 cloudata-files/src/main/java/com/cloudata/files/blobs/CloudObjectStoreBlobStore.java
  9. +86 −0 cloudata-files/src/main/java/com/cloudata/files/blobs/LocalBlobStore.java
  10. +23 −0 cloudata-files/src/main/java/com/cloudata/files/fs/ByteStrings.java
  11. +23 −0 cloudata-files/src/main/java/com/cloudata/files/fs/EmptyChunkedInput.java
  12. +30 −0 cloudata-files/src/main/java/com/cloudata/files/fs/EmptyFsFile.java
  13. +297 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsClient.java
  14. +45 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsContext.java
  15. +5 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsCredentials.java
  16. +19 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsException.java
  17. +15 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsFile.java
  18. +7 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsFileAlreadyExistsException.java
  19. +42 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsPath.java
  20. +31 −0 cloudata-files/src/main/java/com/cloudata/files/fs/FsVolume.java
  21. +62 −0 cloudata-files/src/main/java/com/cloudata/files/fs/Inode.java
  22. +17 −0 cloudata-files/src/main/java/com/cloudata/files/fs/PlaintextPasswordCredentials.java
  23. +69 −0 cloudata-files/src/main/java/com/cloudata/files/fs/SimpleFsFile.java
  24. +81 −0 cloudata-files/src/main/java/com/cloudata/files/fs/SliceByteInput.java
  25. +83 −0 cloudata-files/src/main/java/com/cloudata/files/io/ByteBufferListInputStream.java
  26. +18 −0 cloudata-files/src/main/java/com/cloudata/files/io/EmptyInputStream.java
  27. +8 −0 cloudata-files/src/main/java/com/cloudata/files/locks/CloudLock.java
  28. +5 −0 cloudata-files/src/main/java/com/cloudata/files/locks/CloudLockToken.java
  29. +180 −0 cloudata-files/src/main/java/com/cloudata/files/locks/InMemoryLockService.java
  30. +12 −0 cloudata-files/src/main/java/com/cloudata/files/locks/LockService.java
  31. +19 −0 cloudata-files/src/main/java/com/cloudata/files/web/WebModule.java
  32. +40 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/DateFormatter.java
  33. +122 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/GetHandler.java
  34. +50 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/HttpFetchRange.java
  35. +283 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/MethodHandler.java
  36. +13 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/MimeHelper.java
  37. +32 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/OptionsHandler.java
  38. +126 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/PropfindHandler.java
  39. +60 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/PutHandler.java
  40. +38 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/Urls.java
  41. +23 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/WebdavContext.java
  42. +186 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/WebdavRequest.java
  43. +232 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/WebdavRequestHandler.java
  44. +31 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/WebdavResponseException.java
  45. +23 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/chunks/ByteBufByteSource.java
  46. +23 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/chunks/ByteBufferByteSource.java
  47. +16 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/chunks/ChunkAccumulator.java
  48. +148 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/chunks/SimpleChunkAccumulator.java
  49. +80 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/PropfindRequest.java
  50. +102 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/PropfindResponse.java
  51. +61 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/WebdavProperty.java
  52. +63 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/WebdavPropertyList.java
  53. +24 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/WebdavPropertyValue.java
  54. +108 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/WebdavXmlWriter.java
  55. +9 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/model/XmlSerializable.java
  56. +215 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/netty/WebdavChannelHandler.java
  57. +36 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/netty/WebdavChannelInitializer.java
  58. +49 −0 cloudata-files/src/main/java/com/cloudata/files/webdav/netty/WebdavNettyServer.java
  59. +55 −0 cloudata-files/src/main/java/com/cloudata/util/TempFile.java
  60. +23 −0 cloudata-files/src/main/proto/FilesModel.proto
  61. +17 −0 cloudata-files/src/main/resources/logback.xml
  62. +3 −0 cloudata-files/update-protobuf.sh
  63. +3 −5 cloudata-git/pom.xml
  64. +4 −4 cloudata-git/src/main/java/com/cloudata/git/GitModule.java
  65. +1 −1 cloudata-git/src/main/java/com/cloudata/git/jgit/CloudDfsRepository.java
  66. +1 −1 cloudata-git/src/main/java/com/cloudata/git/jgit/CloudGitRepositoryStore.java
  67. +10 −6 cloudata-git/src/main/java/com/cloudata/git/jgit/CloudRefDatabase.java
  68. +0 −5 cloudata-git/src/main/java/com/cloudata/git/keyvalue/Modifier.java
  69. +2 −1 cloudata-git/src/main/java/com/cloudata/git/model/GitUser.java
  70. +0 −9 cloudata-git/src/main/java/com/cloudata/git/services/AuthenticationManager.java
  71. +1 −0 cloudata-git/src/main/java/com/cloudata/git/services/JCloudsAuthenticationManager.java
  72. +1 −1 cloudata-git/src/main/java/com/cloudata/git/web/WebModule.java
  73. +7 −0 cloudata-server-shared/pom.xml
  74. +5 −0 cloudata-server-shared/src/main/java/com/cloudata/auth/AuthenticatedUser.java
  75. +7 −0 cloudata-server-shared/src/main/java/com/cloudata/auth/AuthenticationManager.java
  76. +5 −7 ...errit/httpd → cloudata-server-shared/src/main/java/com/cloudata/auth}/ProjectBasicAuthFilter.java
  77. 0 ...-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/ByteBufferOutputStream.java
  78. 0 {cloudata-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/JCloudsDirInfo.java
  79. 0 {cloudata-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/JCloudsFileInfo.java
  80. 0 ...a-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/JCloudsObjectInfoBase.java
  81. +21 −0 ...data-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/JCloudsObjectStore.java
  82. 0 {cloudata-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/ObjectInfo.java
  83. +3 −0 {cloudata-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/ObjectStore.java
  84. 0 {cloudata-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/ObjectStorePath.java
  85. 0 ...ta-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/OpenstackObjectStore.java
  86. 0 {cloudata-git → cloudata-server-shared}/src/main/java/com/cloudata/objectstore/SwiftPath.java
  87. +6 −0 cloudata-shared/pom.xml
  88. +1 −1 .../com/cloudata/git → cloudata-shared/src/main/java/com/cloudata/clients}/keyvalue/IfNotExists.java
  89. +1 −1 ...va/com/cloudata/git → cloudata-shared/src/main/java/com/cloudata/clients}/keyvalue/IfVersion.java
  90. +22 −0 cloudata-shared/src/main/java/com/cloudata/clients/keyvalue/KeyValueEntry.java
  91. +1 −1 ...com/cloudata/git → cloudata-shared/src/main/java/com/cloudata/clients}/keyvalue/KeyValuePath.java
  92. +5 −2 ...om/cloudata/git → cloudata-shared/src/main/java/com/cloudata/clients}/keyvalue/KeyValueStore.java
  93. +5 −0 cloudata-shared/src/main/java/com/cloudata/clients/keyvalue/Modifier.java
  94. +34 −2 ...oudata/git → cloudata-shared/src/main/java/com/cloudata/clients}/keyvalue/RedisKeyValueStore.java
  95. +7 −0 pom.xml
View
@@ -0,0 +1,116 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.cloudata</groupId>
<artifactId>cloudata-parent</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>
<artifactId>cloudata-files</artifactId>
<properties>
<jgit.version>3.1.0.201310021548-r</jgit.version>
<jetty.version>9.1.0.v20131115</jetty.version>
<jclouds.version>1.6.0</jclouds.version>
</properties>
<dependencies>
<dependency>
<groupId>com.cloudata</groupId>
<artifactId>cloudata-server-shared</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.provider</groupId>
<artifactId>cloudfiles-us</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.11</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>15.0</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
<version>${jetty.version}</version>
</dependency>
<!-- <dependency> <groupId>com.woorea</groupId> <artifactId>keystone-client</artifactId>
<version>3.2.2-SNAPSHOT</version> </dependency> <dependency> <groupId>com.woorea</groupId>
<artifactId>swift-client</artifactId> <version>3.2.2-SNAPSHOT</version> </dependency>
<dependency> <groupId>com.woorea</groupId> <artifactId>jersey-connector</artifactId>
<version>3.2.2-SNAPSHOT</version> </dependency> -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.http.server</artifactId>
<version>${jgit.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>${jgit.version}</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
<version>3.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2-beta-2</version>
<configuration>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,27 @@
<assembly>
<id>bundle</id>
<formats>
<format>tar.gz</format>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>*.sh</include>
</includes>
<lineEnding>unix</lineEnding>
<fileMode>0744</fileMode>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<scope>runtime</scope>
<useProjectArtifact>true</useProjectArtifact>
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
</dependencySet>
</dependencySets>
</assembly>
@@ -0,0 +1,67 @@
package com.cloudata.files;
import java.io.IOException;
import javax.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudata.files.fs.FsClient;
import com.cloudata.files.fs.FsCredentials;
import com.cloudata.files.fs.FsVolume;
import com.cloudata.files.webdav.netty.WebdavNettyServer;
import com.google.common.base.Preconditions;
import com.google.inject.Guice;
import com.google.inject.Injector;
public class FileServer {
private static final Logger log = LoggerFactory.getLogger(FileServer.class);
int httpPort = 8888;
@Inject
WebdavNettyServer nettyServer;
@Inject
FsClient fsClient;
public synchronized void start() throws Exception {
Preconditions.checkState(!nettyServer.isStarted());
nettyServer.start(httpPort);
}
public static void main(String... args) throws Exception {
Injector injector = Guice.createInjector(new FileServerModule());
final FileServer server = injector.getInstance(FileServer.class);
FsCredentials credentials = null;
server.ensureFilesystem(FsVolume.fromHost("1"), credentials);
server.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
server.stop();
} catch (Exception e) {
log.warn("Error shutting down HTTP server", e);
}
}
});
}
private void ensureFilesystem(FsVolume volume, FsCredentials credentials) throws IOException {
fsClient.ensureFilesystem(volume, credentials);
}
public synchronized void stop() throws Exception {
if (nettyServer != null) {
nettyServer.stop();
nettyServer = null;
}
}
}
@@ -0,0 +1,32 @@
package com.cloudata.files;
import java.io.File;
import java.net.InetSocketAddress;
import com.cloudata.clients.keyvalue.KeyValueStore;
import com.cloudata.clients.keyvalue.RedisKeyValueStore;
import com.cloudata.files.blobs.BlobCache;
import com.cloudata.files.blobs.BlobStore;
import com.cloudata.files.blobs.LocalBlobStore;
import com.cloudata.files.locks.InMemoryLockService;
import com.cloudata.files.locks.LockService;
import com.google.inject.AbstractModule;
public class FileServerModule extends AbstractModule {
@Override
protected void configure() {
bind(LockService.class).to(InMemoryLockService.class);
InetSocketAddress redisSocketAddress = new InetSocketAddress("localhost", 6379);
KeyValueStore keyValueStore = new RedisKeyValueStore(redisSocketAddress);
bind(KeyValueStore.class).toInstance(keyValueStore);
File basePath = new File("/tmp/blobs/store");
File cacheDir = new File("/tmp/blobs/cache");
BlobCache cache = new BlobCache(cacheDir);
LocalBlobStore blobStore = new LocalBlobStore(cache, basePath);
bind(BlobStore.class).toInstance(blobStore);
}
}
Oops, something went wrong.

0 comments on commit f668043

Please sign in to comment.