Skip to content

Commit bbfb919

Browse files
justinsbjustinsb
committed
Support delete, also fix some encoding issues and similar bugs
When we delete, we don't actually delete, we just remove it from the name tree. To enable GC or undelete, we write a record of the deleted inodes in the 'X' keyspace
1 parent 1f34ba7 commit bbfb919

File tree

15 files changed

+740
-24
lines changed

15 files changed

+740
-24
lines changed

‎cloudata-files/src/main/java/com/cloudata/files/FilesModel.java‎

Lines changed: 566 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cloudata-files/src/main/java/com/cloudata/files/fs/FsClient.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import com.cloudata.clients.keyvalue.Modifier;
1818
import com.cloudata.files.FilesModel.ChunkData;
1919
import com.cloudata.files.FilesModel.ChunkData.Builder;
20+
import com.cloudata.files.FilesModel.DeletedData;
2021
import com.cloudata.files.FilesModel.InodeData;
2122
import com.cloudata.files.blobs.BlobCache.CacheFileHandle;
2223
import com.cloudata.files.blobs.BlobStore;
@@ -34,6 +35,7 @@ public class FsClient {
3435

3536
private static final ByteString DIR_CHILDREN = ByteString.copyFrom(new byte[] { 'D' });
3637
private static final ByteString INODES = ByteString.copyFrom(new byte[] { 'I' });
38+
private static final ByteString DELETED_INODES = ByteString.copyFrom(new byte[] { 'X' });
3739

3840
private static final ByteString CHUNK_PREFIX = ByteString.copyFrom(new byte[] { 'C' });
3941

@@ -100,6 +102,13 @@ private ByteString buildDirEntryKey(FsPath parent, ByteString name) {
100102
return key;
101103
}
102104

105+
private ByteString buildDirEntryKey(FsPath path) {
106+
ByteString volumePrefix = path.getVolume().getPrefix();
107+
ByteString key = volumePrefix.concat(DIR_CHILDREN).concat(ByteStrings.encode(path.getParent().getId()))
108+
.concat(path.getNameBytes());
109+
return key;
110+
}
111+
103112
public static class DirEntry {
104113
final FsPath parent;
105114
final ByteString name;
@@ -226,6 +235,8 @@ private void createNewEntry(FsPath parent, ByteString name, InodeData.Builder in
226235

227236
inodeKey = buildInodeKey(parent.getVolume(), id);
228237
if (store.put(inodeKey, created.toByteString(), IfNotExists.INSTANCE)) {
238+
log.debug("Wrote inode entry: {} = {}", id, created);
239+
229240
break;
230241
} else {
231242
inode = InodeData.newBuilder(created);
@@ -301,4 +312,35 @@ public CacheFileHandle findChunk(ChunkData chunkData) throws IOException {
301312
return blobStore.find(key);
302313
}
303314

315+
public boolean delete(FsPath fsPath) throws IOException {
316+
// Store a deletion record
317+
// Allows for more efficient GC, and for undelete
318+
// TODO: Support GC
319+
320+
long id = fsPath.getId();
321+
DeletedData.Builder b = DeletedData.newBuilder();
322+
b.setInode(id);
323+
for (FsPath entry : fsPath.getPathComponentList()) {
324+
b.addPath(entry.getNameBytes());
325+
}
326+
ByteString deletedValue = b.build().toByteString();
327+
328+
ByteString volumePrefix = fsPath.getVolume().getPrefix();
329+
330+
while (true) {
331+
ByteString key = volumePrefix.concat(DELETED_INODES).concat(ByteStrings.encode(id))
332+
.concat(ByteStrings.encode(System.currentTimeMillis()));
333+
334+
if (store.put(key, deletedValue, IfNotExists.INSTANCE)) {
335+
break;
336+
}
337+
}
338+
339+
// Unlink the entry from the name tree
340+
{
341+
ByteString key = buildDirEntryKey(fsPath);
342+
343+
return store.delete(key);
344+
}
345+
}
304346
}

‎cloudata-files/src/main/java/com/cloudata/files/fs/FsPath.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.cloudata.files.fs;
22

3+
import java.util.List;
4+
35
import com.cloudata.files.FilesModel.InodeData;
6+
import com.google.common.collect.Lists;
47
import com.google.protobuf.ByteString;
58

69
public class FsPath extends Inode {
@@ -39,4 +42,26 @@ private void buildHref(StringBuilder sb) {
3942
}
4043
}
4144

45+
public FsPath getParent() {
46+
return this.parent;
47+
}
48+
49+
public List<FsPath> getPathComponentList() {
50+
List<FsPath> components = Lists.newArrayList();
51+
buildPathComponentList(components);
52+
return components;
53+
}
54+
55+
private void buildPathComponentList(List<FsPath> components) {
56+
if (parent != null) {
57+
parent.buildPathComponentList(components);
58+
}
59+
components.add(this);
60+
}
61+
62+
@Override
63+
public String toString() {
64+
return "FsPath [href=" + getHref() + "]";
65+
}
66+
4267
}

‎cloudata-files/src/main/java/com/cloudata/files/fs/FsVolume.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public static FsVolume fromHost(String host) {
2121
bucket = bucket.replace(":8080", "");
2222
bucket = bucket.replace("127.0.0.1", "local");
2323

24-
long key = 1;
24+
long key = 2;
2525
return new FsVolume(key);
2626
}
2727

‎cloudata-files/src/main/java/com/cloudata/files/webdav/DeleteHandler.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package com.cloudata.files.webdav;
2+
3+
import io.netty.handler.codec.http.HttpObject;
4+
import io.netty.handler.codec.http.HttpResponseStatus;
5+
6+
import com.cloudata.files.fs.FsClient;
7+
import com.cloudata.files.fs.FsPath;
8+
9+
public class DeleteHandler extends MethodHandler {
10+
11+
public DeleteHandler(WebdavRequestHandler requestHandler) {
12+
super(requestHandler);
13+
}
14+
15+
@Override
16+
protected HttpObject doAction(FsPath fsPath) throws Exception {
17+
WebdavRequest request = getRequest();
18+
FsClient fsClient = getFsClient();
19+
20+
int depth = request.getDepth();
21+
22+
if (fsPath.isFolder()) {
23+
if (depth != WebdavRequest.DEPTH_INFINITY) {
24+
throw new WebdavResponseException(HttpResponseStatus.CONFLICT);
25+
}
26+
}
27+
28+
if (!fsClient.delete(fsPath)) {
29+
throw new WebdavResponseException(HttpResponseStatus.CONFLICT);
30+
}
31+
32+
return buildFullResponse(HttpResponseStatus.NO_CONTENT);
33+
}
34+
}

‎cloudata-files/src/main/java/com/cloudata/files/webdav/GetHandler.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,17 @@
1313
import java.text.ParseException;
1414
import java.util.Date;
1515

16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
18+
1619
import com.cloudata.files.fs.FsFile;
1720
import com.cloudata.files.fs.FsPath;
1821
import com.google.common.base.Strings;
1922
import com.google.common.io.Closeables;
2023

2124
public class GetHandler extends MethodHandler {
25+
private static final Logger log = LoggerFactory.getLogger(GetHandler.class);
26+
2227
public GetHandler(WebdavRequestHandler requestHandler) {
2328
super(requestHandler);
2429
}
@@ -37,6 +42,7 @@ public void close() {
3742

3843
@Override
3944
public HttpObject doAction(FsPath fsPath) throws Exception {
45+
log.debug("GET {}", fsPath);
4046
WebdavRequest request = getRequest();
4147

4248
// TODO: Handle ETAG??
@@ -57,7 +63,7 @@ public HttpObject doAction(FsPath fsPath) throws Exception {
5763
long ifModifiedSinceDateSeconds = ifModifiedSinceDate.getTime() / 1000;
5864
long fileLastModifiedSeconds = fsPath.getModified() / 1000;
5965
if (ifModifiedSinceDateSeconds == fileLastModifiedSeconds) {
60-
throw new WebdavResponseException(HttpResponseStatus.NOT_MODIFIED);
66+
// throw new WebdavResponseException(HttpResponseStatus.NOT_MODIFIED);
6167
}
6268
}
6369

@@ -107,6 +113,7 @@ public HttpObject doAction(FsPath fsPath) throws Exception {
107113
int httpCacheSeconds = 0;
108114
setDateAndCacheHeaders(fsPath, response, httpCacheSeconds);
109115

116+
log.debug("Sending response to GET: {} {}", fsPath.getHref(), response);
110117
write(response);
111118
if (content != null) {
112119
while (!content.isEndOfInput()) {

‎cloudata-files/src/main/java/com/cloudata/files/webdav/LockHandler.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ protected HttpObject doAction(FsPath fsPath) throws Exception {
3939

4040
log.debug("LockRequest: " + lockRequest);
4141

42-
CloudLockToken lockToken = lockService.tryLock(request.getUri());
42+
String lockSubject = buildLockSubject(request);
43+
44+
CloudLockToken lockToken = lockService.tryLock(lockSubject);
4345
if (lockToken == null) {
4446
throw new WebdavResponseException(HttpResponseStatus.LOCKED);
4547
}

‎cloudata-files/src/main/java/com/cloudata/files/webdav/MethodHandler.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -129,15 +129,15 @@ private boolean checkIfHeader(String ifHeader) {
129129
lockToken = lockToken.substring(LockHandler.LOCK_PREFIX.length());
130130
}
131131

132-
URI subjectUri;
132+
String lockSubject;
133+
133134
if (Strings.isNullOrEmpty(subjectUrl)) {
134-
subjectUri = URI.create(getRequest().getUri());
135+
lockSubject = buildLockSubject(getRequest());
135136
} else {
136-
subjectUri = URI.create(subjectUrl);
137+
lockSubject = buildLockSubject(subjectUrl);
137138
}
138-
String path = subjectUri.getPath();
139139

140-
if (getLockService().findLock(path, lockToken) == null) {
140+
if (getLockService().findLock(lockSubject, lockToken) == null) {
141141
return false;
142142
}
143143
} else {
@@ -274,7 +274,7 @@ public FsPath call() throws Exception {
274274

275275
pathTokens = Lists.newArrayList();
276276
for (String token : Splitter.on('/').split(path)) {
277-
pathTokens.add(token);
277+
pathTokens.add(Urls.decodeComponent(token));
278278
}
279279
}
280280

@@ -291,4 +291,14 @@ protected boolean shouldResolveParent() {
291291
private ListeningExecutorService getExecutor() {
292292
return requestHandler.getExecutor();
293293
}
294+
295+
protected String buildLockSubject(WebdavRequest request) {
296+
String uri = request.getUri();
297+
return buildLockSubject(uri);
298+
}
299+
300+
protected String buildLockSubject(String uriString) {
301+
URI uri = URI.create(uriString);
302+
return uri.getPath();
303+
}
294304
}

‎cloudata-files/src/main/java/com/cloudata/files/webdav/MkdirHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ protected HttpObject doAction(FsPath parentPath) throws Exception {
3434
}
3535

3636
String path = request.getUri();
37-
String name = Urls.getLastPathComponent(path);
37+
String name = Urls.getLastPathComponent(path, true);
3838

3939
InodeData.Builder inode = InodeData.newBuilder();
4040

‎cloudata-files/src/main/java/com/cloudata/files/webdav/PutHandler.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
import io.netty.handler.codec.http.HttpObject;
55
import io.netty.handler.codec.http.HttpResponseStatus;
66

7+
import org.slf4j.Logger;
8+
import org.slf4j.LoggerFactory;
9+
710
import com.cloudata.files.FilesModel.InodeData;
811
import com.cloudata.files.fs.FsPath;
912
import com.cloudata.files.webdav.chunks.ChunkAccumulator;
@@ -13,6 +16,8 @@
1316

1417
public class PutHandler extends MethodHandler {
1518

19+
private static final Logger log = LoggerFactory.getLogger(PutHandler.class);
20+
1621
public PutHandler(WebdavRequestHandler requestHandler) {
1722
super(requestHandler);
1823
}
@@ -29,7 +34,9 @@ public HttpObject doAction(FsPath parentPath) throws Exception {
2934
}
3035

3136
String path = getUri();
32-
String newName = Urls.getLastPathComponent(path);
37+
String newName = Urls.getLastPathComponent(path, true);
38+
39+
log.debug("PUT {} {}", parentPath, newName);
3340

3441
// TODO: Always overwrite??
3542
boolean overwrite = true;

0 commit comments

Comments
 (0)