Skip to content
Browse files

now uploads are supported as streams like downloads

upload attachment now uses the streaming feature
able to upload and download a 15MB movie wihout memory errors now
  • Loading branch information...
1 parent d859287 commit 731ac55748e13f4d05a9d7a7905d24ab29f590ca @mschoch mschoch committed Jul 7, 2012
View
28 TouchDB-Android-Ektorp/src/com/couchbase/touchdb/ektorp/TouchDBHttpClient.java
@@ -1,14 +1,12 @@
package com.couchbase.touchdb.ektorp;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
-import org.apache.commons.io.IOUtils;
-import org.codehaus.jackson.map.ObjectMapper;
import org.ektorp.http.HttpClient;
import org.ektorp.http.HttpResponse;
import org.ektorp.util.Exceptions;
@@ -97,15 +95,12 @@ public HttpResponse post(String uri, String content) {
if(content != null) {
conn.setDoInput(true);
- OutputStream os = conn.getOutputStream();
- os.write(content.getBytes());
+ conn.setRequestInputStream(new ByteArrayInputStream(content.getBytes()));
}
return executeRequest(conn);
} catch (ProtocolException e) {
throw Exceptions.propagate(e);
- } catch (IOException e) {
- throw Exceptions.propagate(e);
}
}
@@ -117,16 +112,12 @@ public HttpResponse post(String uri, InputStream contentStream) {
if(contentStream != null) {
conn.setDoInput(true);
- ObjectMapper mapper = new ObjectMapper();
- OutputStream os = conn.getOutputStream();
- IOUtils.copyLarge(contentStream, os);
+ conn.setRequestInputStream(contentStream);
}
return executeRequest(conn);
} catch (ProtocolException e) {
throw Exceptions.propagate(e);
- } catch (IOException e) {
- throw Exceptions.propagate(e);
}
}
@@ -155,15 +146,12 @@ public HttpResponse put(String uri, String content) {
if(content != null) {
conn.setDoInput(true);
- OutputStream os = conn.getOutputStream();
- os.write(content.getBytes());
+ conn.setRequestInputStream(new ByteArrayInputStream(content.getBytes()));
}
return executeRequest(conn);
} catch (ProtocolException e) {
throw Exceptions.propagate(e);
- } catch (IOException e) {
- throw Exceptions.propagate(e);
}
}
@@ -175,17 +163,13 @@ public HttpResponse put(String uri, InputStream contentStream, String contentTyp
if(contentStream != null) {
conn.setDoInput(true);
- conn.setRequestProperty("Content-Type", contentType);
- ObjectMapper mapper = new ObjectMapper();
- OutputStream os = conn.getOutputStream();
- IOUtils.copyLarge(contentStream, os);
+ conn.setRequestProperty("content-type", contentType);
+ conn.setRequestInputStream(contentStream);
}
return executeRequest(conn);
} catch (ProtocolException e) {
throw Exceptions.propagate(e);
- } catch (IOException e) {
- throw Exceptions.propagate(e);
}
}
View
11 TouchDB-Android-Listener/src/com/couchbase/touchdb/listener/TDHTTPServlet.java
@@ -2,7 +2,6 @@
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.net.URL;
import java.util.Enumeration;
import java.util.List;
@@ -70,15 +69,7 @@ public void service(HttpServletRequest request, final HttpServletResponse respon
//but its blocking on get requests otherwise
if(is != null && is.available() > 0) {
conn.setDoInput(true);
- OutputStream os = conn.getOutputStream();
- byte[] buffer = new byte[1024];
- int lenRead = is.read(buffer, 0, 1024);
- while(lenRead > 0) {
- os.write(buffer, 0, lenRead);
- lenRead = is.read(buffer, 0, 1024);
- }
- is.close();
- os.close();
+ conn.setRequestInputStream(is);
}
final ServletOutputStream os = response.getOutputStream();
View
11 TouchDB-Android-TestApp/src/com/couchbase/touchdb/testapp/tests/Attachments.java
@@ -17,6 +17,7 @@
package com.couchbase.touchdb.testapp.tests;
+import java.io.ByteArrayInputStream;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
@@ -62,7 +63,7 @@ public void testAttachments() throws Exception {
Assert.assertEquals(TDStatus.CREATED, status.getCode());
byte[] attach1 = "This is the body of attach1".getBytes();
- status = db.insertAttachmentForSequenceWithNameAndType(attach1, rev1.getSequence(), "attach", "text/plain", rev1.getGeneration());
+ status = db.insertAttachmentForSequenceWithNameAndType(new ByteArrayInputStream(attach1), rev1.getSequence(), "attach", "text/plain", rev1.getGeneration());
Assert.assertEquals(TDStatus.CREATED, status.getCode());
TDAttachment attachment = db.getAttachmentForSequence(rev1.getSequence(), "attach", status);
@@ -118,7 +119,7 @@ public void testAttachments() throws Exception {
Assert.assertEquals(TDStatus.CREATED, status.getCode());
byte[] attach2 = "<html>And this is attach2</html>".getBytes();
- status = db.insertAttachmentForSequenceWithNameAndType(attach2, rev3.getSequence(), "attach", "text/html", rev2.getGeneration());
+ status = db.insertAttachmentForSequenceWithNameAndType(new ByteArrayInputStream(attach2), rev3.getSequence(), "attach", "text/html", rev2.getGeneration());
Assert.assertEquals(TDStatus.CREATED, status.getCode());
// Check the 2nd revision's attachment:
@@ -201,11 +202,11 @@ public void testPutAttachment() {
// Update the attachment directly:
byte[] attachv2 = "Replaced body of attach".getBytes();
- db.updateAttachment("attach", attachv2, "application/foo", rev1.getDocId(), null, status);
+ db.updateAttachment("attach", new ByteArrayInputStream(attachv2), "application/foo", rev1.getDocId(), null, status);
Assert.assertEquals(TDStatus.CONFLICT, status.getCode());
- db.updateAttachment("attach", attachv2, "application/foo", rev1.getDocId(), "1-bogus", status);
+ db.updateAttachment("attach", new ByteArrayInputStream(attachv2), "application/foo", rev1.getDocId(), "1-bogus", status);
Assert.assertEquals(TDStatus.CONFLICT, status.getCode());
- TDRevision rev2 = db.updateAttachment("attach", attachv2, "application/foo", rev1.getDocId(), rev1.getRevId(), status);
+ TDRevision rev2 = db.updateAttachment("attach", new ByteArrayInputStream(attachv2), "application/foo", rev1.getDocId(), rev1.getRevId(), status);
Assert.assertEquals(TDStatus.CREATED, status.getCode());
Assert.assertEquals(rev1.getDocId(), rev2.getDocId());
Assert.assertEquals(2, rev2.getGeneration());
View
6 TouchDB-Android-TestApp/src/com/couchbase/touchdb/testapp/tests/Router.java
@@ -1,8 +1,8 @@
package com.couchbase.touchdb.testapp.tests;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
@@ -71,8 +71,8 @@ protected TDURLConnection sendRequest(TDServer server, String method, String pat
Map<String, List<String>> allProperties = conn.getRequestProperties();
if(bodyObj != null) {
conn.setDoInput(true);
- OutputStream os = conn.getOutputStream();
- os.write(mapper.writeValueAsBytes(bodyObj));
+ ByteArrayInputStream bais = new ByteArrayInputStream(mapper.writeValueAsBytes(bodyObj));
+ conn.setRequestInputStream(bais);
}
TDRouter router = new TDRouter(server, conn);
View
71 TouchDB-Android/src/com/couchbase/touchdb/TDBlobStore.java
@@ -38,6 +38,8 @@
public class TDBlobStore {
public static String FILE_EXTENSION = ".blob";
+ public static String TMP_FILE_EXTENSION = ".blobtmp";
+ public static String TMP_FILE_PREFIX = "tmp";
private String path;
@@ -70,10 +72,44 @@ public static TDBlobKey keyForBlob(byte[] data) {
return result;
}
+ public static TDBlobKey keyForBlobFromFile(File file) {
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance("SHA-1");
+ } catch (NoSuchAlgorithmException e) {
+ Log.e(TDDatabase.TAG, "Error, SHA-1 digest is unavailable.");
+ return null;
+ }
+ byte[] sha1hash = new byte[40];
+
+ try {
+ FileInputStream fis = new FileInputStream(file);
+ byte[] buffer = new byte[65536];
+ int lenRead = fis.read(buffer);
+ while(lenRead > 0) {
+ md.update(buffer, 0, lenRead);
+ lenRead = fis.read(buffer);
+ }
+ fis.close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error readin tmp file to compute key");
+ }
+
+ sha1hash = md.digest();
+ TDBlobKey result = new TDBlobKey(sha1hash);
+ return result;
+ }
+
public String pathForKey(TDBlobKey key) {
return path + File.separator + TDBlobKey.convertToHex(key.getBytes()) + FILE_EXTENSION;
}
+ public long getSizeOfBlob(TDBlobKey key) {
+ String path = pathForKey(key);
+ File file = new File(path);
+ return file.length();
+ }
+
public boolean getKeyForFilename(TDBlobKey outKey, String filename) {
if(!filename.endsWith(FILE_EXTENSION)) {
return false;
@@ -112,6 +148,41 @@ public InputStream blobStreamForKey(TDBlobKey key) {
return null;
}
+ public boolean storeBlobStream(InputStream inputStream, TDBlobKey outKey) {
+
+ File tmp = null;
+ try {
+ tmp = File.createTempFile(TMP_FILE_PREFIX, TMP_FILE_EXTENSION, new File(path));
+ FileOutputStream fos = new FileOutputStream(tmp);
+ byte[] buffer = new byte[65536];
+ int lenRead = inputStream.read(buffer);
+ while(lenRead > 0) {
+ fos.write(buffer, 0, lenRead);
+ lenRead = inputStream.read(buffer);
+ }
+ inputStream.close();
+ fos.close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error writing blog to tmp file", e);
+ return false;
+ }
+
+ TDBlobKey newKey = keyForBlobFromFile(tmp);
+ outKey.setBytes(newKey.getBytes());
+ String path = pathForKey(outKey);
+ File file = new File(path);
+
+ if(file.canRead()) {
+ // object with this hash already exists, we should delete tmp file and return true
+ tmp.delete();
+ return true;
+ } else {
+ // does not exist, we should rename tmp file to this name
+ tmp.renameTo(file);
+ }
+ return true;
+ }
+
public boolean storeBlob(byte[] data, TDBlobKey outKey) {
TDBlobKey newKey = keyForBlob(data);
outKey.setBytes(newKey.getBytes());
View
22 TouchDB-Android/src/com/couchbase/touchdb/TDDatabase.java
@@ -17,6 +17,7 @@
package com.couchbase.touchdb;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -1334,13 +1335,12 @@ public TDStatus deleteViewNamed(String name) {
/*** TDDatabase+Attachments ***/
/*************************************************************************************************/
- public TDStatus insertAttachmentForSequenceWithNameAndType(byte[] contents, long sequence, String name, String contentType, int revpos) {
- assert(contents != null);
+ public TDStatus insertAttachmentForSequenceWithNameAndType(InputStream contentStream, long sequence, String name, String contentType, int revpos) {
assert(sequence > 0);
assert(name != null);
TDBlobKey key = new TDBlobKey();
- if(!attachments.storeBlob(contents, key)) {
+ if(!attachments.storeBlobStream(contentStream, key)) {
return new TDStatus(TDStatus.INTERNAL_SERVER_ERROR);
}
@@ -1351,7 +1351,7 @@ public TDStatus insertAttachmentForSequenceWithNameAndType(byte[] contents, long
args.put("filename", name);
args.put("key", keyData);
args.put("type", contentType);
- args.put("length", contents.length);
+ args.put("length", attachments.getSizeOfBlob(key));
args.put("revpos", revpos);
database.insert("attachments", null, args);
return new TDStatus(TDStatus.CREATED);
@@ -1601,7 +1601,7 @@ public TDStatus processAttachmentsForRevision(TDRevision rev, long parentSequenc
}
// Finally insert the attachment:
- status = insertAttachmentForSequenceWithNameAndType(newContents, newSequence, name, (String)newAttach.get("content_type"), revpos);
+ status = insertAttachmentForSequenceWithNameAndType(new ByteArrayInputStream(newContents), newSequence, name, (String)newAttach.get("content_type"), revpos);
}
else {
// It's just a stub, so copy the previous revision's attachment entry:
@@ -1620,9 +1620,9 @@ public TDStatus processAttachmentsForRevision(TDRevision rev, long parentSequenc
* Updates or deletes an attachment, creating a new document revision in the process.
* Used by the PUT / DELETE methods called on attachment URLs.
*/
- public TDRevision updateAttachment(String filename, byte[] body, String contentType, String docID, String oldRevID, TDStatus status) {
+ public TDRevision updateAttachment(String filename, InputStream contentStream, String contentType, String docID, String oldRevID, TDStatus status) {
status.setCode(TDStatus.BAD_REQUEST);
- if(filename == null || filename.length() == 0 || (body != null && contentType == null) || (oldRevID != null && docID == null) || (body != null && docID == null)) {
+ if(filename == null || filename.length() == 0 || (contentStream != null && contentType == null) || (oldRevID != null && docID == null) || (contentStream != null && docID == null)) {
return null;
}
@@ -1641,7 +1641,7 @@ public TDRevision updateAttachment(String filename, byte[] body, String contentT
}
Map<String,Object> attachments = (Map<String, Object>) oldRev.getProperties().get("_attachments");
- if(body == null && attachments != null && !attachments.containsKey(filename)) {
+ if(contentStream == null && attachments != null && !attachments.containsKey(filename)) {
status.setCode(TDStatus.NOT_FOUND);
return null;
}
@@ -1672,9 +1672,9 @@ public TDRevision updateAttachment(String filename, byte[] body, String contentT
+ "WHERE sequence=? AND filename != ?", args);
}
- if(body != null) {
+ if(contentStream != null) {
// If not deleting, add a new attachment entry:
- TDStatus insertStatus = insertAttachmentForSequenceWithNameAndType(body, newRev.getSequence(),
+ TDStatus insertStatus = insertAttachmentForSequenceWithNameAndType(contentStream, newRev.getSequence(),
filename, contentType, newRev.getGeneration());
status.setCode(insertStatus.getCode());
@@ -1683,7 +1683,7 @@ public TDRevision updateAttachment(String filename, byte[] body, String contentT
}
}
- status.setCode((body != null) ? TDStatus.CREATED : TDStatus.OK);
+ status.setCode((contentStream != null) ? TDStatus.CREATED : TDStatus.OK);
return newRev;
} catch(SQLException e) {
View
53 TouchDB-Android/src/com/couchbase/touchdb/router/TDRouter.java
@@ -1,8 +1,8 @@
package com.couchbase.touchdb.router;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
@@ -140,23 +140,14 @@ public boolean cacheWithEtag(String etag) {
public Map<String,Object> getBodyAsDictionary() {
try {
- byte[] bodyBytes = ((ByteArrayOutputStream)connection.getOutputStream()).toByteArray();
- Map<String,Object> bodyMap = TDServer.getObjectMapper().readValue(bodyBytes, Map.class);
+ InputStream contentStream = connection.getRequestInputStream();
+ Map<String,Object> bodyMap = TDServer.getObjectMapper().readValue(contentStream, Map.class);
return bodyMap;
} catch (IOException e) {
return null;
}
}
- public byte[] getBody() {
- try {
- byte[] bodyBytes = ((ByteArrayOutputStream)connection.getOutputStream()).toByteArray();
- return bodyBytes;
- } catch (IOException e) {
- return null;
- }
- }
-
public EnumSet<TDContentOptions> getContentOptions() {
EnumSet<TDContentOptions> result = EnumSet.noneOf(TDContentOptions.class);
if(getBooleanQuery("attachments")) {
@@ -265,6 +256,12 @@ public void start() {
List<String> path = splitPath(connection.getURL());
if(path == null) {
connection.setResponseCode(TDStatus.BAD_REQUEST);
+ try {
+ connection.getResponseOutputStream().close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error closing empty output stream");
+ }
+ sendResponse();
return;
}
@@ -278,6 +275,12 @@ public void start() {
db = server.getDatabaseNamed(dbName);
if(db == null) {
connection.setResponseCode(TDStatus.BAD_REQUEST);
+ try {
+ connection.getResponseOutputStream().close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error closing empty output stream");
+ }
+ sendResponse();
return;
}
}
@@ -292,20 +295,38 @@ public void start() {
TDStatus status = openDB();
if(!status.isSuccessful()) {
connection.setResponseCode(status.getCode());
+ try {
+ connection.getResponseOutputStream().close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error closing empty output stream");
+ }
+ sendResponse();
return;
}
String name = path.get(1);
if(!name.startsWith("_")) {
// Regular document
if(!TDDatabase.isValidDocumentId(name)) {
connection.setResponseCode(TDStatus.BAD_REQUEST);
+ try {
+ connection.getResponseOutputStream().close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error closing empty output stream");
+ }
+ sendResponse();
return;
}
docID = name;
} else if("_design".equals(name) || "_local".equals(name)) {
// "_design/____" and "_local/____" are document names
if(pathLen <= 2) {
connection.setResponseCode(TDStatus.NOT_FOUND);
+ try {
+ connection.getResponseOutputStream().close();
+ } catch (IOException e) {
+ Log.e(TDDatabase.TAG, "Error closing empty output stream");
+ }
+ sendResponse();
return;
}
docID = name + "/" + path.get(2);
@@ -1241,13 +1262,13 @@ public TDStatus do_DELETE_Document(TDDatabase _db, String docID, String _attachm
return update(_db, docID, null, true);
}
- public TDStatus updateAttachment(String attachment, String docID, byte[] body) {
+ public TDStatus updateAttachment(String attachment, String docID, InputStream contentStream) {
TDStatus status = new TDStatus();
String revID = getQuery("rev");
if(revID == null) {
revID = getRevIDFromIfMatchHeader();
}
- TDRevision rev = db.updateAttachment(attachment, body, connection.getRequestProperty("content-type"),
+ TDRevision rev = db.updateAttachment(attachment, contentStream, connection.getRequestProperty("content-type"),
docID, revID, status);
if(status.isSuccessful()) {
Map<String, Object> resultDict = new HashMap<String, Object>();
@@ -1256,15 +1277,15 @@ public TDStatus updateAttachment(String attachment, String docID, byte[] body) {
resultDict.put("rev", rev.getRevId());
connection.setResponseBody(new TDBody(resultDict));
cacheWithEtag(rev.getRevId());
- if(body != null) {
+ if(contentStream != null) {
setResponseLocation(connection.getURL());
}
}
return status;
}
public TDStatus do_PUT_Attachment(TDDatabase _db, String docID, String _attachmentName) {
- return updateAttachment(_attachmentName, docID, getBody());
+ return updateAttachment(_attachmentName, docID, connection.getRequestInputStream());
}
public TDStatus do_DELETE_Attachment(TDDatabase _db, String docID, String _attachmentName) {
View
10 TouchDB-Android/src/com/couchbase/touchdb/router/TDURLConnection.java
@@ -42,6 +42,8 @@
private OutputStream responseOutputStream;
private InputStream responseInputStream;
+ private InputStream requestInputStream;
+
public TDURLConnection(URL url) {
super(url);
responseInputStream = new PipedInputStream();
@@ -245,6 +247,14 @@ public InputStream getInputStream() throws IOException {
return responseInputStream;
}
+ public InputStream getRequestInputStream() {
+ return requestInputStream;
+ }
+
+ public void setRequestInputStream(InputStream requestInputStream) {
+ this.requestInputStream = requestInputStream;
+ }
+
}
/**

0 comments on commit 731ac55

Please sign in to comment.
Something went wrong with that request. Please try again.