From b44ea9f6efada0b3ddfea8a9941e26af31ee935c Mon Sep 17 00:00:00 2001 From: Hideki Itakura Date: Mon, 11 May 2015 11:41:07 -0700 Subject: [PATCH 1/4] Fixed storing gzipped attachment. --- .../java/com/couchbase/lite/Attachment.java | 4 +- .../java/com/couchbase/lite/Database.java | 84 +++++++++++-------- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/couchbase/lite/Attachment.java b/src/main/java/com/couchbase/lite/Attachment.java index 4c37218d..797fad04 100644 --- a/src/main/java/com/couchbase/lite/Attachment.java +++ b/src/main/java/com/couchbase/lite/Attachment.java @@ -21,8 +21,8 @@ import com.couchbase.lite.internal.InterfaceAudience; import com.couchbase.lite.util.Log; -import java.io.IOException; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.util.Collections; @@ -132,7 +132,7 @@ public InputStream getContent() throws CouchbaseLiteException { throw new CouchbaseLiteException(Status.INTERNAL_SERVER_ERROR); } Attachment attachment = db.getAttachmentForSequence(sequence, this.name); - body = attachment.getContent(); + body = attachment.getBodyIfNew(); if (attachment.getGZipped()) { // Client does not expect a gzipped stream. // Only Router handles gzipped streams and uses getAttachmentForSequence directly. diff --git a/src/main/java/com/couchbase/lite/Database.java b/src/main/java/com/couchbase/lite/Database.java index 6fb4ec1c..d0b1d256 100644 --- a/src/main/java/com/couchbase/lite/Database.java +++ b/src/main/java/com/couchbase/lite/Database.java @@ -1269,6 +1269,8 @@ public synchronized boolean open() { "key BLOB NOT NULL, " + "type TEXT, " + "length INTEGER NOT NULL, " + + "encoding INTEGER DEFAULT 0, " + + "encoded_length INTEGER, " + "revpos INTEGER DEFAULT 0); " + "CREATE INDEX attachments_by_sequence on attachments(sequence, filename); " + "CREATE INDEX attachments_sequence ON attachments(sequence); " + @@ -1297,9 +1299,16 @@ public synchronized boolean open() { int revPos = (Integer) attachment.get("revpos"); int length = (Integer) attachment.get("length"); String digest = (String) attachment.get("digest"); + int encodedLength = -1; + if(attachment.containsKey("encoded_length")) + encodedLength = (Integer)attachment.get("encoded_length"); + AttachmentInternal.AttachmentEncoding encoding = AttachmentInternal.AttachmentEncoding.AttachmentEncodingNone; + if(attachment.containsKey("encoding") && attachment.get("encoding").equals("gzip")){ + encoding = AttachmentInternal.AttachmentEncoding.AttachmentEncodingGZIP; + } BlobKey key = new BlobKey(digest); try { - insertAttachmentForSequenceWithNameAndType(sequence, name, contentType, revPos, key, length); + insertAttachmentForSequenceWithNameAndType(sequence, name, contentType, revPos, key, length, encoding, encodedLength); } catch (CouchbaseLiteException e) { Log.e(Log.TAG_DATABASE, "Attachment information inserstion error: " + name + "=" + attachment.toString(), e); return false; @@ -2854,10 +2863,15 @@ void insertAttachmentForSequence(AttachmentInternal attachment, long sequence) t attachment.getName(), attachment.getContentType(), attachment.getRevpos(), - attachment.getBlobKey()); + attachment.getBlobKey(), + attachment.getLength(), + attachment.getEncoding(), + attachment.getEncodedLength()); } /** + * Note: This method is used only from unit tests. + * * @exclude */ @InterfaceAudience.Private @@ -2882,37 +2896,28 @@ public void insertAttachmentForSequenceWithNameAndType(InputStream contentStream */ @InterfaceAudience.Private public void insertAttachmentForSequenceWithNameAndType(long sequence, String name, String contentType, int revpos, BlobKey key) throws CouchbaseLiteException { - try { - ContentValues args = new ContentValues(); - args.put("sequence", sequence); - args.put("filename", name); - if (key != null) { - args.put("key", key.getBytes()); - args.put("length", attachments.getSizeOfBlob(key)); - } - args.put("type", contentType); - args.put("revpos", revpos); - long result = database.insert("attachments", null, args); - if (result == -1) { - String msg = "Insert attachment failed (returned -1)"; - Log.e(Database.TAG, msg); - throw new CouchbaseLiteException(msg, Status.INTERNAL_SERVER_ERROR); - } - } catch (SQLException e) { - Log.e(Database.TAG, "Error inserting attachment", e); - throw new CouchbaseLiteException(e, Status.INTERNAL_SERVER_ERROR); - } + insertAttachmentForSequenceWithNameAndType(sequence, name, contentType, revpos, key, key != null ? attachments.getSizeOfBlob(key): -1, AttachmentInternal.AttachmentEncoding.AttachmentEncodingNone, -1); } - private void insertAttachmentForSequenceWithNameAndType(long sequence, String name, String contentType, int revpos, BlobKey key, long length) throws CouchbaseLiteException { + private void insertAttachmentForSequenceWithNameAndType(long sequence, String name, String contentType, int revpos, BlobKey key, long length, AttachmentInternal.AttachmentEncoding encoding, long encodedLength) throws CouchbaseLiteException { try { ContentValues args = new ContentValues(); args.put("sequence", sequence); args.put("filename", name); - args.put("key", key.getBytes()); - args.put("length", length); args.put("type", contentType); args.put("revpos", revpos); + if (key != null) { + args.put("key", key.getBytes()); + } + if (length >= 0) { + args.put("length", length); + } + if(encoding == AttachmentInternal.AttachmentEncoding.AttachmentEncodingGZIP) { + args.put("encoding", encoding.ordinal()); + if (encodedLength >= 0) { + args.put("encoded_length", encodedLength); + } + } long result = database.insert("attachments", null, args); if (result == -1) { String msg = "Insert attachment failed (returned -1)"; @@ -2924,6 +2929,7 @@ private void insertAttachmentForSequenceWithNameAndType(long sequence, String na throw new CouchbaseLiteException(e, Status.INTERNAL_SERVER_ERROR); } } + /** * @exclude */ @@ -3116,7 +3122,7 @@ public Map getAttachmentsDictForSequenceWithContent(long sequence String args[] = { Long.toString(sequence) }; try { - cursor = database.rawQuery("SELECT filename, key, type, length, revpos FROM attachments WHERE sequence=?", args); + cursor = database.rawQuery("SELECT filename, key, type, encoding, length, encoded_length, revpos FROM attachments WHERE sequence=?", args); if(!cursor.moveToNext()) { return null; @@ -3127,7 +3133,10 @@ public Map getAttachmentsDictForSequenceWithContent(long sequence while(!cursor.isAfterLast()) { boolean dataSuppressed = false; - int length = cursor.getInt(3); + int length = cursor.getInt(4); + + AttachmentInternal.AttachmentEncoding encoding = AttachmentInternal.AttachmentEncoding.values()[cursor.getInt(3)]; + int encodedLength = cursor.getInt(5); byte[] keyData = cursor.getBlob(1); BlobKey key = new BlobKey(keyData); @@ -3140,39 +3149,40 @@ public Map getAttachmentsDictForSequenceWithContent(long sequence } else { byte[] data = attachments.blobForKey(key); - if(data != null) { dataBase64 = Base64.encodeBytes(data); // <-- very expensive } else { Log.w(Database.TAG, "Error loading attachment. Sequence: %s", sequence); } - } + } + String encodingStr = null; + if(encoding!=AttachmentInternal.AttachmentEncoding.AttachmentEncodingNone){ + // NOTE: iOS decode if attachment is included int the dict. + encodingStr = "gzip"; } Map attachment = new HashMap(); - - - if(!(dataBase64 != null || dataSuppressed)) { attachment.put("stub", true); } - if(dataBase64 != null) { attachment.put("data", dataBase64); } - if (dataSuppressed == true) { attachment.put("follows", true); } - attachment.put("digest", digestString); String contentType = cursor.getString(2); attachment.put("content_type", contentType); + if (encodingStr != null) + attachment.put("encoding", encodingStr); attachment.put("length", length); - attachment.put("revpos", cursor.getInt(4)); + if (encodingStr != null && encodedLength >= 0) + attachment.put("encoded_length", encodedLength); + attachment.put("revpos", cursor.getInt(6)); String filename = cursor.getString(0); result.put(filename, attachment); @@ -4262,7 +4272,7 @@ Map getAttachmentsFromRevision(RevisionInternal rev, // If there's inline attachment data, decode and store it: byte[] newContents; try { - newContents = Base64.decode(newContentBase64); + newContents = Base64.decode(newContentBase64, Base64.DONT_GUNZIP); } catch (IOException e) { throw new CouchbaseLiteException(e, Status.BAD_ENCODING); } From 7602907511b4701191cb299337ac1f07097213f8 Mon Sep 17 00:00:00 2001 From: Hideki Itakura Date: Tue, 12 May 2015 14:11:28 -0700 Subject: [PATCH 2/4] Added additional comment for `Attachment.getGZipped()` method --- src/main/java/com/couchbase/lite/Attachment.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/couchbase/lite/Attachment.java b/src/main/java/com/couchbase/lite/Attachment.java index 797fad04..0604727a 100644 --- a/src/main/java/com/couchbase/lite/Attachment.java +++ b/src/main/java/com/couchbase/lite/Attachment.java @@ -270,6 +270,9 @@ else if (value != null) { } /** + * NOTE: getGZipped() can return correct value if Attachment is returned from Database.getAttachmentForSequence(long, String). + * This method should be internal use only. + * * @exclude */ @InterfaceAudience.Private From adfc1d558bb8b8e02f1a570a68445415034ddd74 Mon Sep 17 00:00:00 2001 From: Hideki Itakura Date: Tue, 12 May 2015 17:18:07 -0700 Subject: [PATCH 3/4] Added "Content-Encoding" header if an attachment is gzipped. --- .../lite/replicator/PusherInternal.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/couchbase/lite/replicator/PusherInternal.java b/src/main/java/com/couchbase/lite/replicator/PusherInternal.java index 115f92b0..387af92e 100644 --- a/src/main/java/com/couchbase/lite/replicator/PusherInternal.java +++ b/src/main/java/com/couchbase/lite/replicator/PusherInternal.java @@ -612,7 +612,15 @@ else if (attachment.containsKey("content-type")) { " issue #80): %s", attachment); } - FileBody fileBody = new FileBody(file, contentType); + // NOTE: Content-Encoding might not be necessary to set. Apache FileBody does not set Content-Encoding. + // FileBody always return null for getContentEncoding(), and Content-Encoding header is not set in multipart + // CBL iOS: https://github.com/couchbase/couchbase-lite-ios/blob/feb7ff5eda1e80bd00e5eb19f1d46c793f7a1951/Source/CBL_Pusher.m#L449-L452 + String contentEncoding = null; + if(attachment.containsKey("encoding")){ + contentEncoding = (String)attachment.get("encoding"); + } + + FileBody fileBody = new CustomFileBody(file, contentType, contentEncoding); multiPart.addPart(attachmentKey, fileBody); } @@ -714,4 +722,19 @@ private static int findCommonAncestor(RevisionInternal rev, List possibl return generation; } + + // CustomFileBody to support contentEncoding. FileBody returns always null for getContentEncoding() + private static class CustomFileBody extends FileBody { + private String contentEncoding = null; + + public CustomFileBody(final File file, final String mimeType, final String contentEncoding) { + super(file, mimeType); + this.contentEncoding = contentEncoding; + } + + @Override + public String getContentEncoding() { + return contentEncoding; + } + } } From 86d0a03684309d886e98c9582f1b67a6e0e36283 Mon Sep 17 00:00:00 2001 From: Hideki Itakura Date: Wed, 13 May 2015 12:01:07 -0700 Subject: [PATCH 4/4] Applied feedbacks from @pasin --- src/main/java/com/couchbase/lite/Database.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/couchbase/lite/Database.java b/src/main/java/com/couchbase/lite/Database.java index d0b1d256..60ff7171 100644 --- a/src/main/java/com/couchbase/lite/Database.java +++ b/src/main/java/com/couchbase/lite/Database.java @@ -3159,16 +3159,16 @@ public Map getAttachmentsDictForSequenceWithContent(long sequence } String encodingStr = null; - if(encoding!=AttachmentInternal.AttachmentEncoding.AttachmentEncodingNone){ + if (encoding == AttachmentInternal.AttachmentEncoding.AttachmentEncodingGZIP) { // NOTE: iOS decode if attachment is included int the dict. encodingStr = "gzip"; } Map attachment = new HashMap(); - if(!(dataBase64 != null || dataSuppressed)) { + if (!(dataBase64 != null || dataSuppressed)) { attachment.put("stub", true); } - if(dataBase64 != null) { + if (dataBase64 != null) { attachment.put("data", dataBase64); } if (dataSuppressed == true) { @@ -3177,11 +3177,13 @@ public Map getAttachmentsDictForSequenceWithContent(long sequence attachment.put("digest", digestString); String contentType = cursor.getString(2); attachment.put("content_type", contentType); - if (encodingStr != null) + if (encodingStr != null) { attachment.put("encoding", encodingStr); + } attachment.put("length", length); - if (encodingStr != null && encodedLength >= 0) + if (encodingStr != null && encodedLength >= 0) { attachment.put("encoded_length", encodedLength); + } attachment.put("revpos", cursor.getInt(6)); String filename = cursor.getString(0);