Allow sync of large files (~100MB) (#4836).

common/changes/feature_4836_allow-sync-of-large-files

@@ -0,0 +1 @@
+  o Allow sync of large files (~100MB) (#4836).

common/src/leap/soledad/common/couch.py

@@ -27,7 +27,7 @@
 
 
 from couchdb.client import Server
-from couchdb.http import ResourceNotFound, Unauthorized
+from couchdb.http import ResourceNotFound, Unauthorized, Session
 from u1db import errors, query_parser, vectorclock
 from u1db.backends import CommonBackend, CommonSyncTarget
 from u1db.remote import http_app
@@ -41,6 +41,9 @@
 logger = logging.getLogger(__name__)
 
 
+COUCH_TIMEOUT = 120  # timeout for transfers between Soledad server and Couch
+
+
 class InvalidURLError(Exception):
     """
     Exception raised when Soledad encounters a malformed URL.
@@ -206,6 +209,8 @@ def __init__(self, url, dbname, replica_uid=None, full_commit=True,
         # save params
         self._url = url
         self._full_commit = full_commit
+        if session is None:
+            session = Session(timeout=COUCH_TIMEOUT)
         self._session = session
         self._factory = CouchDocument
         self._real_replica_uid = None

common/src/leap/soledad/common/tests/couchdb.ini.template

@@ -6,7 +6,7 @@
 database_dir = %(tempdir)s/lib
 view_index_dir = %(tempdir)s/lib
 max_document_size = 4294967296 ; 4 GB
-os_process_timeout = 5000 ; 5 seconds. for view and external servers.
+os_process_timeout = 120000 ; 120 seconds. for view and external servers.
 max_dbs_open = 100
 delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned
 uri_file = %(tempdir)s/lib/couch.uri

common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py

@@ -100,6 +100,7 @@ def setUp(self):
         self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
 
     def tearDown(self):
+        self.db.delete_database()
         CouchDBTestCase.tearDown(self)
         TestCaseWithServer.tearDown(self)
 

common/src/leap/soledad/common/tests/test_server.py

@@ -25,6 +25,7 @@
 import simplejson as json
 import mock
 import time
+import binascii
 
 
 from leap.common.testing.basetest import BaseLeapTest
@@ -436,6 +437,42 @@ def test_encrypted_sym_sync_with_unicode_passphrase(self):
         self.assertEqual(doc1, doc2)
 
 
+    def test_sync_very_large_files(self):
+        """
+        Test if Soledad can sync very large files.
+        """
+        # define the size of the "very large file"
+        length = 100*(10**6)  # 100 MB
+        self.startServer()
+        # instantiate soledad and create a document
+        sol1 = self._soledad_instance(
+            # token is verified in test_target.make_token_soledad_app
+            auth_token='auth-token'
+        )
+        _, doclist = sol1.get_all_docs()
+        self.assertEqual([], doclist)
+        content = binascii.hexlify(os.urandom(length/2))  # len() == length
+        doc1 = sol1.create_doc({'data': content})
+        # sync with server
+        sol1._server_url = self.getURL()
+        sol1.sync()
+        # instantiate soledad with empty db, but with same secrets path
+        sol2 = self._soledad_instance(prefix='x', auth_token='auth-token')
+        _, doclist = sol2.get_all_docs()
+        self.assertEqual([], doclist)
+        sol2._secrets_path = sol1.secrets_path
+        sol2._load_secrets()
+        sol2._set_secret_id(sol1._secret_id)
+        # sync the new instance
+        sol2._server_url = self.getURL()
+        sol2.sync()
+        _, doclist = sol2.get_all_docs()
+        self.assertEqual(1, len(doclist))
+        doc2 = doclist[0]
+        # assert incoming doc is equal to the first sent doc
+        self.assertEqual(doc1, doc2)
+
+
 class LockResourceTestCase(
         CouchDBTestCase, TestCaseWithServer):
     """

server/changes/feature_4836_allow-sync-of-large-files

@@ -0,0 +1 @@
+  o Allow sync of large files (~100MB) (#4836).

server/src/leap/soledad/server/__init__.py

@@ -114,6 +114,10 @@
 # Soledad WSGI application
 #-----------------------------------------------------------------------------
 
+MAX_REQUEST_SIZE = 200  # in Mb
+MAX_ENTRY_SIZE = 200  # in Mb
+
+
 class SoledadApp(http_app.HTTPApp):
     """
     Soledad WSGI application
@@ -124,6 +128,9 @@ class SoledadApp(http_app.HTTPApp):
     The name of the shared database that holds user's encrypted secrets.
     """
 
+    max_request_size = MAX_REQUEST_SIZE * 1024 * 1024
+    max_entry_size = MAX_ENTRY_SIZE * 1024 * 1024
+
     def __call__(self, environ, start_response):
         """
         Handle a WSGI call to the Soledad application.
@@ -143,6 +150,8 @@ def __call__(self, environ, start_response):
 
 
 http_app.url_to_resource.register(LockResource)
+http_app.SyncResource.max_request_size = MAX_REQUEST_SIZE * 1024 * 1024
+http_app.SyncResource.max_entry_size = MAX_ENTRY_SIZE * 1024 * 1024
 
 
 #-----------------------------------------------------------------------------