Syncmanager: fetch journal entries in chunks.

Before this commit we used to fetch the whole journal entry list in one
go, which caused issues in two cases:
1. On slow internet connections the download may fail.
2. With big journals: Android interrupts sync managers if they don't
    perform any significant network traffic for over a minute[1],
    and because we would first download and only then process, we would
    sometimes hit this threshold.

Current chunk size is set to 50.

1: https://developer.android.com/reference/android/content/AbstractThreadedSyncAdapter.html

app/src/main/java/com/etesync/syncadapter/journalmanager/JournalEntryManager.java

@@ -36,14 +36,18 @@ public JournalEntryManager(OkHttpClient httpClient, HttpUrl remote, String journ
         this.client = httpClient;
     }
 
-    public List<Entry> list(Crypto.CryptoManager crypto, String last) throws Exceptions.HttpException, Exceptions.IntegrityException {
+    public List<Entry> list(Crypto.CryptoManager crypto, String last, int limit) throws Exceptions.HttpException, Exceptions.IntegrityException {
         Entry previousEntry = null;
         HttpUrl.Builder urlBuilder = this.remote.newBuilder();
         if (last != null) {
             urlBuilder.addQueryParameter("last", last);
             previousEntry = Entry.getFakeWithUid(last);
         }
 
+        if (limit > 0) {
+            urlBuilder.addQueryParameter("limit", String.valueOf(limit));
+        }
+
         HttpUrl remote = urlBuilder.build();
 
         Request request = new Request.Builder()

app/src/main/java/com/etesync/syncadapter/syncadapter/CalendarSyncManager.java

@@ -41,8 +41,6 @@
  * <p>Synchronization manager for CardDAV collections; handles contacts and groups.</p>
  */
 public class CalendarSyncManager extends SyncManager {
-    protected static final int MAX_MULTIGET = 10;
-
     final private HttpUrl remote;
 
     public CalendarSyncManager(Context context, Account account, AccountSettings settings, Bundle extras, String authority, SyncResult result, LocalCalendar calendar, HttpUrl remote) throws Exceptions.IntegrityException, Exceptions.GenericCryptoException {

app/src/main/java/com/etesync/syncadapter/syncadapter/ContactsSyncManager.java

@@ -55,8 +55,6 @@
  * <p>Synchronization manager for CardDAV collections; handles contacts and groups.</p>
  */
 public class ContactsSyncManager extends SyncManager {
-    protected static final int MAX_MULTIGET = 10;
-
     final private ContentProviderClient provider;
     final private HttpUrl remote;
 

app/src/main/java/com/etesync/syncadapter/syncadapter/SyncManager.java

@@ -53,6 +53,9 @@
 import static com.etesync.syncadapter.Constants.KEY_ACCOUNT;
 
 abstract public class SyncManager {
+    private static final int MAX_FETCH = 50;
+    private static final int MAX_PUSH = 30;
+
     protected final NotificationHelper notificationManager;
     protected final CollectionInfo info;
 
@@ -151,17 +154,19 @@ public void performSync() {
             App.log.info("Sync phase: " + context.getString(syncPhase));
             prepareLocal();
 
-            if (Thread.interrupted())
-                throw new InterruptedException();
-            syncPhase = R.string.sync_phase_fetch_entries;
-            App.log.info("Sync phase: " + context.getString(syncPhase));
-            fetchEntries();
+            do {
+                if (Thread.interrupted())
+                    throw new InterruptedException();
+                syncPhase = R.string.sync_phase_fetch_entries;
+                App.log.info("Sync phase: " + context.getString(syncPhase));
+                fetchEntries();
 
-            if (Thread.interrupted())
-                throw new InterruptedException();
-            syncPhase = R.string.sync_phase_apply_remote_entries;
-            App.log.info("Sync phase: " + context.getString(syncPhase));
-            applyRemoteEntries();
+                if (Thread.interrupted())
+                    throw new InterruptedException();
+                syncPhase = R.string.sync_phase_apply_remote_entries;
+                App.log.info("Sync phase: " + context.getString(syncPhase));
+                applyRemoteEntries();
+            } while (remoteEntries.size() > 0);
 
             /* Create journal entries out of local changes. */
             if (Thread.interrupted())
@@ -282,7 +287,7 @@ protected void fetchEntries() throws Exceptions.HttpException, ContactsStorageEx
         int count = data.count(EntryEntity.class).where(EntryEntity.JOURNAL.eq(getJournalEntity())).get().value();
         if ((remoteCTag != null) && (count == 0)) {
             // If we are updating an existing installation with no saved journal, we need to add
-            remoteEntries = journal.list(crypto, null);
+            remoteEntries = journal.list(crypto, null, MAX_FETCH);
             int i = 0;
             for (JournalEntryManager.Entry entry : remoteEntries) {
                 SyncEntry cEntry = SyncEntry.fromJournalEntry(crypto, entry);
@@ -294,7 +299,7 @@ protected void fetchEntries() throws Exceptions.HttpException, ContactsStorageEx
                 }
             }
         } else {
-            remoteEntries = journal.list(crypto, remoteCTag);
+            remoteEntries = journal.list(crypto, remoteCTag, MAX_FETCH);
         }
 
         App.log.info("Fetched " + String.valueOf(remoteEntries.size()) + " entries");
@@ -324,7 +329,6 @@ protected void applyRemoteEntries() throws IOException, ContactsStorageException
 
     protected void pushEntries() throws Exceptions.HttpException, IOException, ContactsStorageException, CalendarStorageException {
         // upload dirty contacts
-        final int MAX_PUSH = 30;
         int pushed = 0;
         // FIXME: Deal with failure (someone else uploaded before we go here)
         try {

app/src/test/java/com/etesync/syncadapter/journalmanager/ServiceTest.java

@@ -180,9 +180,9 @@ public void testSyncEntry() throws IOException, Exceptions.HttpException, Except
         entries.add(entry2);
 
         // Check last works:
-        entries = journalEntryManager.list(crypto, entry.getUid());
+        entries = journalEntryManager.list(crypto, entry.getUid(), 0);
         assertEquals(entries.size(), 1);
-        entries = journalEntryManager.list(crypto, entry2.getUid());
+        entries = journalEntryManager.list(crypto, entry2.getUid(), 0);
         assertEquals(entries.size(), 0);
 
         // Corrupt the journal and verify we catch it
@@ -195,7 +195,7 @@ public void testSyncEntry() throws IOException, Exceptions.HttpException, Except
 
         try {
             caught = null;
-            journalEntryManager.list(crypto, null);
+            journalEntryManager.list(crypto, null, 0);
         } catch (Exceptions.IntegrityException e) {
             caught = e;
         }