Permalink
Browse files

Syncmanager: fetch journal entries in chunks.

Before this commit we used to fetch the whole journal entry list in one
go, which caused issues in two cases:
1. On slow internet connections the download may fail.
2. With big journals: Android interrupts sync managers if they don't
    perform any significant network traffic for over a minute[1],
    and because we would first download and only then process, we would
    sometimes hit this threshold.

Current chunk size is set to 50.

1: https://developer.android.com/reference/android/content/AbstractThreadedSyncAdapter.html
  • Loading branch information...
tasn committed May 12, 2017
1 parent 4e2667c commit f7104bbcefc160c073f6ee6b8d4f37696731a4c0
@@ -36,14 +36,18 @@ public JournalEntryManager(OkHttpClient httpClient, HttpUrl remote, String journ
this.client = httpClient;
}
public List<Entry> list(Crypto.CryptoManager crypto, String last) throws Exceptions.HttpException, Exceptions.IntegrityException {
public List<Entry> list(Crypto.CryptoManager crypto, String last, int limit) throws Exceptions.HttpException, Exceptions.IntegrityException {
Entry previousEntry = null;
HttpUrl.Builder urlBuilder = this.remote.newBuilder();
if (last != null) {
urlBuilder.addQueryParameter("last", last);
previousEntry = Entry.getFakeWithUid(last);
}
if (limit > 0) {
urlBuilder.addQueryParameter("limit", String.valueOf(limit));
}
HttpUrl remote = urlBuilder.build();
Request request = new Request.Builder()
@@ -41,8 +41,6 @@
* <p>Synchronization manager for CardDAV collections; handles contacts and groups.</p>
*/
public class CalendarSyncManager extends SyncManager {
protected static final int MAX_MULTIGET = 10;
final private HttpUrl remote;
public CalendarSyncManager(Context context, Account account, AccountSettings settings, Bundle extras, String authority, SyncResult result, LocalCalendar calendar, HttpUrl remote) throws Exceptions.IntegrityException, Exceptions.GenericCryptoException {
@@ -55,8 +55,6 @@
* <p>Synchronization manager for CardDAV collections; handles contacts and groups.</p>
*/
public class ContactsSyncManager extends SyncManager {
protected static final int MAX_MULTIGET = 10;
final private ContentProviderClient provider;
final private HttpUrl remote;
@@ -53,6 +53,9 @@
import static com.etesync.syncadapter.Constants.KEY_ACCOUNT;
abstract public class SyncManager {
private static final int MAX_FETCH = 50;
private static final int MAX_PUSH = 30;
protected final NotificationHelper notificationManager;
protected final CollectionInfo info;
@@ -151,17 +154,19 @@ public void performSync() {
App.log.info("Sync phase: " + context.getString(syncPhase));
prepareLocal();
if (Thread.interrupted())
throw new InterruptedException();
syncPhase = R.string.sync_phase_fetch_entries;
App.log.info("Sync phase: " + context.getString(syncPhase));
fetchEntries();
do {
if (Thread.interrupted())
throw new InterruptedException();
syncPhase = R.string.sync_phase_fetch_entries;
App.log.info("Sync phase: " + context.getString(syncPhase));
fetchEntries();
if (Thread.interrupted())
throw new InterruptedException();
syncPhase = R.string.sync_phase_apply_remote_entries;
App.log.info("Sync phase: " + context.getString(syncPhase));
applyRemoteEntries();
if (Thread.interrupted())
throw new InterruptedException();
syncPhase = R.string.sync_phase_apply_remote_entries;
App.log.info("Sync phase: " + context.getString(syncPhase));
applyRemoteEntries();
} while (remoteEntries.size() > 0);
/* Create journal entries out of local changes. */
if (Thread.interrupted())
@@ -282,7 +287,7 @@ protected void fetchEntries() throws Exceptions.HttpException, ContactsStorageEx
int count = data.count(EntryEntity.class).where(EntryEntity.JOURNAL.eq(getJournalEntity())).get().value();
if ((remoteCTag != null) && (count == 0)) {
// If we are updating an existing installation with no saved journal, we need to add
remoteEntries = journal.list(crypto, null);
remoteEntries = journal.list(crypto, null, MAX_FETCH);
int i = 0;
for (JournalEntryManager.Entry entry : remoteEntries) {
SyncEntry cEntry = SyncEntry.fromJournalEntry(crypto, entry);
@@ -294,7 +299,7 @@ protected void fetchEntries() throws Exceptions.HttpException, ContactsStorageEx
}
}
} else {
remoteEntries = journal.list(crypto, remoteCTag);
remoteEntries = journal.list(crypto, remoteCTag, MAX_FETCH);
}
App.log.info("Fetched " + String.valueOf(remoteEntries.size()) + " entries");
@@ -324,7 +329,6 @@ protected void applyRemoteEntries() throws IOException, ContactsStorageException
protected void pushEntries() throws Exceptions.HttpException, IOException, ContactsStorageException, CalendarStorageException {
// upload dirty contacts
final int MAX_PUSH = 30;
int pushed = 0;
// FIXME: Deal with failure (someone else uploaded before we go here)
try {
@@ -180,9 +180,9 @@ public void testSyncEntry() throws IOException, Exceptions.HttpException, Except
entries.add(entry2);
// Check last works:
entries = journalEntryManager.list(crypto, entry.getUid());
entries = journalEntryManager.list(crypto, entry.getUid(), 0);
assertEquals(entries.size(), 1);
entries = journalEntryManager.list(crypto, entry2.getUid());
entries = journalEntryManager.list(crypto, entry2.getUid(), 0);
assertEquals(entries.size(), 0);
// Corrupt the journal and verify we catch it
@@ -195,7 +195,7 @@ public void testSyncEntry() throws IOException, Exceptions.HttpException, Except
try {
caught = null;
journalEntryManager.list(crypto, null);
journalEntryManager.list(crypto, null, 0);
} catch (Exceptions.IntegrityException e) {
caught = e;
}

0 comments on commit f7104bb

Please sign in to comment.