Merge pull request #398 from matrix-org/feature/migrate_keys_store_to_db

Migrate keys store from file serialization to realm db

.travis.yml

@@ -23,7 +23,7 @@ android:
     - platform-tools
 
     # The BuildTools version used by your project
-    - build-tools-27.0.3
+    - build-tools-28.0.3
 
     # The SDK version used to compile your project
     - android-27

CHANGES.rst

@@ -6,6 +6,7 @@ Features:
 
 Improvements:
  - Any Account data element, even if the type is not known is persisted.
+ - The crypto store is now implemented using a Realm database. The existing file store will be migrated at first usage (#398)
 
 Bugfix:
  - Room members who left are listed with the actual members (vector-im/riot-android#2744)
@@ -23,7 +24,7 @@ Others:
  -
 
 Build:
- -
+ - Introduce Kotlin to the SDK
 
 Test:
  -

build.gradle

@@ -1,6 +1,8 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 
 buildscript {
+    ext.kotlin_version = '1.3.10'
+
     repositories {
         google()
         jcenter()
@@ -9,9 +11,11 @@ buildscript {
         }
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.1.2'
+        classpath 'com.android.tools.build:gradle:3.2.1'
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
 
         classpath 'org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:2.6.2'
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
     }
@@ -21,6 +25,7 @@ allprojects {
     repositories {
         google()
         jcenter()
+        mavenCentral()
     }
 }
 

matrix-sdk/build.gradle

@@ -1,8 +1,20 @@
 apply plugin: 'com.android.library'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlin-kapt'
+apply plugin: 'realm-android'
+
+buildscript {
+    repositories {
+        jcenter()
+    }
+
+    dependencies {
+        classpath "io.realm:realm-gradle-plugin:5.7.0"
+    }
+}
 
 android {
     compileSdkVersion 27
-    buildToolsVersion '27.0.3'
 
     packagingOptions {
         exclude 'LICENSE.txt'
@@ -123,6 +135,9 @@ static def gitRevisionDate() {
 }
 
 dependencies {
+    // Kotlin
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+
     implementation 'com.android.support:appcompat-v7:27.1.1'
     implementation 'com.squareup.retrofit2:retrofit:2.4.0'
     implementation 'com.squareup.retrofit2:converter-gson:2.4.0'
@@ -134,6 +149,9 @@ dependencies {
 
     implementation(name: 'olm-sdk', ext: 'aar')
 
+    // Database
+    kapt 'dk.ilios:realmfieldnameshelper:1.1.1'
+
     // replace the compile 'io.pristine:libjingle:9690@aar'
     implementation(name: 'react-native-webrtc', ext: 'aar')
 

matrix-sdk/src/androidTest/java/org/matrix/androidsdk/common/CommonTestHelper.java

@@ -31,6 +31,7 @@
 import org.matrix.androidsdk.data.RoomMediaMessage;
 import org.matrix.androidsdk.data.store.IMXStore;
 import org.matrix.androidsdk.data.store.MXFileStore;
+import org.matrix.androidsdk.data.store.MXStoreListener;
 import org.matrix.androidsdk.listeners.MXEventListener;
 import org.matrix.androidsdk.rest.client.LoginRestClient;
 import org.matrix.androidsdk.rest.model.Event;
@@ -178,8 +179,7 @@ private MXSession createAccount(@NonNull final String userNamePrefix,
                 context,
                 userNamePrefix + "_" + System.currentTimeMillis() + UUID.randomUUID(),
                 password,
-                testParams.withInitialSync,
-                testParams.withCryptoEnabled
+                testParams
         );
         Assert.assertNotNull(session);
         return session;
@@ -205,17 +205,15 @@ private MXSession logIntoAccount(@NonNull final String userId,
     /**
      * Create an account and a dedicated session
      *
-     * @param context         the context
-     * @param userName        the account username
-     * @param password        the password
-     * @param withInitialSync true to perform an initial sync
-     * @param enableCrypto    true to set enableCryptoWhenStarting
+     * @param context           the context
+     * @param userName          the account username
+     * @param password          the password
+     * @param sessionTestParams parameters for the test
      */
     private MXSession createAccountAndSync(Context context,
                                            String userName,
                                            String password,
-                                           boolean withInitialSync,
-                                           boolean enableCrypto) throws InterruptedException {
+                                           SessionTestParams sessionTestParams) throws InterruptedException {
         final HomeServerConnectionConfig hs = createHomeServerConfig(null);
 
         final LoginRestClient loginRestClient = new LoginRestClient(hs);
@@ -284,16 +282,17 @@ public void onSuccess(Credentials credentials) {
 
         MXDataHandler dataHandler = new MXDataHandler(store, credentials);
         // TODO Use sessionTestParam parameter when other PR will be merged
-        dataHandler.setLazyLoadingEnabled(true);
+        dataHandler.setLazyLoadingEnabled(sessionTestParams.getWithLazyLoading());
 
         MXSession mxSession = new MXSession.Builder(hs, dataHandler, context)
+                .withLegacyCryptoStore(sessionTestParams.getWithLegacyCryptoStore())
                 .build();
 
-        if (enableCrypto) {
+        if (sessionTestParams.getWithCryptoEnabled()) {
             mxSession.enableCryptoWhenStarting();
         }
-        if (withInitialSync) {
-            syncSession(mxSession, enableCrypto);
+        if (sessionTestParams.getWithInitialSync()) {
+            syncSession(mxSession, sessionTestParams.getWithCryptoEnabled());
         }
         return mxSession;
     }
@@ -335,16 +334,17 @@ public void onSuccess(Credentials credentials) {
         final IMXStore store = new MXFileStore(hs, false, context);
 
         MXDataHandler mxDataHandler = new MXDataHandler(store, credentials);
-        mxDataHandler.setLazyLoadingEnabled(sessionTestParams.withLazyLoading);
+        mxDataHandler.setLazyLoadingEnabled(sessionTestParams.getWithLazyLoading());
 
         final MXSession mxSession = new MXSession.Builder(hs, mxDataHandler, context)
+                .withLegacyCryptoStore(sessionTestParams.getWithLegacyCryptoStore())
                 .build();
 
-        if (sessionTestParams.withCryptoEnabled) {
+        if (sessionTestParams.getWithCryptoEnabled()) {
             mxSession.enableCryptoWhenStarting();
         }
-        if (sessionTestParams.withInitialSync) {
-            syncSession(mxSession, sessionTestParams.withCryptoEnabled);
+        if (sessionTestParams.getWithInitialSync()) {
+            syncSession(mxSession, sessionTestParams.getWithCryptoEnabled());
         }
         return mxSession;
     }
@@ -371,4 +371,62 @@ public void clearAllSessions(List<MXSession> sessions) {
             session.clear(context);
         }
     }
+
+    /**
+     * Clone a session
+     * // TODO Use this method where it should be (after merge of keys backup)
+     *
+     * @param from the session to clone
+     * @return the duplicated session
+     */
+    @NonNull
+    public MXSession createNewSession(@NonNull MXSession from, SessionTestParams sessionTestParams) throws InterruptedException {
+        final Context context = InstrumentationRegistry.getContext();
+
+        Credentials aliceCredentials = from.getCredentials();
+        HomeServerConnectionConfig hs = createHomeServerConfig(aliceCredentials);
+        MXFileStore store = new MXFileStore(hs, false, context);
+        MXDataHandler dataHandler = new MXDataHandler(store, aliceCredentials);
+        store.setDataHandler(dataHandler);
+        MXSession session2 = new MXSession.Builder(hs, dataHandler, context)
+                .withLegacyCryptoStore(sessionTestParams.getWithLegacyCryptoStore())
+                .build();
+
+        final Map<String, Object> results = new HashMap<>();
+
+        final CountDownLatch lock = new CountDownLatch(1);
+        MXStoreListener listener = new MXStoreListener() {
+            @Override
+            public void postProcess(String accountId) {
+                results.put("postProcess", "postProcess " + accountId);
+            }
+
+            @Override
+            public void onStoreReady(String accountId) {
+                results.put("onStoreReady", "onStoreReady");
+                lock.countDown();
+            }
+
+            @Override
+            public void onStoreCorrupted(String accountId, String description) {
+                results.put("onStoreCorrupted", description);
+                lock.countDown();
+            }
+
+            @Override
+            public void onStoreOOM(String accountId, String description) {
+                results.put("onStoreOOM", "onStoreOOM");
+                lock.countDown();
+            }
+        };
+
+        store.addMXStoreListener(listener);
+        store.open();
+
+        await(lock);
+
+        Assert.assertTrue(results.toString(), results.containsKey("onStoreReady"));
+
+        return session2;
+    }
 }

matrix-sdk/src/androidTest/java/org/matrix/androidsdk/common/SessionTestParams.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.androidsdk.common
+
+class SessionTestParams @JvmOverloads constructor(val withInitialSync: Boolean = false,
+                                                  val withCryptoEnabled: Boolean = false,
+                                                  val withLazyLoading: Boolean = false,
+                                                  val withLegacyCryptoStore: Boolean = true)

matrix-sdk/src/androidTest/java/org/matrix/androidsdk/crypto/CryptoRestTest.java

@@ -54,9 +54,7 @@ public class CryptoRestTest {
     @Test
     public void test01_testDeviceKeys() throws Exception {
         final Context context = InstrumentationRegistry.getContext();
-        final SessionTestParams testParams = SessionTestParams.newBuilder()
-                .withInitialSync(true)
-                .build();
+        final SessionTestParams testParams = new SessionTestParams(true);
         final MXSession bobSession = mTestHelper.createAccount(TestConstants.USER_BOB, testParams);
         final Map<String, Object> results = new HashMap<>();
 
@@ -122,9 +120,7 @@ public void onSuccess(KeysQueryResponse keysQueryResponse) {
     @Test
     public void test02_testOneTimeKeys() throws Exception {
         Context context = InstrumentationRegistry.getContext();
-        final SessionTestParams testParams = SessionTestParams.newBuilder()
-                .withInitialSync(true)
-                .build();
+        final SessionTestParams testParams = new SessionTestParams(true);
         final MXSession bobSession = mTestHelper.createAccount(TestConstants.USER_BOB, testParams);
 
         final Map<String, Object> results = new HashMap<>();
@@ -157,9 +153,7 @@ public void onSuccess(KeysUploadResponse keysUploadResponse) {
     @Test
     public void test03_testClaimOneTimeKeysForUsersDevices() throws Exception {
         Context context = InstrumentationRegistry.getContext();
-        final SessionTestParams testParams = SessionTestParams.newBuilder()
-                .withInitialSync(true)
-                .build();
+        final SessionTestParams testParams = new SessionTestParams(true);
         final MXSession bobSession = mTestHelper.createAccount(TestConstants.USER_BOB, testParams);
         final MXSession aliceSession = mTestHelper.createAccount(TestConstants.USER_ALICE, testParams);