Rust Crypto SDK is now the default, and the build will replace the existing application

.github/workflows/build.yml

@@ -33,7 +33,7 @@ jobs:
         with:
           cache-read-only: ${{ github.ref != 'refs/heads/develop' }}
       - name: Assemble ${{ matrix.target }} debug apk
-        run: ./gradlew assemble${{ matrix.target }}KotlinCryptoDebug $CI_GRADLE_ARG_PROPERTIES
+        run: ./gradlew assemble${{ matrix.target }}RustCryptoDebug $CI_GRADLE_ARG_PROPERTIES
       - name: Upload ${{ matrix.target }} debug APKs
         uses: actions/upload-artifact@v3
         with:
@@ -57,7 +57,7 @@ jobs:
         with:
           cache-read-only: ${{ github.ref != 'refs/heads/develop' }}
       - name: Assemble GPlay unsigned apk
-        run: ./gradlew clean assembleGplayKotlinCryptoRelease $CI_GRADLE_ARG_PROPERTIES
+        run: ./gradlew clean assembleGplayRustCryptoRelease $CI_GRADLE_ARG_PROPERTIES
       - name: Upload Gplay unsigned APKs
         uses: actions/upload-artifact@v3
         with:
@@ -79,7 +79,7 @@ jobs:
       - name: Execute exodus-standalone
         uses: docker://exodusprivacy/exodus-standalone:latest
         with:
-          args: /github/workspace/gplayKotlinCrypto/release/vector-gplay-kotlinCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json
+          args: /github/workspace/gplayRustCrypto/release/vector-gplay-rustCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json
       - name: Upload exodus json report
         uses: actions/upload-artifact@v3
         with:

.github/workflows/nightly.yml

@@ -34,7 +34,7 @@ jobs:
           yes n | towncrier build --version nightly
       - name: Build and upload Gplay Nightly APK
         run: |
-          ./gradlew assembleGplayKotlinCryptoNightly appDistributionUploadGplayKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES
+          ./gradlew assembleGplayRustCryptoNightly appDistributionUploadGplayRustCryptoNightly $CI_GRADLE_ARG_PROPERTIES
         env:
           ELEMENT_ANDROID_NIGHTLY_KEYID: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYID }}
           ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD }}

build.gradle

@@ -323,7 +323,7 @@ tasks.register("recordScreenshots", GradleBuild) {
 
 tasks.register("verifyScreenshots", GradleBuild) {
     startParameter.projectProperties.screenshot = ""
-    tasks = [':vector:verifyPaparazziKotlinCryptoDebug']
+    tasks = [':vector:verifyPaparazziRustCryptoDebug']
 }
 
 ext.initScreenshotTests = { project ->

changelog.d/8390.feature

@@ -0,0 +1 @@
+Element Android is now using the Crypto Rust SDK. Migration of user's data should be done at first launch after application upgrade.

changelog.d/8405.sdk

@@ -0,0 +1 @@
+Add crypto database migration 22, that extract account and olm session to the new rust DB format

docs/nightly_build.md

@@ -48,7 +48,7 @@ mv towncrier.toml towncrier.toml.bak
 sed 's/CHANGES\.md/CHANGES_NIGHTLY\.md/' towncrier.toml.bak > towncrier.toml
 rm towncrier.toml.bak
 yes n | towncrier build --version nightly
-./gradlew assembleGplayKotlinCryptoNightly appDistributionUploadGplayKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES
+./gradlew assembleGplayRustCryptoNightly appDistributionUploadRustKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES
 ```
 
 Then you can reset the change on the codebase.

flavor.gradle

@@ -5,13 +5,13 @@ android {
     productFlavors {
         kotlinCrypto {
             dimension "crypto"
-            isDefault = true
             // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}"
 //            buildConfigField "String", "SHORT_FLAVOR_DESCRIPTION", "\"JC\""
 //            buildConfigField "String", "FLAVOR_DESCRIPTION", "\"KotlinCrypto\""
         }
         rustCrypto {
             dimension "crypto"
+            isDefault = true
 //            // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}"
 //            buildConfigField "String", "SHORT_FLAVOR_DESCRIPTION", "\"RC\""
 //            buildConfigField "String", "FLAVOR_DESCRIPTION", "\"RustCrypto\""

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt → matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt

@@ -46,11 +46,15 @@ class CryptoSanityMigrationTest {
     @Test
     fun cryptoDatabaseShouldMigrateGracefully() {
         val realmName = "crypto_store_20.realm"
-        val migration = RealmCryptoStoreMigration(object : Clock {
-            override fun epochMillis(): Long {
-                return 0L
-            }
-        })
+
+        val migration = RealmCryptoStoreMigration(
+                object : Clock {
+                    override fun epochMillis(): Long {
+                        return 0L
+                    }
+                }
+        )
+
         val realmConfiguration = configurationFactory.createConfiguration(
                 realmName,
                 "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca",

matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt → matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt

@@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration
 import android.content.Context
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
+import io.mockk.spyk
 import io.realm.Realm
 import io.realm.kotlin.where
 import org.amshove.kluent.internal.assertEquals
@@ -30,56 +31,82 @@ import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
-import org.matrix.android.sdk.InstrumentedTest
+import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
+import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
+import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
 import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
 import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
+import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider
 import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
+import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
+import org.matrix.android.sdk.internal.database.RealmKeysUtils
 import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory
-import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
 import org.matrix.android.sdk.internal.util.time.Clock
+import org.matrix.android.sdk.test.shared.createTimberTestRule
 import org.matrix.olm.OlmAccount
 import org.matrix.olm.OlmManager
 import org.matrix.rustcomponents.sdk.crypto.OlmMachine
 import java.io.File
+import java.security.KeyStore
 
 @RunWith(AndroidJUnit4::class)
-class ElementAndroidToElementRMigrationTest : InstrumentedTest {
+class DynamicElementAndroidToElementRMigrationTest {
 
     @get:Rule val configurationFactory = TestRealmConfigurationFactory()
 
-    lateinit var context: Context
+    @Rule
+    fun timberTestRule() = createTimberTestRule()
+
+    var context: Context = InstrumentationRegistry.getInstrumentation().context
     var realm: Realm? = null
 
     @Before
     fun setUp() {
         // Ensure Olm is initialized
         OlmManager()
-        context = InstrumentationRegistry.getInstrumentation().context
     }
 
     @After
     fun tearDown() {
         realm?.close()
     }
 
+    private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
+
+    private val rustEncryptionConfiguration = RustEncryptionConfiguration(
+            "foo",
+            RealmKeysUtils(
+                    context,
+                    SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
+            )
+    )
+
+    private val fakeClock = object : Clock {
+        override fun epochMillis() = 0L
+    }
+
     @Test
     fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
         testMigrate(false)
     }
 
     @Test
-    @Ignore("We don't migrate group session for now, and it makes test suite unstable")
+    @Ignore("We don't migrate group sessions for now, and it's making this test suite unstable")
     fun given_a_valid_crypto_store_realm_file_no_lazy_then_migration_should_be_successful() {
         testMigrate(true)
     }
 
     private fun testMigrate(migrateGroupSessions: Boolean) {
+        val targetFile = File(configurationFactory.root, "rust-sdk")
+
         val realmName = "crypto_store_migration_16.realm"
-        val migration = RealmCryptoStoreMigration(object : Clock {
-            override fun epochMillis() = 0L
-        })
+        val infoProvider = RustMigrationInfoProvider(
+                targetFile,
+                rustEncryptionConfiguration
+        ).apply {
+            migrateMegolmGroupSessions = migrateGroupSessions
+        }
+        val migration = RealmCryptoStoreMigration(fakeClock, infoProvider)
 
         val realmConfiguration = configurationFactory.createConfiguration(
                 realmName,
@@ -91,19 +118,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
         configurationFactory.copyRealmFromAssets(context, realmName, realmName)
 
         realm = Realm.getInstance(realmConfiguration)
-
         val metaData = realm!!.where<CryptoMetadataEntity>().findFirst()!!
         val userId = metaData.userId!!
         val deviceId = metaData.deviceId!!
         val olmAccount = metaData.getOlmAccount()!!
 
-        val extractor = MigrateEAtoEROperation(migrateGroupSessions)
-
-        val targetFile = File(configurationFactory.root, "rust-sdk")
-
-        extractor.execute(realmConfiguration, targetFile, null)
-
-        val machine = OlmMachine(userId, deviceId, targetFile.path, null)
+        val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase())
 
         assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"])
         assertNotNull(machine.getBackupKeys())
@@ -113,28 +133,15 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
         assertTrue(crossSigningStatus.hasUserSigning)
 
         if (migrateGroupSessions) {
-            val inboundGroupSessionEntities = realm!!.where<OlmInboundGroupSessionEntity>().findAll()
-            assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt())
-
-            val backedUpInboundGroupSessionEntities = realm!!
-                    .where<OlmInboundGroupSessionEntity>()
-                    .equalTo(OlmInboundGroupSessionEntityFields.BACKED_UP, true)
-                    .findAll()
-            assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt())
+            assertTrue("Some outbound sessions should be migrated", machine.roomKeyCounts().total.toInt() > 0)
+            assertTrue("There are some backed-up sessions", machine.roomKeyCounts().backedUp.toInt() > 0)
+        } else {
+            assertTrue(machine.roomKeyCounts().total.toInt() == 0)
+            assertTrue(machine.roomKeyCounts().backedUp.toInt() == 0)
         }
-    }
 
-//    @Test
-//    fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() {
-//        val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false)
-//        Realm.getInstance(realmConfiguration).use {
-//            assertTrue(it.isEmpty)
-//        }
-//        val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
-//        assertNull(machine.getBackupKeys())
-//        val crossSigningStatus = machine.crossSigningStatus()
-//        assertFalse(crossSigningStatus.hasMaster)
-//        assertFalse(crossSigningStatus.hasSelfSigning)
-//        assertFalse(crossSigningStatus.hasUserSigning)
-//    }
+        // legacy olm sessions should have been deleted
+        val remainingOlmSessions = realm!!.where<OlmSessionEntity>().findAll().size
+        assertEquals("legacy olm sessions should have been removed from store", 0, remainingOlmSessions)
+    }
 }

matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2023 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database
+
+import android.content.Context
+import androidx.test.platform.app.InstrumentationRegistry
+import io.mockk.spyk
+import io.realm.Realm
+import org.junit.After
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
+import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
+import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
+import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
+import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
+import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider
+import org.matrix.android.sdk.internal.util.time.Clock
+import org.matrix.olm.OlmManager
+import java.io.File
+import java.security.KeyStore
+
+class CryptoSanityMigrationTest {
+    @get:Rule val configurationFactory = TestRealmConfigurationFactory()
+
+    lateinit var context: Context
+    var realm: Realm? = null
+
+    @Before
+    fun setUp() {
+        // Ensure Olm is initialized
+        OlmManager()
+        context = InstrumentationRegistry.getInstrumentation().context
+    }
+
+    @After
+    fun tearDown() {
+        realm?.close()
+    }
+
+    private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
+
+    @Test
+    fun cryptoDatabaseShouldMigrateGracefully() {
+        val realmName = "crypto_store_20.realm"
+
+        val rustMigrationInfo = RustMigrationInfoProvider(
+                File(configurationFactory.root, "test_rust"),
+                RustEncryptionConfiguration(
+                        "foo",
+                        RealmKeysUtils(
+                                context,
+                                SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
+                        )
+                ),
+        )
+        val migration = RealmCryptoStoreMigration(
+                object : Clock {
+                    override fun epochMillis(): Long {
+                        return 0L
+                    }
+                },
+                rustMigrationInfo
+        )
+
+        val realmConfiguration = configurationFactory.createConfiguration(
+                realmName,
+                "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca",
+                RealmCryptoStoreModule(),
+                migration.schemaVersion,
+                migration
+        )
+        configurationFactory.copyRealmFromAssets(context, realmName, realmName)
+
+        realm = Realm.getInstance(realmConfiguration)
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt

@@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService
 import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
 import org.matrix.android.sdk.api.session.typing.TypingUsersTracker
 import org.matrix.android.sdk.api.util.md5
-import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
 import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService
 import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask
 import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask
@@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory
 import org.matrix.android.sdk.internal.di.Authenticated
 import org.matrix.android.sdk.internal.di.CacheDirectory
-import org.matrix.android.sdk.internal.di.CryptoDatabase
 import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory
@@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker
 import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService
 import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter
 import retrofit2.Retrofit
-import timber.log.Timber
 import java.io.File
 import javax.inject.Provider
 import javax.inject.Qualifier
-import kotlin.system.measureTimeMillis
 
 @Qualifier
 @Retention(AnnotationRetention.RUNTIME)
@@ -189,17 +185,8 @@ internal abstract class SessionModule {
         @SessionScope
         fun providesRustCryptoFilesDir(
                 @SessionFilesDirectory parent: File,
-                @CryptoDatabase realmConfiguration: RealmConfiguration,
-                rustEncryptionConfiguration: RustEncryptionConfiguration,
         ): File {
-            val target = File(parent, "rustFlavor")
-            val file: File
-            measureTimeMillis {
-                file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase())
-            }.let { duration ->
-                Timber.v("Migrating to ER in $duration ms")
-            }
-            return file
+            return File(parent, "rustFlavor")
         }
 
         @JvmStatic