Merging Element R in Element Android as a new flavor

.gitattributes

@@ -1,2 +1,3 @@
 **/snapshots/**/*.png filter=lfs diff=lfs merge=lfs -text
 **/src/androidTest/assets/*.realm filter=lfs diff=lfs merge=lfs -text
+**/matrix-rust-sdk-crypto.aar filter=lfs diff=lfs merge=lfs -text

.github/workflows/build.yml

@@ -33,7 +33,7 @@ jobs:
         with:
           cache-read-only: ${{ github.ref != 'refs/heads/develop' }}
       - name: Assemble ${{ matrix.target }} debug apk
-        run: ./gradlew assemble${{ matrix.target }}Debug $CI_GRADLE_ARG_PROPERTIES
+        run: ./gradlew assemble${{ matrix.target }}KotlinCryptoDebug $CI_GRADLE_ARG_PROPERTIES
       - name: Upload ${{ matrix.target }} debug APKs
         uses: actions/upload-artifact@v3
         with:
@@ -57,7 +57,7 @@ jobs:
         with:
           cache-read-only: ${{ github.ref != 'refs/heads/develop' }}
       - name: Assemble GPlay unsigned apk
-        run: ./gradlew clean assembleGplayRelease $CI_GRADLE_ARG_PROPERTIES
+        run: ./gradlew clean assembleGplayKotlinCryptoRelease $CI_GRADLE_ARG_PROPERTIES
       - name: Upload Gplay unsigned APKs
         uses: actions/upload-artifact@v3
         with:
@@ -79,7 +79,7 @@ jobs:
       - name: Execute exodus-standalone
         uses: docker://exodusprivacy/exodus-standalone:latest
         with:
-          args: /github/workspace/gplay/release/vector-gplay-universal-release-unsigned.apk -j -o /github/workspace/exodus.json
+          args: /github/workspace/gplayKotlinCrypto/release/vector-gplay-kotlinCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json
       - name: Upload exodus json report
         uses: actions/upload-artifact@v3
         with:

.github/workflows/elementr.yml

@@ -0,0 +1,37 @@
+name: ER APK Build
+
+on:
+  pull_request: { }
+  push:
+    branches: [ develop ]
+
+# Enrich gradle.properties for CI/CD
+env:
+  GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx3072m -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError" -Dkotlin.daemon.jvm.options="-Xmx2560m" -Dkotlin.incremental=false
+  CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 2 --no-daemon
+
+jobs:
+  debug:
+    name: Build debug APKs ER
+    runs-on: ubuntu-latest
+    if: github.ref != 'refs/heads/main'
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [ Gplay, Fdroid ]
+    # Allow all jobs on develop. Just one per PR.
+    concurrency:
+      group: ${{ github.ref == 'refs/heads/develop' && format('elementr-{0}-{1}', matrix.target, github.sha) || format('build-er-debug-{0}-{1}', matrix.target, github.ref)  }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Assemble ${{ matrix.target }} debug apk
+        run: ./gradlew assemble${{ matrix.target }}RustCryptoDebug $CI_GRADLE_ARG_PROPERTIES

.github/workflows/nightly.yml

@@ -34,7 +34,7 @@ jobs:
           yes n | towncrier build --version nightly
       - name: Build and upload Gplay Nightly APK
         run: |
-          ./gradlew assembleGplayNightly appDistributionUploadGplayNightly $CI_GRADLE_ARG_PROPERTIES
+          ./gradlew assembleGplayKotlinCryptoNightly appDistributionUploadGplayKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES
         env:
           ELEMENT_ANDROID_NIGHTLY_KEYID: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYID }}
           ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD }}

.github/workflows/nightly_er.yml

@@ -0,0 +1,46 @@
+name: Build and release Element R nightly APK
+
+on:
+  schedule:
+    # Every nights at 4
+    - cron: "0 4 * * *"
+
+env:
+  GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx3072m -Dfile.encoding=UTF-8 -XX:+HeapDumpOnOutOfMemoryError" -Dkotlin.daemon.jvm.options="-Xmx2560m" -Dkotlin.incremental=false
+  CI_GRADLE_ARG_PROPERTIES: --stacktrace -PpreDexEnable=false --max-workers 2 --no-daemon
+
+jobs:
+  nightly:
+    name: Build and publish ER nightly Gplay APK to Firebase
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Install towncrier
+        run: |
+          python3 -m pip install towncrier
+      - name: Prepare changelog file
+        run: |
+          mv towncrier.toml towncrier.toml.bak
+          sed 's/CHANGES\.md/CHANGES_NIGHTLY\.md/' towncrier.toml.bak > towncrier.toml
+          rm towncrier.toml.bak
+          yes n | towncrier build --version nightly
+      - name: Build and upload Gplay Nightly ER APK
+        run: |
+          ./gradlew assembleGplayRustCryptoNightly appDistributionUploadGplayRustCryptoNightly $CI_GRADLE_ARG_PROPERTIES
+        env:
+          ELEMENT_ANDROID_NIGHTLY_KEYID: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYID }}
+          ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD }}
+          ELEMENT_ANDROID_NIGHTLY_STOREPASSWORD: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_STOREPASSWORD }}
+          FIREBASE_TOKEN: ${{ secrets.ELEMENT_R_NIGHTLY_FIREBASE_TOKEN }}

.github/workflows/quality.yml

@@ -49,8 +49,10 @@ jobs:
       - name: Run lint
         # Not always, if ktlint or detekt fail, avoid running the long lint check.
         run: |
-          ./gradlew vector-app:lintGplayRelease $CI_GRADLE_ARG_PROPERTIES
-          ./gradlew vector-app:lintFdroidRelease $CI_GRADLE_ARG_PROPERTIES
+          ./gradlew vector-app:lintGplayKotlinCryptoRelease $CI_GRADLE_ARG_PROPERTIES
+          ./gradlew vector-app:lintFdroidKotlinCryptoRelease $CI_GRADLE_ARG_PROPERTIES
+          ./gradlew vector-app:lintGplayRustCryptoRelease $CI_GRADLE_ARG_PROPERTIES
+          ./gradlew vector-app:lintFdroidRustCryptoRelease $CI_GRADLE_ARG_PROPERTIES
       - name: Upload reports
         if: always()
         uses: actions/upload-artifact@v3

.github/workflows/tests.yml

@@ -73,7 +73,7 @@ jobs:
           disable-animations: true
           # emulator-build: 7425822
           script: |
-            ./gradlew gatherGplayDebugStringTemplates $CI_GRADLE_ARG_PROPERTIES
+            ./gradlew gatherGplayKotlinCryptoDebugStringTemplates $CI_GRADLE_ARG_PROPERTIES
             ./gradlew unitTestsWithCoverage $CI_GRADLE_ARG_PROPERTIES
             ./gradlew instrumentationTestsWithCoverage $CI_GRADLE_ARG_PROPERTIES
             ./gradlew generateCoverageReport $CI_GRADLE_ARG_PROPERTIES

.gitignore

@@ -12,6 +12,9 @@
 /benchmark-out
 /captures
 .externalNativeBuild
+rust-sdk/target/*
+rust-sdk/src/uniffi/*
+Cargo.lock
 
 /tmp
 /fastlane/private
@@ -24,3 +27,10 @@
 /yarn.lock
 /node_modules
 **/out/failures
+
+# For manual dependency to rust crypto sdk
+matrix-sdk-android/src/main/jniLibs/
+
+matrix-sdk-android/libs/crypto-android-release.aar
+
+matrix-sdk-android/libs/matrix-rust-sdk-crypto.aar

build.gradle

@@ -121,6 +121,15 @@ allprojects {
                 groups.jcenter.group.each { includeGroup it }
             }
         }
+
+        maven {
+            url 'https://s01.oss.sonatype.org/content/repositories/snapshots'
+            content {
+                groups.mavenSnapshots.regex.each { includeGroupByRegex it }
+                groups.mavenSnapshots.group.each { includeGroup it }
+            }
+        }
+
     }
 
     tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {
@@ -314,7 +323,7 @@ tasks.register("recordScreenshots", GradleBuild) {
 
 tasks.register("verifyScreenshots", GradleBuild) {
     startParameter.projectProperties.screenshot = ""
-    tasks = [':vector:verifyPaparazziDebug']
+    tasks = [':vector:verifyPaparazziKotlinCryptoDebug']
 }
 
 ext.initScreenshotTests = { project ->

coverage.gradle

@@ -87,5 +87,5 @@ task unitTestsWithCoverage(type: GradleBuild) {
 task instrumentationTestsWithCoverage(type: GradleBuild) {
     startParameter.projectProperties.coverage = "true"
     startParameter.projectProperties['android.testInstrumentationRunnerArguments.notPackage'] = 'im.vector.app.ui'
-    tasks = [':vector-app:connectedGplayDebugAndroidTest', ':vector:connectedDebugAndroidTest', 'matrix-sdk-android:connectedDebugAndroidTest']
+    tasks = [':vector-app:connectedGplayKotlinCryptoDebugAndroidTest', ':vector:connectedKotlinCryptoDebugAndroidTest', 'matrix-sdk-android:connectedKotlinCryptoDebugAndroidTest']
 }

dependencies_groups.gradle

@@ -1,5 +1,5 @@
 ext.groups = [
-        jitpack     : [
+        jitpack       : [
                 regex: [
                 ],
                 group: [
@@ -15,7 +15,7 @@ ext.groups = [
                         'com.github.Zhuinden',
                 ]
         ],
-        jitsi       : [
+        jitsi         : [
                 regex: [
                 ],
                 group: [
@@ -24,7 +24,7 @@ ext.groups = [
                         'org.webkit',
                 ]
         ],
-        google      : [
+        google        : [
                 regex: [
                         'androidx\\..*',
                         'com\\.android\\.tools\\..*',
@@ -44,6 +44,13 @@ ext.groups = [
                 group: [
                 ]
         ],
+        mavenSnapshots: [
+                regex: [
+                ],
+                group: [
+                        'org.matrix.rustcomponents'
+                ]
+        ],
         mavenCentral: [
                 regex: [
                 ],
@@ -204,6 +211,7 @@ ext.groups = [
                         'org.jvnet.staxex',
                         'org.maplibre.gl',
                         'org.matrix.android',
+                        'org.matrix.rustcomponents',
                         'org.mockito',
                         'org.mongodb',
                         'org.objenesis',
@@ -223,7 +231,7 @@ ext.groups = [
                         'xml-apis',
                 ]
         ],
-        jcenter     : [
+        jcenter       : [
                 regex: [
                 ],
                 group: [

docs/nightly_build.md

@@ -48,7 +48,7 @@ mv towncrier.toml towncrier.toml.bak
 sed 's/CHANGES\.md/CHANGES_NIGHTLY\.md/' towncrier.toml.bak > towncrier.toml
 rm towncrier.toml.bak
 yes n | towncrier build --version nightly
-./gradlew assembleGplayNightly appDistributionUploadGplayNightly $CI_GRADLE_ARG_PROPERTIES
+./gradlew assembleGplayKotlinCryptoNightly appDistributionUploadGplayKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES
 ```
 
 Then you can reset the change on the codebase.

docs/rust_crypto_integration.md

@@ -0,0 +1,63 @@
+## Overview
+
+Until the final migration to [rust crypto sdk](https://github.com/matrix-org/matrix-rust-components-kotlin), the Element Android project will support two
+different SDK as a product flavor.
+
+The `matrix-sdk-android` module is defining a new flavor dimension `crypto`, with two flavors `kotlinCrypto` and `rustCrypto`.
+The crypto module cannot be changed at runtime, it's a build time configuration. The app supports migration from kotlinCrypto to rustCrypto but not the other
+way around.
+
+The code that is not shared between the flavors is located in dedicated source sets (`src/kotlinCrypto/`, `src/rustCrypto/`). Some tests are also extracted
+in different source sets because they were accessing internal API and won't work with the rust crypto sdk.
+
+## Noticeable changes
+
+As a general rule, if you stick to the `kotlinCrypto` the app should behave as it was before the integration of favours.
+There is a noticeable exception though:
+In order to integrate the rust crypto several APIs had to be migrated from callback code to suspendable code. This change
+impacted a lot the key verification engine (user and device verification), so this part has been refactored for `kotlinCrypto`. The UI is also impacted,
+the verification flows now match the web experience.
+
+TLDR; Verification UI and engine has been refactored.
+
+## Testing with a local rust aar
+
+In order to run a custom rust SDK branch you can follow the direction in the [bindings repository](https://github.com/matrix-org/matrix-rust-components-kotlin) 
+in order to build the `matrix-rust-sdk-crypto.aar`.
+
+Copy this lib in `library/rustCrypto/`, and rename it `matrix-rust-sdk-crypto.aar`.
+
+Then go to `matrix-sdk-android/build.gradle` and toggle the comments between the following lines.
+
+````
+ rustCryptoImplementation("org.matrix.rustcomponents:crypto-android:0.3.1")
+ // rustCryptoApi project(":library:rustCrypto")
+````
+
+## Changes in CI
+
+The workflow files have been updated to use the `kotlinCrypto` flavor, e.g
+
+`assembleGplayNightly` => `assembleGplayKotlinCryptoNightly`
+
+So building the unsigned release kotlin crypto apk is now:
+
+`> ./gradlew assembleGplayKotlinCryptoRelease`
+
+An additional workflow has been added to build the `rustCrypto` flavor (elementr.yml, ` Build debug APKs ER`).
+
+
+## Database migration from kotlin to rust
+
+With the kotlin flavor, the crypto information are persisted in the crypto realm database.
+With the rust flavor, the crypto information are in a sqllite database.
+
+The migration is handled when injecting `@SessionRustFilesDirectory` in the olmMachine. 
+When launching the first time after migration, the app will detect that there is no rust data repository and it will
+create one. If there is an existing realm database, the data will then migrated to rust. See `ExtractMigrationDataUseCase`.
+This will extract your device keys, account secrets, active olm and megolm sessions.
+
+There is no inverse migration for now, as there is not yet rust pickle to olm pickle support in the sdk.
+
+If you migrate your app to rust, and want to revert to kotlin you have to logout then login again.
+

flavor.gradle

@@ -0,0 +1,20 @@
+android {
+
+    flavorDimensions "crypto"
+
+    productFlavors {
+        kotlinCrypto {
+            dimension "crypto"
+            isDefault = true
+            // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}"
+//            buildConfigField "String", "SHORT_FLAVOR_DESCRIPTION", "\"JC\""
+//            buildConfigField "String", "FLAVOR_DESCRIPTION", "\"KotlinCrypto\""
+        }
+        rustCrypto {
+            dimension "crypto"
+//            // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}"
+//            buildConfigField "String", "SHORT_FLAVOR_DESCRIPTION", "\"RC\""
+//            buildConfigField "String", "FLAVOR_DESCRIPTION", "\"RustCrypto\""
+        }
+    }
+}

library/rustCrypto/build.gradle

@@ -0,0 +1,2 @@
+configurations.maybeCreate("default")
+artifacts.add("default", file('matrix-rust-sdk-crypto.aar'))