[RNET-6] Add support for client resync (#1901)

* Add support for client resync

* Changelog + tests

CHANGELOG.md

@@ -9,7 +9,9 @@ vNEXT (TBD)
 * Exposed an API - `SyncConfigurationBase.EnableSessionMultiplexing()` that allows toggling session multiplexing on the sync client. (PR [1896](https://github.com/realm/realm-dotnet/pull/1896))
 * Added support for faster initial downloads when using `Realm.GetInstanceAsync`. (Issue [1847](https://github.com/realm/realm-dotnet/issues/1847))
 * Added an optional `cancellationToken` argument to `Realm.GetInstanceAsync` enabling clean cancelation of the in-progress download. (PR [1859](https://github.com/realm/realm-dotnet/pull/1859))
-
+* Added support for Client Resync which automatically will recover the local Realm in case the server is rolled back. This largely replaces the Client Reset mechanism for fully synchronized Realms. Can be configured using `FullSyncConfiguration.ClientResyncMode`. (PR [#1901](https://github.com/realm/realm-dotnet/pull/1901))
+* Made the `createUser` argument in `Credentials.UsernamePassword` optional. If not specified, the user will be created or logged in if they already exist. (PR [#1901](https://github.com/realm/realm-dotnet/pull/1901))
+
 ### Fixed
 * Fixed an infinite recursion when calling `RealmCollectionBase<T>.IndexOf`. (Issue [#1892](https://github.com/realm/realm-dotnet/issues/1892))
 

Realm/Realm/Configurations/FullSyncConfiguration.cs

@@ -33,6 +33,13 @@ public class FullSyncConfiguration : SyncConfigurationBase
     {
         internal override bool IsFullSync => true;
 
+        internal override ClientResyncMode ResyncMode => ClientResyncMode;
+
+        /// <summary>
+        /// Gets or sets a value controlling the behavior in case of a Client Resync. Default is <see cref="ClientResyncMode.RecoverLocalRealm"/>
+        /// </summary>
+        public ClientResyncMode ClientResyncMode { get; set; } = ClientResyncMode.RecoverLocalRealm;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="FullSyncConfiguration"/> class.
         /// </summary>

Realm/Realm/Configurations/QueryBasedSyncConfiguration.cs

@@ -37,6 +37,8 @@ public class QueryBasedSyncConfiguration : SyncConfigurationBase
     {
         internal override bool IsFullSync => false;
 
+        internal override ClientResyncMode ResyncMode => ClientResyncMode.Manual;
+
         internal static readonly Type[] _queryBasedPermissionTypes =
         {
             typeof(ClassPermission),

Realm/Realm/Configurations/SyncConfigurationBase.cs

@@ -90,6 +90,8 @@ public abstract class SyncConfigurationBase : RealmConfigurationBase
         /// </summary>
         public Action<SyncProgress> OnProgress { get; set; }
 
+        internal abstract ClientResyncMode ResyncMode { get; }
+
         /// <summary>
         /// Gets or sets a value indicating how detailed the sync client's logs will be.
         /// </summary>
@@ -302,7 +304,8 @@ internal Native.SyncConfiguration ToNative()
                 client_validate_ssl = EnableSSLValidation,
                 TrustedCAPath = TrustedCAPath,
                 is_partial = !IsFullSync,
-                PartialSyncIdentifier = null
+                PartialSyncIdentifier = null,
+                client_resync_mode = ResyncMode,
             };
         }
 

Realm/Realm/Native/SyncConfiguration.cs

@@ -38,6 +38,7 @@ internal SyncUserHandle SyncUserHandle
 
         [MarshalAs(UnmanagedType.LPWStr)]
         private string url;
+
         private IntPtr url_len;
 
         internal string Url
@@ -54,6 +55,7 @@ internal string Url
 
         [MarshalAs(UnmanagedType.LPWStr)]
         private string trusted_ca_path;
+
         private IntPtr trusted_ca_path_len;
 
         internal string TrustedCAPath
@@ -70,6 +72,7 @@ internal string TrustedCAPath
 
         [MarshalAs(UnmanagedType.LPWStr)]
         private string partial_sync_identifier;
+
         private IntPtr partial_sync_identifier_len;
 
         internal string PartialSyncIdentifier
@@ -80,5 +83,8 @@ internal string PartialSyncIdentifier
                 partial_sync_identifier_len = (IntPtr)(value?.Length ?? 0);
             }
         }
+
+        [MarshalAs(UnmanagedType.U1)]
+        internal ClientResyncMode client_resync_mode;
     }
 }

Realm/Realm/Sync/ClientResyncMode.cs

@@ -0,0 +1,45 @@
+using Realms.Sync.Exceptions;
+
+namespace Realms.Sync
+{
+    /// <summary>
+    /// Enum describing what should happen in case of a Client Resync.
+    /// </summary>
+    /// <remarks>
+    /// A Client Resync is triggered if the device and server cannot agree on a common shared history
+    /// for the Realm file, thus making it impossible for the device to upload or receive any changes.
+    /// This can happen if the server is rolled back or restored from backup.
+    /// <br/>
+    /// IMPORTANT: Just having the device offline will not trigger a Client Resync.
+    /// </remarks>
+    public enum ClientResyncMode : byte
+    {
+        /// <summary>
+        /// Realm will compare the local Realm with the Realm on the server and automatically transfer
+        /// any changes from the local Realm that makes sense to the Realm provided by the server.
+        /// <br/>
+        /// This is the default mode for fully synchronized Realms. It is not yet supported by
+        /// Query-based Realms.
+        /// </summary>
+        RecoverLocalRealm = 0,
+
+        /// <summary>
+        /// The local Realm will be discarded and replaced with the server side Realm.
+        /// All local changes will be lost.
+        /// <br/>
+        /// This mode is not yet supported by Query-based Realms.
+        /// </summary>
+        DiscardLocalRealm = 1,
+
+        /// <summary>
+        /// A manual Client Resync is also known as a Client Reset.
+        /// <br/>
+        /// A <see cref="ClientResetException"/> will be sent to <see cref="Session.Error"/>,
+        /// triggering a Client Reset. Doing this provides a handle to both the old and new Realm file, enabling
+        /// full control over which changes to move, if any.
+        /// <br/>
+        /// This is the only supported mode for Query-based Realms.
+        /// </summary>
+        Manual = 2,
+    }
+}

Realm/Realm/Sync/Credentials.cs

@@ -121,13 +121,19 @@ public static Credentials Google(string googleToken)
         /// <param name="password">The user's password.</param>
         /// <param name="createUser"><c>true</c> if the user should be created, <c>false</c> otherwise. It is not possible to create a user twice when logging in, so this flag should only be set to true the first time a user logs in.</param>
         /// <returns>An instance of <see cref="Credentials"/> that can be used in <see cref="User.LoginAsync"/></returns>
-        public static Credentials UsernamePassword(string username, string password, bool createUser)
+        public static Credentials UsernamePassword(string username, string password, bool? createUser = null)
         {
+            var userInfo = new Dictionary<string, object> { [Keys.Password] = password };
+            if (createUser != null)
+            {
+                userInfo[Keys.CreateUser] = createUser;
+            }
+
             return new Credentials
             {
                 IdentityProvider = Provider.UsernamePassword,
                 Token = username,
-                UserInfo = new Dictionary<string, object> { [Keys.CreateUser] = createUser, [Keys.Password] = password }
+                UserInfo = userInfo,
             };
         }
 

Tests/Realm.Tests/Sync/SyncTestBase.cs

@@ -100,6 +100,16 @@ protected Session GetSession(Realm realm)
             return result;
         }
 
+        /// <summary>
+        /// Waits for upload and immediately disposes of the managed handle to ensure the Realm can be safely deleted.
+        /// </summary>
+        protected async Task WaitForUploadAsync(Realm realm)
+        {
+            var session = realm.GetSession();
+            await session.WaitForUploadAsync();
+            session.CloseHandle();
+        }
+
         protected Realm GetRealm(RealmConfigurationBase config)
         {
             var result = Realm.GetInstance(config);

Tests/Realm.Tests/Sync/SynchronizedInstanceTests.cs

@@ -17,6 +17,7 @@
 ////////////////////////////////////////////////////////////////////////////
 
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -287,6 +288,172 @@ public void GetInstance_WhenDynamicAndDoesntExist_ReturnsEmptySchema()
             });
         }
 
+        // Used by TestClientResync and TestClientResync2. Must be either RecoverLocal or DiscardLocal. Manual is tested
+        // by TestManualClientResync.
+        private ClientResyncMode _clientResyncMode = ClientResyncMode.DiscardLocalRealm;
+
+        private async Task<FullSyncConfiguration> GetClientResyncConfig(ClientResyncMode? _mode = null)
+        {
+            if (!_mode.HasValue)
+            {
+                _mode = _clientResyncMode;
+            }
+
+            var user = await User.LoginAsync(Credentials.UsernamePassword("foo", "bar"), SyncTestHelpers.AuthServerUri);
+            return new FullSyncConfiguration(SyncTestHelpers.RealmUri($"~/{_mode.Value}"), user, $"{_mode}.realm")
+            {
+                ClientResyncMode = _mode.Value,
+                ObjectClasses = new[] { typeof(IntPrimaryKeyWithValueObject) },
+            };
+        }
+
+        [Test, NUnit.Framework.Explicit("Requires debugger and a lot of manual steps")]
+        public void TestClientResync()
+        {
+            SyncTestHelpers.RunRosTestAsync(async () =>
+            {
+                var config = await GetClientResyncConfig();
+                // Let's delete anything local.
+                Realm.DeleteRealm(config);
+                Exception ex = null;
+                Session.Error += (s, e) =>
+                {
+                    if (e.Exception.Message != "End of input")
+                    {
+                        Debugger.Break();
+                        ex = e.Exception;
+                    }
+                };
+
+                using (var realm = await Realm.GetInstanceAsync(config))
+                {
+                    realm.Write(() =>
+                    {
+                        realm.Add(new IntPrimaryKeyWithValueObject
+                        {
+                            Id = 1,
+                            StringValue = "1"
+                        });
+                    });
+
+                    await WaitForUploadAsync(realm);
+                }
+
+                // Stop ROS and backup the file. Then restart
+                Debugger.Break();
+
+                using (var realm = await GetRealmAsync(config))
+                {
+                    realm.Write(() =>
+                    {
+                        realm.Add(new IntPrimaryKeyWithValueObject
+                        {
+                            Id = 2,
+                            StringValue = "2"
+                        });
+                    });
+
+                    await WaitForUploadAsync(realm);
+
+                    // Stop ROS
+                    Debugger.Break();
+
+                    realm.Write(() =>
+                    {
+                        realm.Add(new IntPrimaryKeyWithValueObject
+                        {
+                            Id = 3,
+                            StringValue = "3"
+                        });
+                    });
+                }
+
+                // Replace the file from backup. Restart ROS and run TestClientResync2
+                Debugger.Break();
+
+                Assert.That(ex, Is.Null);
+            }, (int)TimeSpan.FromMinutes(10).TotalMilliseconds);
+        }
+
+        [Test, NUnit.Framework.Explicit("Requires debugger and a lot of manual steps")]
+        public void TestClientResync2()
+        {
+            Assert.That(new[] { ClientResyncMode.DiscardLocalRealm, ClientResyncMode.RecoverLocalRealm }, Does.Contain(_clientResyncMode));
+
+            SyncTestHelpers.RunRosTestAsync(async () =>
+            {
+                var config = await GetClientResyncConfig();
+
+                Exception ex = null;
+                Session.Error += (s, e) =>
+                {
+                    if (e.Exception.Message != "End of input")
+                    {
+                        ex = e.Exception;
+                    }
+                };
+
+                using (var realm = await GetRealmAsync(config))
+                {
+                    var values = realm.All<IntPrimaryKeyWithValueObject>().AsEnumerable().Select(i => i.StringValue).ToArray();
+
+                    // Verify expected result:
+                    //   - RecoverLocalRealm: we have 2 objects - "1" and "3". The "2" is lost because the client had uploaded them to the server already.
+                    //   - DiscardLocalRealm: we have 1 object - "1". The "2" is lost because we restored from backup and the "3" is discarded.
+                    switch (_clientResyncMode)
+                    {
+                        case ClientResyncMode.DiscardLocalRealm:
+                            Assert.That(values.Length, Is.EqualTo(1));
+                            Assert.That(values[0], Is.EqualTo("1"));
+                            Assert.That(ex, Is.Null);
+                            break;
+
+                        case ClientResyncMode.RecoverLocalRealm:
+                            Assert.That(values.Length, Is.EqualTo(2));
+                            CollectionAssert.AreEquivalent(values, new[] { "1", "3" });
+                            Assert.That(ex, Is.Null);
+                            break;
+                    }
+                }
+            }, (int)TimeSpan.FromMinutes(10).TotalMilliseconds);
+        }
+
+        [Test, NUnit.Framework.Explicit("Requires debugger and a lot of manual steps")]
+        public void TestManualClientResync()
+        {
+            SyncTestHelpers.RunRosTestAsync(async () =>
+            {
+                var config = await GetClientResyncConfig(ClientResyncMode.Manual);
+
+                Realm.DeleteRealm(config);
+                using (var realm = await Realm.GetInstanceAsync(config))
+                {
+                    realm.Write(() =>
+                    {
+                        realm.Add(new IntPrimaryKeyWithValueObject());
+                    });
+
+                    await WaitForUploadAsync(realm);
+                }
+
+                // Delete Realm in ROS
+                Debugger.Break();
+
+                Exception ex = null;
+                Session.Error += (s, e) =>
+                {
+                    ex = e.Exception;
+                };
+
+                using (var realm = Realm.GetInstance(config))
+                {
+                    await Task.Delay(100);
+                }
+
+                Assert.That(ex, Is.InstanceOf<ClientResetException>());
+            });
+        }
+
         private static void AddDummyData(Realm realm, bool singleTransaction)
         {
             Action<Action> write;

wrappers/dependencies.list

@@ -1,3 +1,3 @@
-REALM_CORE_VERSION=5.23.1
-REALM_SYNC_VERSION=4.7.1
+REALM_CORE_VERSION=5.23.3
+REALM_SYNC_VERSION=4.7.5
 ANDROID_OPENSSL_VERSION=1.0.2k

wrappers/src/sync_manager_cs.cpp

@@ -97,6 +97,7 @@ Realm::Config get_shared_realm_config(Configuration configuration, SyncConfigura
     config.sync_config = std::make_shared<SyncConfig>(*sync_configuration.user, realm_url);
     config.sync_config->bind_session_handler = bind_session;
     config.sync_config->error_handler = handle_session_error;
+    config.sync_config->client_resync_mode = sync_configuration.client_resync_mode;
     config.path = Utf16StringAccessor(configuration.path, configuration.path_len);
 
     // by definition the key is only allowed to be 64 bytes long, enforced by C# code

wrappers/src/sync_manager_cs.hpp

@@ -19,6 +19,9 @@
 #pragma once
 
 #include "realm_export_decls.hpp"
+#include "sync/sync_config.hpp"
+
+using namespace realm;
 
 namespace realm {
     class SyncUser;
@@ -38,9 +41,6 @@ namespace realm {
         bool is_partial;
         uint16_t* partial_sync_identifier;
         size_t partial_sync_identifier_len;
+        realm::ClientResyncMode client_resync_mode;
     };
-
-    namespace binding {
-        REALM_EXPORT bool has_feature(StringData feature);
-    }
 }