Enable compact for synchronized Realms (#6002)

CHANGELOG.md

@@ -2,8 +2,15 @@
 
 ### Enhancements
 
+* [ObjectServer] `Realm.compactRealm(config)` now works on synchronized Realms (#5937).
+* [ObjectServer] `SyncConfiguration.compactOnLaunch()` and `SyncConfiguration.compactOnLaunch(callback)` has been added (#5937).
 * Added `RealmQuery.getRealm()`, `RealmResults.getRealm()`, `RealmList.getRealm()` and `OrderedRealmCollectionSnapshot.getRealm()` (#5997).
 
+### Internal
+
+* Upgraded to Realm Core 5.6.0
+* Upgraded to Realm Sync 3.5.2
+
 
 ## 5.2.0 (2018-06-06)
 

Jenkinsfile

@@ -42,7 +42,9 @@ try {
           // Docker image for testing Realm Object Server
           def dependProperties = readProperties file: 'dependencies.list'
           def rosVersion = dependProperties["REALM_OBJECT_SERVER_VERSION"]
-          rosEnv = docker.build 'ros:snapshot', "--build-arg ROS_VERSION=${rosVersion} tools/sync_test_server"
+          withCredentials([string(credentialsId: 'realm-sync-feature-token-enterprise', variable: 'realmFeatureToken')]) {
+            rosEnv = docker.build 'ros:snapshot', "--build-arg ROS_VERSION=${rosVersion} --build-arg REALM_FEATURE_TOKEN=${realmFeatureToken} tools/sync_test_server"
+          }
         }
 
 	    rosContainer = rosEnv.run()

dependencies.list

@@ -1,8 +1,8 @@
 # Realm Sync Core release used by Realm Java
 # https://github.com/realm/realm-sync/releases
-REALM_SYNC_VERSION=3.0.1
-REALM_SYNC_SHA256=7764304d5dc7db7b4b9be9916f753c14c61c40e9f09fd1d92abeee3d8474405f
+REALM_SYNC_VERSION=3.5.2
+REALM_SYNC_SHA256=a056338471770ee915f1bdc3efb69177de18710b1cf98193822520ccf326ba2c
 
 # Object Server Release used by Integration tests. Installed using NPM.
 # Use `npm view realm-object-server versions` to get a list of available versions.
-REALM_OBJECT_SERVER_VERSION=3.1.5
+REALM_OBJECT_SERVER_VERSION=3.6.6

realm/realm-library/src/androidTestObjectServer/java/io/realm/SyncConfigurationTests.java

@@ -441,16 +441,6 @@ public void toString_nonEmpty() {
         assertTrue(configStr != null && !configStr.isEmpty());
     }
 
-    // FIXME: This test can be removed when https://github.com/realm/realm-core/issues/2345 is resolved
-    @Test(expected = UnsupportedOperationException.class)
-    public void compact_NotAllowed() {
-        SyncUser user = createTestUser();
-        String url = "realm://objectserver.realm.io/default";
-        SyncConfiguration config = user.createConfiguration(url).build();
-
-        Realm.compactRealm(config);
-    }
-
     // Check that it is possible for multiple users to reference the same Realm URL while each user still use their
     // own copy on the filesystem. This is e.g. what happens if a Realm is shared using a PermissionOffer.
     @Test

realm/realm-library/src/androidTestObjectServer/java/io/realm/SyncedRealmTests.java

@@ -23,11 +23,20 @@
 import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 
+import java.io.File;
+import java.io.IOException;
+
+import io.realm.entities.AllJavaTypes;
+import io.realm.entities.AllTypes;
+import io.realm.internal.util.Pair;
 import io.realm.objectserver.model.PartialSyncObjectA;
+import io.realm.objectserver.utils.Constants;
 import io.realm.rule.RunInLooperThread;
 import io.realm.rule.RunTestInLooperThread;
 import io.realm.util.SyncTestUtils;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -52,6 +61,9 @@ public void tearDown() {
         if (realm != null && !realm.isClosed()) {
             realm.close();
         }
+        for (SyncUser user : SyncUser.all().values()) {
+            user.logOut();
+        }
     }
 
     private Realm getNormalRealm() {
@@ -196,4 +208,51 @@ public void delete_throws() {
         }
     }
 
+    @Test
+    public void compactRealm_populatedRealm() {
+        SyncConfiguration config = configFactory.createSyncConfigurationBuilder(SyncTestUtils.createTestUser(), Constants.DEFAULT_REALM).build();
+        realm = Realm.getInstance(config);
+        realm.executeTransaction(r -> {
+            for (int i = 0; i < 10; i++) {
+                r.insert(new AllJavaTypes(i));
+            }
+        });
+        realm.close();
+        assertTrue(Realm.compactRealm(config));
+        realm = Realm.getInstance(config);
+        assertEquals(10, realm.where(AllJavaTypes.class).count());
+    }
+
+    @Test
+    public void compactOnLaunch_shouldCompact() throws IOException {
+        SyncUser user = SyncTestUtils.createTestUser();
+
+        // Fill Realm with data and record size
+        SyncConfiguration config1 = configFactory.createSyncConfigurationBuilder(user, Constants.DEFAULT_REALM).build();
+        realm = Realm.getInstance(config1);
+        byte[] oneMBData = new byte[1024 * 1024];
+        realm.beginTransaction();
+        for (int i = 0; i < 10; i++) {
+            realm.createObject(AllTypes.class).setColumnBinary(oneMBData);
+        }
+        realm.commitTransaction();
+        realm.close();
+        long originalSize = new File(realm.getPath()).length();
+
+        // Open Realm with CompactOnLaunch
+        SyncConfiguration config2 = configFactory.createSyncConfigurationBuilder(user, Constants.DEFAULT_REALM)
+                .compactOnLaunch(new CompactOnLaunchCallback() {
+                    @Override
+                    public boolean shouldCompact(long totalBytes, long usedBytes) {
+                        return true;
+                    }
+                })
+                .build();
+        realm = Realm.getInstance(config2);
+        realm.close();
+        long compactedSize = new File(realm.getPath()).length();
+
+        assertTrue(originalSize > compactedSize);
+    }
+
 }

realm/realm-library/src/main/cpp/io_realm_internal_Table.cpp

@@ -308,9 +308,6 @@ JNIEXPORT void JNICALL Java_io_realm_internal_Table_nativeConvertColumnToNullabl
                                                                                   jlong j_column_index,
                                                                                   jboolean)
 {
-#if REALM_ENABLE_SYNC
-    REALM_ASSERT(false);
-#endif
     Table* table = TBL(native_table_ptr);
     if (!TBL_AND_COL_INDEX_VALID(env, table, j_column_index)) {
         return;
@@ -470,9 +467,6 @@ JNIEXPORT void JNICALL Java_io_realm_internal_Table_nativeConvertColumnToNotNull
                                                                                      jlong j_column_index,
                                                                                      jboolean is_primary_key)
 {
-#if REALM_ENABLE_SYNC
-    REALM_ASSERT(false);
-#endif
     try {
         Table* table = TBL(native_table_ptr);
         if (!TBL_AND_COL_INDEX_VALID(env, table, j_column_index)) {

realm/realm-library/src/main/java/io/realm/Realm.java

@@ -1702,13 +1702,8 @@ public static boolean deleteRealm(RealmConfiguration configuration) {
      *
      * @param configuration a {@link RealmConfiguration} pointing to a Realm file.
      * @return {@code true} if successful, {@code false} if any file operation failed.
-     * @throws UnsupportedOperationException if Realm is synchronized.
      */
     public static boolean compactRealm(RealmConfiguration configuration) {
-        // FIXME: remove this restriction when https://github.com/realm/realm-core/issues/2345 is resolved
-        if (configuration.isSyncConfiguration()) {
-            throw new UnsupportedOperationException("Compacting is not supported yet on synced Realms. See https://github.com/realm/realm-core/issues/2345");
-        }
         return BaseRealm.compactRealm(configuration);
     }
 

realm/realm-library/src/objectServer/java/io/realm/SyncConfiguration.java

@@ -143,7 +143,8 @@ private SyncConfiguration(File directory,
                                 String serverCertificateFilePath,
                                 boolean waitForInitialData,
                                 OsRealmConfig.SyncSessionStopPolicy sessionStopPolicy,
-                                boolean isPartial
+                                boolean isPartial,
+                                CompactOnLaunchCallback compactOnLaunch
     ) {
         super(directory,
                 filename,
@@ -158,7 +159,7 @@ private SyncConfiguration(File directory,
                 rxFactory,
                 initialDataTransaction,
                 readOnly,
-                null,
+                compactOnLaunch,
                 false
         );
 
@@ -487,6 +488,8 @@ public static final class Builder  {
         private String serverCertificateFilePath;
         private OsRealmConfig.SyncSessionStopPolicy sessionStopPolicy = OsRealmConfig.SyncSessionStopPolicy.AFTER_CHANGES_UPLOADED;
         private boolean isPartial = true; // Partial Synchronization is enabled by default
+        private CompactOnLaunchCallback compactOnLaunch;
+
         /**
          * Creates an instance of the Builder for the SyncConfiguration. This SyncConfiguration
          * will be for a fully synchronized Realm.
@@ -978,6 +981,31 @@ public SyncConfiguration.Builder fullSynchronization() {
             return this;
         }
 
+        /**
+         * Setting this will cause Realm to compact the Realm file if the Realm file has grown too large and a
+         * significant amount of space can be recovered. See {@link DefaultCompactOnLaunchCallback} for details.
+         */
+        public SyncConfiguration.Builder compactOnLaunch() {
+            return compactOnLaunch(new DefaultCompactOnLaunchCallback());
+        }
+
+        /**
+         * Sets this to determine if the Realm file should be compacted before returned to the user. It is passed the
+         * total file size (data + free space) and the bytes used by data in the file.
+         *
+         * @param compactOnLaunch a callback called when opening a Realm for the first time during the life of a process
+         *                        to determine if it should be compacted before being returned to the user. It is passed
+         *                        the total file size (data + free space) and the bytes used by data in the file.
+         */
+        public SyncConfiguration.Builder compactOnLaunch(CompactOnLaunchCallback compactOnLaunch) {
+            //noinspection ConstantConditions
+            if (compactOnLaunch == null) {
+                throw new IllegalArgumentException("A non-null compactOnLaunch must be provided");
+            }
+            this.compactOnLaunch = compactOnLaunch;
+            return this;
+        }
+
         private String MD5(String in) {
             try {
                 MessageDigest digest = MessageDigest.getInstance("MD5");
@@ -1129,7 +1157,8 @@ public SyncConfiguration build() {
                     serverCertificateFilePath,
                     waitForServerChanges,
                     sessionStopPolicy,
-                    isPartial
+                    isPartial,
+                    compactOnLaunch
             );
         }
 

tools/sync_test_server/Dockerfile

@@ -5,6 +5,9 @@ RUN cp /usr/share/zoneinfo/Europe/Copenhagen /etc/localtime
 RUN echo "Europe/Copenhagen" >  /etc/timezone
 
 ARG ROS_VERSION
+ARG REALM_FEATURE_TOKEN
+RUN if [ "x$ROS_VERSION" = "x" ] ; then echo Non-empty ROS_VERSION required ; exit 1; fi
+RUN if [ "x$REALM_FEATURE_TOKEN" = "x" ] ; then echo Non-empty REALM_FEATURE_TOKEN required ; exit 1; fi
 
 # Install netstat (used for debugging)
 RUN apt-get update \
@@ -18,6 +21,7 @@ RUN apt-get update \
 COPY ros /ros
 WORKDIR "/ros"
 RUN sed -i -e "s/%ROS_VERSION%/$ROS_VERSION/g" package.json
+RUN sed -i -e "s/%REALM_FEATURE_TOKEN%/$REALM_FEATURE_TOKEN/g" src/index.ts
 RUN npm install
 WORKDIR "/"
 

tools/sync_test_server/ros/src/index.ts

@@ -7,6 +7,8 @@ server.start({
         // For all the full list of configuration parameters see:
         // https://realm.io/docs/realm-object-server/latest/api/ros/interfaces/serverconfig.html
 
+        featureToken: '%REALM_FEATURE_TOKEN%',
+
         // This is the location where ROS will store its runtime data
         dataPath: path.join(__dirname, '../data'),
 

tools/sync_test_server/start_server.sh

@@ -1,5 +1,11 @@
 #!/bin/sh
 
+if [ -z "$REALM_FEATURE_TOKEN" ]
+then
+    echo 'The environment variable $REALM_FEATURE_TOKEN was not set'
+    exit 1
+fi
+
 # Get the script dir which contains the Dockerfile
 DOCKERFILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
@@ -11,7 +17,7 @@ adb reverse tcp:9443 tcp:9443 && \
 adb reverse tcp:9080 tcp:9080 && \
 adb reverse tcp:8888 tcp:8888 || { echo "Failed to reverse adb port." ; exit 1 ; }
 
-docker build $DOCKERFILE_DIR --build-arg ROS_VERSION=$ROS_VERSION -t sync-test-server || { echo "Failed to build Docker image." ; exit 1 ; }
+docker build $DOCKERFILE_DIR --build-arg ROS_VERSION=$ROS_VERSION --build-arg REALM_FEATURE_TOKEN=$REALM_FEATURE_TOKEN -t sync-test-server || { echo "Failed to build Docker image." ; exit 1 ; }
 
 echo "See log files in $TMP_DIR"
 docker run -p 9080:9080 -p 9443:9443 -p 8888:8888 -v$TMP_DIR:/tmp --name sync-test-server sync-test-server