Consistent ordering of entries in jsondb

Resolves openhab#2436.

Signed-off-by: Sami Salonen <ssalonen@gmail.com>

bundles/org.openhab.core.automation/src/main/java/org/openhab/core/automation/internal/parser/gson/AbstractGSONParser.java

@@ -13,6 +13,7 @@
 package org.openhab.core.automation.internal.parser.gson;
 
 import java.io.OutputStreamWriter;
+import java.util.Map;
 import java.util.Set;
 
 import org.eclipse.jdt.annotation.NonNullByDefault;
@@ -23,6 +24,8 @@
 import org.openhab.core.config.core.Configuration;
 import org.openhab.core.config.core.ConfigurationDeserializer;
 import org.openhab.core.config.core.ConfigurationSerializer;
+import org.openhab.core.config.core.OrderingMapSerializer;
+import org.openhab.core.config.core.OrderingSetSerializer;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -32,6 +35,7 @@
  *
  * @author Kai Kreuzer - Initial contribution
  * @author Ana Dimova - add Instance Creators
+ * @author Sami Salonen - add sorting for maps and sets for minimal diffs
  *
  * @param <T> the type of the entities to parse
  */
@@ -45,6 +49,8 @@ public abstract class AbstractGSONParser<T> implements Parser<T> {
             .registerTypeAdapter(CompositeTriggerType.class, new TriggerInstanceCreator()) //
             .registerTypeAdapter(Configuration.class, new ConfigurationDeserializer()) //
             .registerTypeAdapter(Configuration.class, new ConfigurationSerializer()) //
+            .registerTypeAdapter(Map.class, new OrderingMapSerializer()) //
+            .registerTypeAdapter(Set.class, new OrderingSetSerializer()) //
             .create();
 
     @Override

bundles/org.openhab.core.config.core/src/main/java/org/openhab/core/config/core/ConfigurationSerializer.java

@@ -28,14 +28,15 @@
  *
  * @author Yordan Mihaylov - Initial contribution
  * @author Ana Dimova - provide serialization of multiple configuration values.
+ * @author Sami Salonen - property names are sorted for serialization for minimal diffs
  */
 public class ConfigurationSerializer implements JsonSerializer<Configuration> {
 
     @SuppressWarnings("unchecked")
     @Override
     public JsonElement serialize(Configuration src, Type typeOfSrc, JsonSerializationContext context) {
         JsonObject result = new JsonObject();
-        for (String propName : src.keySet()) {
+        src.keySet().stream().sorted().forEachOrdered((String propName) -> {
             Object value = src.get(propName);
             if (value instanceof List) {
                 JsonArray array = new JsonArray();
@@ -46,7 +47,7 @@ public JsonElement serialize(Configuration src, Type typeOfSrc, JsonSerializatio
             } else {
                 result.add(propName, serializePrimitive(value));
             }
-        }
+        });
         return result;
     }
 

bundles/org.openhab.core.config.core/src/main/java/org/openhab/core/config/core/OrderingMapSerializer.java

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2010-2021 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.config.core;
+
+import java.lang.reflect.Type;
+import java.util.Comparator;
+import java.util.Map;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+/**
+ * Serializes map data by ordering the keys
+ *
+ * @author Sami Salonen - Initial contribution
+ */
+@NonNullByDefault
+public class OrderingMapSerializer implements JsonSerializer<Map<String, Object>> {
+
+    @Override
+    public JsonElement serialize(Map<String, Object> src, Type typeOfSrc, JsonSerializationContext context) {
+        JsonObject ordered = new JsonObject();
+        src.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getKey)).forEachOrdered(entry -> {
+            ordered.add(entry.getKey(), context.serialize(entry.getValue()));
+        });
+        return ordered;
+    }
+}

bundles/org.openhab.core.config.core/src/main/java/org/openhab/core/config/core/OrderingSetSerializer.java

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2010-2021 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.config.core;
+
+import java.lang.reflect.Type;
+import java.util.Set;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+/**
+ * Serializes set by ordering the elements
+ *
+ * @author Sami Salonen - Initial contribution
+ */
+@NonNullByDefault
+public class OrderingSetSerializer implements JsonSerializer<Set<Object>> {
+
+    @Override
+    public JsonElement serialize(Set<Object> src, Type typeOfSrc, JsonSerializationContext context) {
+        JsonArray ordered = new JsonArray();
+        src.stream().map(context::serialize).sorted().forEachOrdered(ordered::add);
+        return ordered;
+    }
+}

bundles/org.openhab.core.storage.json/src/main/java/org/openhab/core/storage/json/internal/JsonStorage.java

@@ -21,6 +21,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
@@ -29,6 +30,8 @@
 import org.eclipse.jdt.annotation.Nullable;
 import org.openhab.core.config.core.Configuration;
 import org.openhab.core.config.core.ConfigurationDeserializer;
+import org.openhab.core.config.core.OrderingMapSerializer;
+import org.openhab.core.config.core.OrderingSetSerializer;
 import org.openhab.core.storage.Storage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,6 +54,8 @@
  * @author Stefan Triller - Removed dependency to internal GSon packages
  * @author Simon Kaufmann - Distinguish between inner and outer
  *         de-/serialization, keep json structures in map
+ * @author Sami Salonen - ordered inner and outer serialization of Maps,
+ *         Sets and properties of Configuration
  */
 @NonNullByDefault
 public class JsonStorage<T> implements Storage<T> {
@@ -88,11 +93,19 @@ public JsonStorage(File file, @Nullable ClassLoader classLoader, int maxBackupFi
         this.writeDelay = writeDelay;
         this.maxDeferredPeriod = maxDeferredPeriod;
 
-        this.internalMapper = new GsonBuilder()
-                .registerTypeHierarchyAdapter(Map.class, new StorageEntryMapDeserializer()).setPrettyPrinting()
+        this.internalMapper = new GsonBuilder() //
+                .registerTypeHierarchyAdapter(Map.class, new OrderingMapSerializer()) //
+                .registerTypeAdapter(Set.class, new OrderingSetSerializer()) //
+                .registerTypeHierarchyAdapter(Configuration.class, new OrderingConfigurationSerializer()) //
+                .registerTypeHierarchyAdapter(Map.class, new StorageEntryMapDeserializer()) //
+                .setPrettyPrinting() //
+                .create();
+        this.entityMapper = new GsonBuilder() //
+                .registerTypeHierarchyAdapter(Map.class, new OrderingMapSerializer()) //
+                .registerTypeAdapter(Set.class, new OrderingSetSerializer()) //
+                .registerTypeAdapter(Configuration.class, new ConfigurationDeserializer()) //
+                .setPrettyPrinting() //
                 .create();
-        this.entityMapper = new GsonBuilder().registerTypeAdapter(Configuration.class, new ConfigurationDeserializer())
-                .setPrettyPrinting().create();
 
         commitTimer = new Timer();
 

bundles/org.openhab.core.storage.json/src/main/java/org/openhab/core/storage/json/internal/OrderingConfigurationSerializer.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2010-2021 Contributors to the openHAB project
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.openhab.core.storage.json.internal;
+
+import java.lang.reflect.Type;
+import java.util.Comparator;
+import java.util.Map;
+
+import org.eclipse.jdt.annotation.NonNullByDefault;
+import org.openhab.core.config.core.Configuration;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import com.google.gson.internal.LinkedTreeMap;
+
+/**
+ * Serializes Configuration object with properties ordered
+ *
+ * @author Sami Salonen - Initial contribution
+ */
+@NonNullByDefault
+public class OrderingConfigurationSerializer implements JsonSerializer<Configuration> {
+
+    @Override
+    public JsonElement serialize(Configuration src, Type typeOfSrc, JsonSerializationContext context) {
+        Map<String, Object> orderedProperties = new LinkedTreeMap<>();
+        src.getProperties().entrySet().stream().sorted(Comparator.comparing(Map.Entry::getKey))
+                .forEachOrdered(entry -> {
+                    orderedProperties.put(entry.getKey(), entry.getValue());
+                });
+
+        return context.serialize(new Configuration(orderedProperties)); // XXX: re-normalizes unnecessarily
+    }
+}

bundles/org.openhab.core.storage.json/src/test/java/org/openhab/core/storage/json/internal/JsonStorageTest.java

@@ -12,6 +12,7 @@
  */
 package org.openhab.core.storage.json.internal;
 
+import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.File;
@@ -26,10 +27,15 @@
 import org.openhab.core.config.core.Configuration;
 import org.openhab.core.test.java.JavaTest;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
+
 /**
  * This test makes sure that the JsonStorage loads all stored numbers as BigDecimal
  *
  * @author Stefan Triller - Initial contribution
+ * @author Samie Salonen - test for ensuring ordering of keys in json
  */
 public class JsonStorageTest extends JavaTest {
 
@@ -129,6 +135,48 @@ public void testStableOutput() throws IOException {
         assertEquals(storageString1, storageString2);
     }
 
+    @SuppressWarnings("null")
+    @Test
+    public void testOrdering() throws IOException {
+        objectStorage.put("DummyObject", new DummyObject());
+        {
+            objectStorage.put("a", new DummyObject());
+            objectStorage.put("b", new DummyObject());
+            persistAndReadAgain();
+        }
+        String storageStringAB = Files.readString(tmpFile.toPath());
+
+        {
+            objectStorage.remove("a");
+            objectStorage.remove("b");
+            objectStorage.put("b", new DummyObject());
+            objectStorage.put("a", new DummyObject());
+            persistAndReadAgain();
+        }
+        String storageStringBA = Files.readString(tmpFile.toPath());
+        assertEquals(storageStringAB, storageStringBA);
+
+        {
+            objectStorage = new JsonStorage<>(tmpFile, this.getClass().getClassLoader(), 0, 0, 0);
+            objectStorage.flush();
+        }
+        String storageStringReserialized = Files.readString(tmpFile.toPath());
+        assertEquals(storageStringAB, storageStringReserialized);
+        Gson gson = new GsonBuilder().create();
+
+        // Parse json. Gson preserves json object key ordering when we parse only JsonObject
+        JsonObject orderedMap = gson.fromJson(storageStringAB, JsonObject.class);
+        // Assert ordering of top level keys (uppercase first in alphabetical order, then lowercase items in
+        // alphabetical order)
+        assertArrayEquals(new String[] { "DummyObject", "a", "b" }, orderedMap.keySet().toArray());
+        // Ordering is ensured also for sub-keys of Configuration object
+        assertArrayEquals(
+                new String[] { "multiInt", "testBigDecimal", "testBoolean", "testDouble", "testFloat", "testInt",
+                        "testLong", "testShort", "testString" },
+                orderedMap.getAsJsonObject("DummyObject").getAsJsonObject("value").getAsJsonObject("configuration")
+                        .getAsJsonObject("properties").keySet().toArray());
+    }
+
     private static class DummyObject {
 
         private final Configuration configuration = new Configuration();