Skip to content

Commit

Permalink
Consistent ordering of entries in jsondb (openhab#2437)
Browse files Browse the repository at this point in the history
* Consistent ordering of entries in jsondb

Resolves openhab#2436.

Signed-off-by: Sami Salonen <ssalonen@gmail.com>
  • Loading branch information
ssalonen authored and ghys committed Sep 9, 2021
1 parent efdb184 commit aa1da69
Show file tree
Hide file tree
Showing 6 changed files with 260 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
package org.openhab.core.automation.internal.parser.gson;

import java.io.OutputStreamWriter;
import java.util.Map;
import java.util.Set;

import org.eclipse.jdt.annotation.NonNullByDefault;
Expand All @@ -23,6 +24,8 @@
import org.openhab.core.config.core.Configuration;
import org.openhab.core.config.core.ConfigurationDeserializer;
import org.openhab.core.config.core.ConfigurationSerializer;
import org.openhab.core.config.core.OrderingMapSerializer;
import org.openhab.core.config.core.OrderingSetSerializer;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
Expand All @@ -32,6 +35,7 @@
*
* @author Kai Kreuzer - Initial contribution
* @author Ana Dimova - add Instance Creators
* @author Sami Salonen - add sorting for maps and sets for minimal diffs
*
* @param <T> the type of the entities to parse
*/
Expand All @@ -45,6 +49,8 @@ public abstract class AbstractGSONParser<T> implements Parser<T> {
.registerTypeAdapter(CompositeTriggerType.class, new TriggerInstanceCreator()) //
.registerTypeAdapter(Configuration.class, new ConfigurationDeserializer()) //
.registerTypeAdapter(Configuration.class, new ConfigurationSerializer()) //
.registerTypeHierarchyAdapter(Map.class, new OrderingMapSerializer()) //
.registerTypeHierarchyAdapter(Set.class, new OrderingSetSerializer()) //
.create();

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,15 @@
*
* @author Yordan Mihaylov - Initial contribution
* @author Ana Dimova - provide serialization of multiple configuration values.
* @author Sami Salonen - property names are sorted for serialization for minimal diffs
*/
public class ConfigurationSerializer implements JsonSerializer<Configuration> {

@SuppressWarnings("unchecked")
@Override
public JsonElement serialize(Configuration src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject result = new JsonObject();
for (String propName : src.keySet()) {
src.keySet().stream().sorted().forEachOrdered((String propName) -> {
Object value = src.get(propName);
if (value instanceof List) {
JsonArray array = new JsonArray();
Expand All @@ -46,7 +47,7 @@ public JsonElement serialize(Configuration src, Type typeOfSrc, JsonSerializatio
} else {
result.add(propName, serializePrimitive(value));
}
}
});
return result;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/**
* Copyright (c) 2010-2021 Contributors to the openHAB project
*
* See the NOTICE file(s) distributed with this work for additional
* information.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.core.config.core;

import java.lang.reflect.Type;
import java.util.Comparator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Stream;

import org.eclipse.jdt.annotation.NonNullByDefault;
import org.eclipse.jdt.annotation.Nullable;

import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;

/**
* Serializes map data by ordering the keys
*
* @author Sami Salonen - Initial contribution
*/
@NonNullByDefault
public class OrderingMapSerializer implements JsonSerializer<Map<@Nullable Object, @Nullable Object>> {

@SuppressWarnings({ "rawtypes", "unchecked", "null" })
@Override
public JsonElement serialize(Map<@Nullable Object, @Nullable Object> src, Type typeOfSrc,
JsonSerializationContext context) {
JsonObject ordered = new JsonObject();
final Stream<Entry<@Nullable Object, @Nullable Object>> possiblySortedStream;
if (OrderingSetSerializer.allSameClassAndComparable(src.keySet())) {
// Map keys are comparable as verified above so casting to plain Comparator is safe
possiblySortedStream = src.entrySet().stream().sorted((Comparator) Map.Entry.comparingByKey());
} else {
possiblySortedStream = src.entrySet().stream();
}
possiblySortedStream.forEachOrdered(entry -> {
Object key = entry.getKey();
if (key instanceof String) {
ordered.add((String) key, context.serialize(entry.getValue()));
} else {
JsonElement serialized = context.serialize(key);
ordered.add(serialized.isJsonPrimitive() ? serialized.getAsString() : serialized.toString(),
context.serialize(entry.getValue()));
}

});
return ordered;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/**
* Copyright (c) 2010-2021 Contributors to the openHAB project
*
* See the NOTICE file(s) distributed with this work for additional
* information.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.core.config.core;

import java.lang.reflect.Type;
import java.util.Set;
import java.util.stream.Stream;

import org.eclipse.jdt.annotation.NonNullByDefault;
import org.eclipse.jdt.annotation.Nullable;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;

/**
* Serializes set by ordering the elements
*
* @author Sami Salonen - Initial contribution
*/
@NonNullByDefault
public class OrderingSetSerializer implements JsonSerializer<Set<@Nullable Object>> {

public static boolean allSameClassAndComparable(Set<@Nullable Object> src) {
Class<?> expectedClass = null;
for (Object object : src) {
if (!(object instanceof Comparable<?>)) {
// not comparable or simply null
return false;
} else if (expectedClass == null) {
// first item
expectedClass = object.getClass();
} else if (!object.getClass().equals(expectedClass)) {
// various classes in the Set, let's not try to sort
return false;
}
}
return true;
}

@Override
public JsonElement serialize(Set<@Nullable Object> src, Type typeOfSrc, JsonSerializationContext context) {
JsonArray ordered = new JsonArray();
final Stream<@Nullable Object> possiblySortedStream;
if (allSameClassAndComparable(src)) {
possiblySortedStream = src.stream().sorted();
} else {
possiblySortedStream = src.stream();
}
possiblySortedStream.map(context::serialize).forEachOrdered(ordered::add);
return ordered;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
Expand All @@ -29,6 +30,8 @@
import org.eclipse.jdt.annotation.Nullable;
import org.openhab.core.config.core.Configuration;
import org.openhab.core.config.core.ConfigurationDeserializer;
import org.openhab.core.config.core.OrderingMapSerializer;
import org.openhab.core.config.core.OrderingSetSerializer;
import org.openhab.core.storage.Storage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -51,6 +54,8 @@
* @author Stefan Triller - Removed dependency to internal GSon packages
* @author Simon Kaufmann - Distinguish between inner and outer
* de-/serialization, keep json structures in map
* @author Sami Salonen - ordered inner and outer serialization of Maps,
* Sets and properties of Configuration
*/
@NonNullByDefault
public class JsonStorage<T> implements Storage<T> {
Expand Down Expand Up @@ -88,11 +93,18 @@ public JsonStorage(File file, @Nullable ClassLoader classLoader, int maxBackupFi
this.writeDelay = writeDelay;
this.maxDeferredPeriod = maxDeferredPeriod;

this.internalMapper = new GsonBuilder()
.registerTypeHierarchyAdapter(Map.class, new StorageEntryMapDeserializer()).setPrettyPrinting()
this.internalMapper = new GsonBuilder() //
.registerTypeHierarchyAdapter(Map.class, new OrderingMapSerializer())//
.registerTypeHierarchyAdapter(Set.class, new OrderingSetSerializer())//
.registerTypeHierarchyAdapter(Map.class, new StorageEntryMapDeserializer()) //
.setPrettyPrinting() //
.create();
this.entityMapper = new GsonBuilder() //
.registerTypeHierarchyAdapter(Map.class, new OrderingMapSerializer())//
.registerTypeHierarchyAdapter(Set.class, new OrderingSetSerializer())//
.registerTypeAdapter(Configuration.class, new ConfigurationDeserializer()) //
.setPrettyPrinting() //
.create();
this.entityMapper = new GsonBuilder().registerTypeAdapter(Configuration.class, new ConfigurationDeserializer())
.setPrettyPrinting().create();

commitTimer = new Timer();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,38 @@
*/
package org.openhab.core.storage.json.internal;

import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.*;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.openhab.core.config.core.Configuration;
import org.openhab.core.test.java.JavaTest;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;

/**
* This test makes sure that the JsonStorage loads all stored numbers as BigDecimal
*
* @author Stefan Triller - Initial contribution
* @author Samie Salonen - test for ensuring ordering of keys in json
*/
public class JsonStorageTest extends JavaTest {

Expand Down Expand Up @@ -129,8 +143,82 @@ public void testStableOutput() throws IOException {
assertEquals(storageString1, storageString2);
}

@SuppressWarnings({ "null", "unchecked" })
@Test
public void testOrdering() throws IOException {
objectStorage.put("DummyObject", new DummyObject());
{
objectStorage.put("a", new DummyObject());
objectStorage.put("b", new DummyObject());
persistAndReadAgain();
}
String storageStringAB = Files.readString(tmpFile.toPath());

{
objectStorage.remove("a");
objectStorage.remove("b");
objectStorage.put("b", new DummyObject());
objectStorage.put("a", new DummyObject());
persistAndReadAgain();
}
String storageStringBA = Files.readString(tmpFile.toPath());
assertEquals(storageStringAB, storageStringBA);

{
objectStorage = new JsonStorage<>(tmpFile, this.getClass().getClassLoader(), 0, 0, 0);
objectStorage.flush();
}
String storageStringReserialized = Files.readString(tmpFile.toPath());
assertEquals(storageStringAB, storageStringReserialized);
Gson gson = new GsonBuilder().create();

// Parse json. Gson preserves json object key ordering when we parse only JsonObject
JsonObject orderedMap = gson.fromJson(storageStringAB, JsonObject.class);
// Assert ordering of top level keys (uppercase first in alphabetical order, then lowercase items in
// alphabetical order)
assertArrayEquals(new String[] { "DummyObject", "a", "b" }, orderedMap.keySet().toArray());
// Ordering is ensured also for sub-keys of Configuration object
assertArrayEquals(
new String[] { "multiInt", "testBigDecimal", "testBoolean", "testDouble", "testFloat", "testInt",
"testLong", "testShort", "testString" },
orderedMap.getAsJsonObject("DummyObject").getAsJsonObject("value").getAsJsonObject("configuration")
.getAsJsonObject("properties").keySet().toArray());
// Set having non-comparable items remains unordered
assertArrayEquals(
new String[] { "http://www.example.com/key2", "http://www.example.com/key1",
"http://www.example.com/key3" },
gson.fromJson(orderedMap.getAsJsonObject("DummyObject").getAsJsonObject("value")
.getAsJsonArray("innerSetWithNonComparableElements"), LinkedList.class).toArray());
// ...while Set having all Comparable keys is ordered:
assertArrayEquals(new Object[] { -5, 0, 3, 50 },
((LinkedList<Integer>) gson.fromJson(
orderedMap.getAsJsonObject("DummyObject").getAsJsonObject("value")
.getAsJsonArray("innerSetWithComparableElements"),
TypeToken.getParameterized(LinkedList.class, Integer.class).getType())).toArray());

// Map having non-comparable items remains unordered
assertArrayEquals(
new String[] { "http://www.example.com/key2", "http://www.example.com/key1",
"http://www.example.com/key3" },
gson.fromJson(orderedMap.getAsJsonObject("DummyObject").getAsJsonObject("value")
.getAsJsonObject("innerMapWithNonComparableKeys"), LinkedHashMap.class).keySet().toArray());
// ...while Map with Comparable keys is ordered
assertArrayEquals(new Integer[] { -5, 0, 3, 50 },
((LinkedHashMap<Integer, Object>) gson.fromJson(
orderedMap.getAsJsonObject("DummyObject").getAsJsonObject("value")
.getAsJsonObject("innerMapWithComparableKeys"),
TypeToken.getParameterized(LinkedHashMap.class, Integer.class, Object.class).getType()))
.keySet().toArray());
}

private static class DummyObject {

// For the test here we use Linked variants of Map and Set which preserve the insertion order
// In tests we verify that collections having Comparable items (keys) are ordered on serialization
private final Map<URL, Object> innerMapWithNonComparableKeys = new LinkedHashMap<>();
private final Map<Integer, Object> innerMapWithComparableKeys = new LinkedHashMap<>();
private final Set<URL> innerSetWithNonComparableElements = new LinkedHashSet<>();
private final Set<Integer> innerSetWithComparableElements = new LinkedHashSet<>();
private final Configuration configuration = new Configuration();
public List<InnerObject> channels = new ArrayList<>();

Expand All @@ -148,6 +236,27 @@ public DummyObject() {
InnerObject inner = new InnerObject();
inner.configuration.put("testChildLong", Long.valueOf("12"));
channels.add(inner);
innerMapWithComparableKeys.put(3, 1);
innerMapWithComparableKeys.put(0, 2);
innerMapWithComparableKeys.put(50, 3);
innerMapWithComparableKeys.put(-5, 4);

innerSetWithComparableElements.add(3);
innerSetWithComparableElements.add(0);
innerSetWithComparableElements.add(50);
innerSetWithComparableElements.add(-5);

try {
innerMapWithNonComparableKeys.put(new URL("http://www.example.com/key2"), 1);
innerMapWithNonComparableKeys.put(new URL("http://www.example.com/key1"), 2);
innerMapWithNonComparableKeys.put(new URL("http://www.example.com/key3"), 3);

innerSetWithNonComparableElements.add(new URL("http://www.example.com/key2"));
innerSetWithNonComparableElements.add(new URL("http://www.example.com/key1"));
innerSetWithNonComparableElements.add(new URL("http://www.example.com/key3"));
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
}

Expand Down

0 comments on commit aa1da69

Please sign in to comment.