Load encoded values from disk #2542
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| /* | ||
| * Licensed to GraphHopper GmbH under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with this work for | ||
| * additional information regarding copyright ownership. | ||
| * | ||
| * GraphHopper GmbH licenses this file to you under the Apache License, | ||
| * Version 2.0 (the "License"); you may not use this file except in | ||
| * compliance with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.graphhopper.routing.ev; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonAutoDetect; | ||
| import com.fasterxml.jackson.annotation.PropertyAccessor; | ||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.node.ObjectNode; | ||
|
|
||
| public class EncodedValueSerializer { | ||
| private final static ObjectMapper MAPPER = new ObjectMapper(); | ||
|
|
||
| static { | ||
| MAPPER.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE); | ||
| MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); | ||
| } | ||
|
|
||
| public static String serializeEncodedValue(EncodedValue encodedValue) { | ||
| try { | ||
| JsonNode tree = MAPPER.valueToTree(encodedValue); | ||
| ((ObjectNode) tree).put("version", encodedValue.getVersion()); | ||
| return MAPPER.writeValueAsString(tree); | ||
| } catch (JsonProcessingException e) { | ||
| throw new IllegalStateException("Could not serialize encoded value: " + encodedValue + ", error: " + e.getMessage()); | ||
| } | ||
| } | ||
|
|
||
| public static EncodedValue deserializeEncodedValue(String serializedEncodedValue) { | ||
| try { | ||
| JsonNode jsonNode = MAPPER.readTree(serializedEncodedValue); | ||
| int storedVersion = jsonNode.get("version").asInt(); | ||
| ((ObjectNode) jsonNode).remove("version"); | ||
| EncodedValue encodedValue = MAPPER.treeToValue(jsonNode, EncodedValue.class); | ||
| if (storedVersion != encodedValue.getVersion()) | ||
| throw new IllegalStateException("Version does not match. Cannot properly read encoded value: " + encodedValue.getName() + ". " + | ||
| "You need to use the same version of GraphHopper you used to import the data"); | ||
|
Comment on lines
+51
to
+56
There was a problem hiding this comment. This is a bit self-made and probably can also be done using Jackson Annotations, but since we aren't really planning to de/serialize encoded values with something other than these two methods I think this is fine. I'm not so experienced with Jackson and open for improvements of course. There was a problem hiding this comment. I'm also no expert. I tried to get it working and embedded your work into a common Deserializer (and using But the problem was that we don't want to manually handle the className and I wasn't able to find a way to use both: The only solution was with some wrapper class but this here looks simpler IMO. |
||
| return encodedValue; | ||
| } catch (JsonProcessingException e) { | ||
| throw new IllegalStateException("Could not deserialize encoded value: " + serializedEncodedValue + ", error: " + e.getMessage()); | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,10 @@ | ||
| package com.graphhopper.routing.ev; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonCreator; | ||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import com.graphhopper.storage.IntsRef; | ||
|
|
||
| import java.util.*; | ||
|
|
||
| /** | ||
| * This class holds a List of up to {@link #maxValues} encountered Strings and stores | ||
|
|
@@ -19,7 +16,7 @@ | |
| public final class StringEncodedValue extends IntEncodedValueImpl { | ||
| private final int maxValues; | ||
| private final List<String> values; | ||
| private final Map<String, Integer> indexMap; | ||
|
|
||
| public StringEncodedValue(String name, int expectedValueCount) { | ||
| this(name, expectedValueCount, false); | ||
|
|
@@ -30,7 +27,7 @@ public StringEncodedValue(String name, int expectedValueCount, boolean storeTwoD | |
|
|
||
| this.maxValues = roundUp(expectedValueCount); | ||
| this.values = new ArrayList<>(maxValues); | ||
| this.indexMap = new HashMap<>(maxValues); | ||
|
There was a problem hiding this comment. Could we avoid this with a dedicated getter instead of using field access? I'm not sure if we want to deal with auto(un)boxing in a potentially hot code path. There was a problem hiding this comment. Does this really not work because we use field access? I thought we need a custom serializer for hppc, but I might be wrong. I thought about the auto unboxing for a moment but then thought it's probably not as bad bc the maps will be very small. But you are more than welcome to make this work with the hppc map again. I just did not feel like spending a lot more time on this, also because currently the default GraphHopper server does not use StringEncodedValue currently. There was a problem hiding this comment. I tried changing There was a problem hiding this comment. It works when we add |
||
| } | ||
|
|
||
| public StringEncodedValue(String name, int bits, List<String> values, boolean storeTwoDirections) { | ||
|
|
@@ -42,13 +39,33 @@ public StringEncodedValue(String name, int bits, List<String> values, boolean st | |
| + values.size() + " > " + maxValues); | ||
|
|
||
| this.values = new ArrayList<>(values); | ||
| this.indexMap = new HashMap<>(values.size()); | ||
| int index = 1; | ||
| for (String value : values) { | ||
| indexMap.put(value, index++); | ||
| } | ||
| } | ||
|
|
||
| @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) | ||
| public StringEncodedValue( | ||
| @JsonProperty("name") String name, | ||
| @JsonProperty("bits") int bits, | ||
| @JsonProperty("minValue") int minValue, | ||
| @JsonProperty("maxValue") int maxValue, | ||
| @JsonProperty("negateReverseDirection") boolean negateReverseDirection, | ||
| @JsonProperty("storeTwoDirections") boolean storeTwoDirections, | ||
| @JsonProperty("maxValues") int maxValues, | ||
| @JsonProperty("values") List<String> values, | ||
| @JsonProperty("indexMap") HashMap<String, Integer> indexMap) { | ||
| super(name, bits, minValue, maxValue, negateReverseDirection, storeTwoDirections); | ||
| if (values.size() > maxValues) | ||
| throw new IllegalArgumentException("Number of values is higher than the maximum value count: " | ||
| + values.size() + " > " + maxValues); | ||
| this.maxValues = maxValues; | ||
| this.values = values; | ||
| this.indexMap = indexMap; | ||
| } | ||
|
|
||
| public final void setString(boolean reverse, IntsRef ref, String value) { | ||
| if (value == null) { | ||
| super.setInt(reverse, ref, 0); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
for consistency shall we add
MAPPER.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE);
here?