Skip to content

Commit

Permalink
speedup of the ratio computation
Browse files Browse the repository at this point in the history 
comparing both tag arrays (keys(2)&values(2)), if same -> used in initial filter
  • Loading branch information
kowatsch committed May 23, 2018
1 parent 66da3bf commit 31ecae1
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 118 deletions.
150 changes: 32 additions & 118 deletions ...g/heigit/bigspatialdata/ohsome/ohsomeApi/controller/executor/ElementsRequestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,105 +558,6 @@ public static GroupByResponse executeCountPerimeterAreaGroupByType(
return response;
}

/**
* Performs a count-ratio calculation grouped by the boundary.
* <p>
* The other parameters are described in the
* {@link org.heigit.bigspatialdata.ohsome.ohsomeApi.controller.dataAggregation.CountController#getCountRatio(String, String, String, String[], String[], String[], String[], String[], String, String[], String[], String[])
* getCountRatio} method.
*
* @param rPs <code>RequestParameters</code> object, which holds those parameters that are used in
* every request.
* @return {@link org.heigit.bigspatialdata.ohsome.ohsomeApi.output.dataAggregationResponse.groupByResponse.RatioGroupByBoundaryResponse
* RatioGroupByBoundaryResponse Content}
*/
public static RatioGroupByBoundaryResponse executeCountRatioGroupByBoundary(RequestParameters rPs,
String[] types2, String[] keys2, String[] values2)
throws UnsupportedOperationException, Exception {

long startTime = System.currentTimeMillis();
ExecutionUtils exeUtils = new ExecutionUtils();
SortedMap<OSHDBTimestampAndIndex<Integer>, ? extends Number> result1;
SortedMap<OSHDBTimestampAndIndex<Integer>, ? extends Number> result2;
MapReducer<OSMEntitySnapshot> mapRed1 = null;
MapReducer<OSMEntitySnapshot> mapRed2 = null;
SortedMap<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> groupByResult1;
SortedMap<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> groupByResult2;
InputProcessor iP = new InputProcessor();
String requestURL = null;
DecimalFormat ratioDf = exeUtils.defineDecimalFormat("#.######");
RequestParameters rPs2 = new RequestParameters(rPs.isPost(), rPs.isSnapshot(), rPs.isDensity(),
rPs.getBboxes(), rPs.getBcircles(), rPs.getBpolys(), types2, keys2, values2,
rPs.getUserids(), rPs.getTime(), rPs.getShowMetadata());
if (!rPs.isPost())
requestURL = RequestInterceptor.requestUrl;
mapRed1 = iP.processParameters(mapRed1, rPs);
result1 = exeUtils.computeCountLengthPerimeterAreaGBB(RequestResource.COUNT,
iP.getBoundaryType(), mapRed1, iP.getGeomBuilder(), rPs.isSnapshot());
mapRed2 = iP.processParameters(mapRed2, rPs2);
result2 = exeUtils.computeCountLengthPerimeterAreaGBB(RequestResource.COUNT,
iP.getBoundaryType(), mapRed2, iP.getGeomBuilder(), rPs.isSnapshot());
groupByResult1 = MapAggregatorByTimestampAndIndex.nest_IndexThenTime(result1);
groupByResult2 = MapAggregatorByTimestampAndIndex.nest_IndexThenTime(result2);
GroupByResult[] resultSet = new GroupByResult[groupByResult1.size()];
RatioGroupByResult[] ratioResultSet = new RatioGroupByResult[groupByResult1.size()];
String groupByName = "";
Utils utils = iP.getUtils();
String[] boundaryIds = utils.getBoundaryIds();
int count = 0;
int innerCount = 0;
for (Entry<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> entry : groupByResult1
.entrySet()) {
ElementsResult[] results = new ElementsResult[entry.getValue().entrySet().size()];
innerCount = 0;
groupByName = boundaryIds[count];
for (Entry<OSHDBTimestamp, ? extends Number> innerEntry : entry.getValue().entrySet()) {
results[innerCount] =
new ElementsResult(TimestampFormatter.getInstance().isoDateTime(innerEntry.getKey()),
innerEntry.getValue().intValue());
innerCount++;
}
resultSet[count] = new GroupByResult(groupByName, results);
count++;
}
count = 0;
innerCount = 0;
for (Entry<Integer, ? extends SortedMap<OSHDBTimestamp, ? extends Number>> entry : groupByResult2
.entrySet()) {
RatioResult[] ratioResults = new RatioResult[entry.getValue().entrySet().size()];
innerCount = 0;
groupByName = boundaryIds[count];
for (Entry<OSHDBTimestamp, ? extends Number> innerEntry : entry.getValue().entrySet()) {
double value = resultSet[count].getResult()[innerCount].getValue();
double value2 = innerEntry.getValue().doubleValue();
double ratio = value2 / value;
// in case ratio has the values "NaN", "Infinity", etc.
try {
ratio = Double.parseDouble(ratioDf.format(ratio));
} catch (Exception e) {
// do nothing --> just return ratio without rounding (trimming)
}
ratioResults[innerCount] =
new RatioResult(TimestampFormatter.getInstance().isoDateTime(innerEntry.getKey()),
value, value2, ratio);
innerCount++;
}
ratioResultSet[count] = new RatioGroupByResult(groupByName, ratioResults);
count++;
}
Metadata metadata = null;
if (iP.getShowMetadata()) {
long duration = System.currentTimeMillis() - startTime;
metadata = new Metadata(duration,
"Amount of items satisfying types2, keys2, values2 parameters (= value2 output) within items "
+ "selected by types, keys, values parameters (= value output) and ratio of value2:value grouped on the boundary objects.",
requestURL);
}
RatioGroupByBoundaryResponse response = new RatioGroupByBoundaryResponse(
new Attribution(url, text), Application.apiVersion, metadata, ratioResultSet);
return response;
}

/**
* Performs a count|length|perimeter|area calculation grouped by the key.
* <p>
Expand Down Expand Up @@ -1058,9 +959,11 @@ public static RatioResponse executeCountLengthPerimeterAreaRatio(RequestResource
DecimalFormat ratioDf = exeUtils.defineDecimalFormat("#.######");
TagTranslator tt = DbConnData.tagTranslator;
rPs = iP.fillWithEmptyIfNull(rPs);
// for input processing/checking only
iP.processParameters(mapRed, rPs);
iP.checkKeysValues(keys2, values2, false);
values2 = iP.createEmptyArrayIfNull(values2);
keys2 = iP.createEmptyArrayIfNull(keys2);
Integer[] keysInt1 = new Integer[rPs.getKeys().length];
Integer[] valuesInt1 = new Integer[rPs.getValues().length];
Integer[] keysInt2 = new Integer[keys2.length];
Expand All @@ -1083,15 +986,20 @@ public static RatioResponse executeCountLengthPerimeterAreaRatio(RequestResource
osmTypes.addAll(osmTypes2);
String[] osmTypesString =
osmTypes.stream().map(OSMType::toString).map(String::toLowerCase).toArray(String[]::new);
mapRed = iP.processParameters(mapRed,
new RequestParameters(rPs.isPost(), rPs.isSnapshot(), rPs.isDensity(), rPs.getBboxes(),
rPs.getBcircles(), rPs.getBpolys(), osmTypesString, new String[] {}, new String[] {},
rPs.getUserids(), rPs.getTime(), rPs.getShowMetadata()));
mapRed = mapRed.where(entity -> {
boolean matches1 = exeUtils.entityMatches(entity, osmTypes1, keysInt1, valuesInt1);
boolean matches2 = exeUtils.entityMatches(entity, osmTypes2, keysInt2, valuesInt2);
return matches1 || matches2;
});
if (!iP.compareKeysValues(rPs.getKeys(), keys2, rPs.getValues(), values2)) {
mapRed = iP.processParameters(mapRed,
new RequestParameters(rPs.isPost(), rPs.isSnapshot(), rPs.isDensity(), rPs.getBboxes(),
rPs.getBcircles(), rPs.getBpolys(), osmTypesString, new String[] {}, new String[] {},
rPs.getUserids(), rPs.getTime(), rPs.getShowMetadata()));
mapRed = mapRed.where(entity -> {
boolean matches1 = exeUtils.entityMatches(entity, osmTypes1, keysInt1, valuesInt1);
boolean matches2 = exeUtils.entityMatches(entity, osmTypes2, keysInt2, valuesInt2);
return matches1 || matches2;
});
} else {
mapRed = iP.processParameters(mapRed, rPs);
mapRed = mapRed.osmTypes(osmTypes);
}
preResult = mapRed.aggregateByTimestamp().aggregateBy(f -> {
OSMEntity entity = f.getEntity();
boolean matches1 = exeUtils.entityMatches(entity, osmTypes1, keysInt1, valuesInt1);
Expand Down Expand Up @@ -1219,6 +1127,7 @@ public static RatioGroupByBoundaryResponse executeCountLengthPerimeterAreaRatioG
GeometryBuilder geomBuilder = iP.getGeomBuilder();
iP.checkKeysValues(keys2, values2, false);
values2 = iP.createEmptyArrayIfNull(values2);
keys2 = iP.createEmptyArrayIfNull(keys2);
Integer[] keysInt1 = new Integer[rPs.getKeys().length];
Integer[] valuesInt1 = new Integer[rPs.getValues().length];
Integer[] keysInt2 = new Integer[keys2.length];
Expand All @@ -1241,15 +1150,20 @@ public static RatioGroupByBoundaryResponse executeCountLengthPerimeterAreaRatioG
osmTypes.addAll(osmTypes2);
String[] osmTypesString =
osmTypes.stream().map(OSMType::toString).map(String::toLowerCase).toArray(String[]::new);
mapRed = iP.processParameters(mapRed,
new RequestParameters(rPs.isPost(), rPs.isSnapshot(), rPs.isDensity(), rPs.getBboxes(),
rPs.getBcircles(), rPs.getBpolys(), osmTypesString, new String[] {}, new String[] {},
rPs.getUserids(), rPs.getTime(), rPs.getShowMetadata()));
mapRed = mapRed.where(entity -> {
boolean matches1 = exeUtils.entityMatches(entity, osmTypes1, keysInt1, valuesInt1);
boolean matches2 = exeUtils.entityMatches(entity, osmTypes2, keysInt2, valuesInt2);
return matches1 || matches2;
});
if (!iP.compareKeysValues(rPs.getKeys(), keys2, rPs.getValues(), values2)) {
mapRed = iP.processParameters(mapRed,
new RequestParameters(rPs.isPost(), rPs.isSnapshot(), rPs.isDensity(), rPs.getBboxes(),
rPs.getBcircles(), rPs.getBpolys(), osmTypesString, new String[] {}, new String[] {},
rPs.getUserids(), rPs.getTime(), rPs.getShowMetadata()));
mapRed = mapRed.where(entity -> {
boolean matches1 = exeUtils.entityMatches(entity, osmTypes1, keysInt1, valuesInt1);
boolean matches2 = exeUtils.entityMatches(entity, osmTypes2, keysInt2, valuesInt2);
return matches1 || matches2;
});
} else {
mapRed = iP.processParameters(mapRed, rPs);
mapRed = mapRed.osmTypes(osmTypes);
}
Utils utils = iP.getUtils();
ArrayList<Geometry> geoms = geomBuilder.getGeometry(iP.getBoundaryType());
List<Pair<Integer, MatchType>> zeroFill = new LinkedList<>();
Expand Down Expand Up @@ -1345,7 +1259,7 @@ else if (matches2)
}
timeArrayFilled = true;
} else {
// on MatchType.MATCHESNONE aggregated values are not needed
// on MatchType.MATCHESNONE aggregated values are not needed / do not exist
}
if (count % 3 == 0) {
groupByName = boundaryIds[gBNCount];
Expand Down
15 changes: 15 additions & 0 deletions src/main/java/org/heigit/bigspatialdata/ohsome/ohsomeApi/inputProcessing/InputProcessor.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.heigit.bigspatialdata.ohsome.ohsomeApi.inputProcessing;

import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;
Expand Down Expand Up @@ -240,6 +241,20 @@ public void checkKeysValues(String[] keys, String[] values, boolean isShare)
"There cannot be more input values in the values|values2 than in the keys|keys2 parameter, as values_n must fit to keys_n.");
}

/**
* Compares the keys and values arrays with each other. Returns true only if
* keys==keys2&&values==values2.
*/
public boolean compareKeysValues(String[] keys, String[] keys2, String[] values,
String[] values2) {

if (keys.length == keys2.length && values.length == values2.length) {
if (Arrays.equals(keys, keys2) && Arrays.equals(values, values2))
return true;
}
return false;
}

/**
* Checks the given keys and values parameters on their length and includes them in the
* {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer#where(String) where(key)}, or
Expand Down

0 comments on commit 31ecae1

Please sign in to comment.