Skip to content
Permalink
Browse files

Merge branch 'develop' into bugfix/prefixed-filter-propagation

  • Loading branch information...
bjornandre committed Mar 5, 2019
2 parents 71f335f + cc59a17 commit 30e57d7a51d036aace27d2d59b1061d2ac5a3957
@@ -56,16 +56,10 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>randomizedtesting-runner</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-artifact</artifactId>
<version>LATEST</version>
<version>3.6.0</version>
<scope>test</scope>
</dependency>

@@ -74,6 +68,13 @@
<artifactId>dec</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.7.25</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
@@ -58,6 +58,14 @@ public DataPoint getDataPoint() {
return this.dataPoint;
}

/**
* Update the underlying Datapoint value.
*/
public DataPointMap withDataPoint(DataPoint dataPoint) {
this.dataPoint = dataPoint;
return this;
}

/**
* Update the underlying Datapoint value.
*/
@@ -0,0 +1,53 @@
package no.ssb.vtl.script.operations;

/*
* -
* ========================LICENSE_START=================================
* Java VTL
* %%
* Copyright (C) 2019 Arild Johan Takvam-Borge
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =========================LICENSE_END==================================
*/

import no.ssb.vtl.model.Ordering;
import no.ssb.vtl.model.VTLObject;
import no.ssb.vtl.model.VtlOrdering;

import java.util.Comparator;

public class DataPointMapComparator implements Comparator<DataPointMap> {

private final VtlOrdering vtlOrdering;

@SuppressWarnings("unchecked")
private final static Comparator<VTLObject> vtlObjectComparator = Comparator.comparing((VTLObject object) ->
(Comparable) object.get(), Comparator.nullsLast(Comparator.naturalOrder()));

public DataPointMapComparator(VtlOrdering vtlOrdering) {
this.vtlOrdering = vtlOrdering;
}

@Override
public int compare(DataPointMap o1, DataPointMap o2) {
int result = 0;
for (String column : vtlOrdering.columns()) {
result = vtlObjectComparator.compare(o1.get(column), o2.get(column));
if (result != 0) {
return vtlOrdering.getDirection(column) == Ordering.Direction.ASC ? result : -result;
}
}
return result;
}
}
@@ -33,6 +33,10 @@
import java.util.Map;
import java.util.function.Supplier;

/**
* Use {@link no.ssb.vtl.script.operations.DataPointMap}
*/
@Deprecated
public final class DataPointMap {

private final ImmutableList<String> names;
@@ -81,6 +81,11 @@ public DataPoint apply(DataPoint left, DataPoint right) {
private DataPoint dp = DataPoint.create(0);


/**
* Use {@link no.ssb.vtl.script.operations.DataPointMap} (maybe merge)
*
*/
@Deprecated
public DataPointView(DataStructure structure) {
ImmutableList<String> list = ImmutableSet.copyOf(structure.keySet()).asList();
this.hash = list::indexOf;
@@ -29,7 +29,6 @@
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataPoint;
import no.ssb.vtl.model.DataStructure;
import no.ssb.vtl.model.DatapointNormalizer;
import no.ssb.vtl.model.Dataset;
import no.ssb.vtl.model.Filtering;
import no.ssb.vtl.model.FilteringSpecification;
@@ -38,18 +37,26 @@
import no.ssb.vtl.model.VtlFiltering;
import no.ssb.vtl.model.VtlOrdering;
import no.ssb.vtl.script.operations.AbstractDatasetOperation;
import no.ssb.vtl.script.operations.DataPointMap;
import no.ssb.vtl.script.operations.DataPointMapComparator;
import no.ssb.vtl.script.operations.VtlStream;
import no.ssb.vtl.script.operations.join.DataPointCapacityExpander;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Arrays.asList;
import static no.ssb.vtl.model.DataStructure.Entry;
import static no.ssb.vtl.model.DataStructure.builder;

/**
* Union operator
@@ -71,7 +78,29 @@ public UnionOperation(List<Dataset> datasets) {

@Override
protected DataStructure computeDataStructure() {
return getChildren().get(0).getDataStructure();
if (getChildren().size() == 1) {
return getChildren().get(0).getDataStructure();
}

// Get base structure, that is, structure from first parameter, without attributes
List<Entry<String, Component>> baseStructure
= new ArrayList<>(getChildren().get(0).getDataStructure().entrySet())
.stream().filter(entry -> entry.getValue().getRole() != Component.Role.ATTRIBUTE)
.collect(Collectors.toList());

// Add attributes, sorted by name
List<Entry<String, Component>> allAttributes = new ArrayList<>();
for(Dataset dataset : getChildren()) {
List<Entry<String, Component>> childAttributes = dataset.getDataStructure().entrySet()
.stream().filter(entry -> entry.getValue().getRole() == Component.Role.ATTRIBUTE)
.filter(entry -> allAttributes.stream()
.noneMatch(attributeEntry -> attributeEntry.getKey().equals(entry.getKey())))
.collect(Collectors.toList());
allAttributes.addAll(childAttributes);
}
allAttributes.sort(Comparator.comparing(Entry::getKey));
baseStructure.addAll(allAttributes);
return builder().putAll(baseStructure).build();
}

@Override
@@ -147,27 +176,73 @@ private void checkDataStructures(DataStructure baseDataStructure, DataStructure
VtlOrdering unionOrder = (VtlOrdering) computeRequiredOrdering(ordering);

DataStructure structure = getDataStructure();
ImmutableList.Builder<Stream<DataPoint>> streams = ImmutableList.builder();
ImmutableList.Builder<Stream<DataPointMap>> streams = ImmutableList.builder();
ImmutableList.Builder<Stream<DataPoint>> originals = ImmutableList.builder();
for (AbstractDatasetOperation child : getChildren()) {

VtlOrdering unionOrdering = new VtlOrdering(unionOrder, child.getDataStructure());
VtlFiltering unionFilter = VtlFiltering.using(child).with(childFiltering);

Stream<DataPoint> stream = child.computeData(unionOrdering, unionFilter, components);
originals.add(stream);
streams.add(stream.map(new DatapointNormalizer(child.getDataStructure(), structure)));
Stream<DataPointMap> dataPointMapStream = getChildDataStream(
components, childFiltering, unionOrder, structure, originals, child);
streams.add(dataPointMapStream);
}

VtlOrdering unionOrdering = new VtlOrdering(unionOrder, structure);
Comparator<DataPointMap> comparing = new DataPointMapComparator(unionOrdering);

ImmutableList<Stream<DataPointMap>> build = streams.build();

DataPointMap resultMap = new DataPointMap(structure);
Stream<DataPoint> result = StreamUtils.interleave(
createSelector(unionOrdering), streams.build()
).map(new DuplicateChecker(unionOrdering, structure));
createSelector(comparing), build)
.map(source -> {
resultMap.setDataPoint(DataPoint.create(structure.size()));
structure.keySet().forEach(col -> resultMap.set(col, source.get(col)));
return resultMap.getDataPoint();
}).map(new DuplicateChecker(unionOrdering, structure));

return new VtlStream(
this, result, originals.build(), ordering, filtering, unionOrdering, childFiltering);
}

private Stream<DataPointMap> getChildDataStream(Set<String> components, VtlFiltering childFiltering, VtlOrdering unionOrder, DataStructure structure, ImmutableList.Builder<Stream<DataPoint>> originals, AbstractDatasetOperation child) {
DataStructure childStructure = getNormalizedChildStructure(child.getDataStructure(), structure);
VtlOrdering unionOrdering = new VtlOrdering(unionOrder, childStructure);
VtlFiltering unionFilter = VtlFiltering.using(child).transpose(childFiltering);

Stream<DataPoint> stream = child.computeData(unionOrdering, unionFilter, components)
.peek(new DataPointCapacityExpander(structure.size()));

originals.add(stream);

DataPointMap map = new DataPointMap(childStructure);

return StreamSupport.stream(stream.spliterator(), false).map(map::withDataPoint);
}

/**
* Concatenates the child's structure with the base structure, to add attributes not present in child
*
* @param childStructure the child's structure
* @param baseStructure the base structure for the expression, containing the sum of all attributes from all
* parameters
* @return the concatenated structure
*/
private DataStructure getNormalizedChildStructure(DataStructure childStructure, DataStructure baseStructure) {
if (childStructure.getRoles().size() == baseStructure.getRoles().size()
&& childStructure.getRoles().keySet().equals(baseStructure.getRoles().keySet())) {
return childStructure;
}

List<Entry<String, Component>> childStructureList = new ArrayList<>(childStructure.entrySet());

// append missing attributes from baseStructure
childStructureList.addAll(baseStructure.entrySet().stream()
.filter(entry -> childStructureList.stream()
.map(Entry::getKey).noneMatch(key -> key.equals(entry.getKey())))
.collect(Collectors.toList()));

return builder().putAll(childStructureList).build();
}

private <T> Selector<T> createSelector(Comparator<T> comparator) {
return new MinimumSelector<>(comparator);
}
Oops, something went wrong.

0 comments on commit 30e57d7

Please sign in to comment.
You can’t perform that action at this time.