Merge branch 'develop' into feature/expression-statement-refactoring

README.md

@@ -29,7 +29,7 @@ The project is divided in modules;
 - java-vtl-parent
     - java-vtl-parser, contains the lexer and parser for VTL.
     - java-vtl-model, VTL data model.
-    - java-vtl-script, JSR-223 implementation.
+    - java-vtl-script, JSR-223 (ScriptEngine) implementation.
     - java-vtl-connector, connector API.
     - java-vtl-tools, various tools.
 
@@ -55,7 +55,7 @@ engine.eval("ds1 := get(\"foo\")" +
             "ds2 := get(\"bar\")" +
             "ds3 := [ds1, ds2] {" +
             "   filter ds1.id = \"string\"," +
-            "   sum := ds1.measure + ds2.measure" +
+            "   total := ds1.measure + ds2.measure" +
             "}");
 
 System.out.println(bindings.get("ds3"))

java-vtl-dependency-parser/src/main/java/no/ssb/vtl/dependencies/AssignmentListener.java

@@ -1,65 +0,0 @@
-package no.ssb.vtl.dependencies;
-
-/*-
- * ========================LICENSE_START=================================
- * Java VTL
- * %%
- * Copyright (C) 2016 - 2017 Hadrien Kohl
- * %%
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =========================LICENSE_END==================================
- */
-
-import com.google.common.collect.ImmutableSet;
-import no.ssb.vtl.parser.VTLBaseListener;
-import no.ssb.vtl.parser.VTLParser;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class AssignmentListener extends VTLBaseListener {
-
-    private Map<String, Assignment> variableDependency;
-
-    private Set<ComponentRef> componentRefs = new HashSet<>();
-
-    public AssignmentListener() {
-        variableDependency = new HashMap<>();
-    }
-
-    @Override
-    public void exitJoinCalcClause(VTLParser.JoinCalcClauseContext ctx) {
-        String identifier = ctx.variable().getText();
-        String expression = ctx.getText();
-        Assignment assignment =
-                new Assignment(identifier, expression,ImmutableSet.copyOf(componentRefs));
-        variableDependency.put(identifier, assignment);
-        componentRefs.clear();
-    }
-
-    @Override
-    public void exitComponentRef(VTLParser.ComponentRefContext ctx) {
-
-        String datasetRef = ctx.datasetRef().getText();
-        String variableRef = ctx.variable().getText();
-        componentRefs.add(new ComponentRef(datasetRef, variableRef));
-    }
-
-    public Map<String, Assignment> getVariableDependency() {
-        return variableDependency;
-    }
-
-}
-

java-vtl-documentation/docs/reference.md

@@ -203,18 +203,18 @@ The operator nvl replaces null values with a value given as a parameter.
 <div vtl-example>
     <vtl-code>
 join := [outer left, right] {
-    nvl_result := nvl(right.measure, "was null")
+    nvl_result := nvl(right.value, "was null")
 }
     </vtl-code>
     <vtl-dataset name="left">
-id1[I,String],id2[I,String],measure[M,String],attribute[A,String]
+id1[I,String],id2[I,String],value[M,String],attribute[A,String]
 1,3,left value 3, left attribute 3
 1,4,left value 4, left attribute 4
 1,5,left value 5, left attribute 5
 1,6,left value 6, left attribute 6
     </vtl-dataset>
     <vtl-dataset name="right">
-id1[I,String],id2[I,String],measure[M,String],attribute[A,String]
+id1[I,String],id2[I,String],value[M,String],attribute[A,String]
 1,1,right value 1, right attribute 1
 1,2,right value 2, right attribute 2
 1,3,right value 3, right attribute 3

java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4

@@ -39,7 +39,7 @@ datasetExpression : <assoc=right>datasetExpression clauseExpression #withClause
            | hierarchyExpression                                    #withHierarchy
            | relationalExpression                                   #withRelational
            | function                                               #withFunction
-           | variable                                               #withAtom
+           | exprAtom                                               #withAtom
            ;
 
 hierarchyExpression : 'hierarchy' '(' datasetRef ',' componentRef ',' hierarchyReference ',' BOOLEAN_CONSTANT ( ',' ('sum' | 'prod') )? ')' ;
@@ -66,14 +66,17 @@ GROUP_BY : 'group by' ;
 
 datasetId : STRING_CONSTANT ;
 
+/* Atom */
+exprAtom : datasetRef;
+
 checkFunction : 'check' '(' checkParam ')';
 
 checkParam : datasetExpression (',' checkRows)? (',' checkColumns)? (',' 'errorcode' '(' errorCode ')' )? (',' 'errorlevel' '=' '(' errorLevel ')' )?;
 
 checkRows : ( 'not_valid' | 'valid' | 'all' ) ;
 checkColumns : ( 'measures' | 'condition' ) ;
 errorCode : STRING_CONSTANT ;
-errorLevel : INTEGER_CONSTANT;
+errorLevel : INTEGER_CONSTANT ;
 
 datasetRef: variable ;
 

java-vtl-script/src/main/java/no/ssb/vtl/script/operations/AggregationOperation.java

@@ -80,9 +80,10 @@ public Stream<DataPoint> getData() {
         groupBy.forEach(component -> builder.put(component, Order.Direction.ASC));
         Order order = builder.build();
 
-        Stream<DataPoint> data = getChild().getData(order).orElse(getChild().getData().sorted(order));
+        Stream<DataPoint> data = getChild().getData(order).orElseGet(() -> getChild().getData().sorted(order));
         Stream<List<DataPoint>> groupedDataPoints = StreamUtils.aggregate(data,
-                (dataPoint1, dataPoint2) -> order.compare(dataPoint1, dataPoint2) == 0);
+                (dataPoint1, dataPoint2) -> order.compare(dataPoint1, dataPoint2) == 0)
+                .onClose(data::close);
 
         @SuppressWarnings("UnnecessaryLocalVariable")
         Stream<DataPoint> aggregatedDataPoints = groupedDataPoints.map(dataPoints -> {