Merge f1b407a into a923d1d

main/src/com/google/refine/LookupCacheManager.java

@@ -37,7 +37,6 @@
 import com.google.refine.expr.ExpressionUtils;
 import com.google.refine.expr.HasFieldsListImpl;
 import com.google.refine.expr.WrappedRow;
-import com.google.refine.expr.functions.Cross;
 import com.google.refine.model.Column;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
@@ -50,6 +49,8 @@
  */
 public class LookupCacheManager {
 
+    public static final String INDEX_COLUMN_NAME = "_OpenRefine_Index_Column_Name_";
+
     protected final Map<String, ProjectLookup> _lookups = new HashMap<>();
 
     /**
@@ -111,7 +112,7 @@ protected void computeLookup(ProjectLookup lookup) throws LookupException {
         }
 
         // if this is a lookup on the index column
-        if (lookup.targetColumnName.equals(Cross.INDEX_COLUMN_NAME)) {
+        if (INDEX_COLUMN_NAME.equals(lookup.targetColumnName)) {
             for (int r = 0; r < targetProject.rows.size(); r++) {
                 lookup.valueToRowIndices.put(String.valueOf(r), Collections.singletonList(r));
             }

main/src/com/google/refine/browsing/facets/TextSearchFacet.java

@@ -33,6 +33,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 
 package com.google.refine.browsing.facets;
 
+import java.util.Properties;
 import java.util.regex.Pattern;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -44,7 +45,6 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 import com.google.refine.browsing.filters.AnyRowRecordFilter;
 import com.google.refine.browsing.filters.ExpressionStringComparisonRowFilter;
 import com.google.refine.expr.Evaluable;
-import com.google.refine.grel.ast.VariableExpr;
 import com.google.refine.model.Column;
 import com.google.refine.model.Project;
 import com.google.refine.util.PatternSyntaxExceptionParser;
@@ -156,7 +156,14 @@ public RowFilter getRowFilter(Project project) {
             return null;
         }
 
-        Evaluable eval = new VariableExpr("value");
+        Evaluable eval = new Evaluable() {
+
+            @Override
+            public Object evaluate(Properties bindings) {
+                return bindings.get("value");
+            }
+
+        };
 
         if ("regex".equals(_config._mode)) {
             return new ExpressionStringComparisonRowFilter(eval, _config._invert, _config._columnName, _cellIndex) {

main/src/com/google/refine/expr/ClojureParser.java

@@ -0,0 +1,88 @@
+/*
+
+Copyright 2010,2011. Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+package com.google.refine.expr;
+
+import java.io.StringReader;
+import java.util.Properties;
+
+import clojure.lang.IFn;
+import clojure.lang.RT;
+
+/**
+ * A parser for expressions written in Clojure.
+ */
+public class ClojureParser implements LanguageSpecificParser {
+
+    @Override
+    public Evaluable parse(String s) throws ParsingException {
+        try {
+//                    RT.load("clojure/core"); // Make sure RT is initialized
+            Object foo = RT.CURRENT_NS; // Make sure RT is initialized
+            IFn fn = (IFn) clojure.lang.Compiler.load(new StringReader(
+                    "(fn [value cell cells row rowIndex] " + s + ")"));
+
+            // TODO: We should to switch from using Compiler.load
+            // because it's technically an internal interface
+//                    Object code = CLOJURE_READ_STRING.invoke(
+//                            "(fn [value cell cells row rowIndex] " + s + ")"
+//                            );
+
+            return new Evaluable() {
+
+                private IFn _fn;
+
+                public Evaluable init(IFn fn) {
+                    _fn = fn;
+                    return this;
+                }
+
+                @Override
+                public Object evaluate(Properties bindings) {
+                    try {
+                        return _fn.invoke(
+                                bindings.get("value"),
+                                bindings.get("cell"),
+                                bindings.get("cells"),
+                                bindings.get("row"),
+                                bindings.get("rowIndex"));
+                    } catch (Exception e) {
+                        return new EvalError(e.getMessage());
+                    }
+                }
+            }.init(fn);
+        } catch (Exception e) {
+            throw new ParsingException(e.getMessage());
+        }
+    }
+}

main/src/com/google/refine/expr/MetaParser.java

@@ -33,19 +33,13 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 
 package com.google.refine.expr;
 
-import java.io.StringReader;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Set;
 
-import clojure.lang.IFn;
-import clojure.lang.RT;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-import com.google.refine.grel.Parser;
-
 abstract public class MetaParser {
 
     static public class LanguageInfo {
@@ -66,65 +60,6 @@ static public class LanguageInfo {
 
     static final protected Map<String, LanguageInfo> s_languages = new HashMap<String, LanguageInfo>();
 
-    // TODO: We should switch from using the internal compiler class
-//    final static private Var CLOJURE_READ_STRING = RT.var("clojure.core", "read-string");
-//    final static private Var CLOJURE_EVAL = RT.var("clojure.core", "eval");
-
-    static {
-        registerLanguageParser("grel", "General Refine Expression Language (GREL)", new LanguageSpecificParser() {
-
-            @Override
-            public Evaluable parse(String s) throws ParsingException {
-                return parseGREL(s);
-            }
-        }, "value");
-
-        registerLanguageParser("clojure", "Clojure", new LanguageSpecificParser() {
-
-            @Override
-            public Evaluable parse(String s) throws ParsingException {
-                try {
-//                    RT.load("clojure/core"); // Make sure RT is initialized
-                    Object foo = RT.CURRENT_NS; // Make sure RT is initialized
-                    IFn fn = (IFn) clojure.lang.Compiler.load(new StringReader(
-                            "(fn [value cell cells row rowIndex] " + s + ")"));
-
-                    // TODO: We should to switch from using Compiler.load
-                    // because it's technically an internal interface
-//                    Object code = CLOJURE_READ_STRING.invoke(
-//                            "(fn [value cell cells row rowIndex] " + s + ")"
-//                            );
-
-                    return new Evaluable() {
-
-                        private IFn _fn;
-
-                        public Evaluable init(IFn fn) {
-                            _fn = fn;
-                            return this;
-                        }
-
-                        @Override
-                        public Object evaluate(Properties bindings) {
-                            try {
-                                return _fn.invoke(
-                                        bindings.get("value"),
-                                        bindings.get("cell"),
-                                        bindings.get("cells"),
-                                        bindings.get("row"),
-                                        bindings.get("rowIndex"));
-                            } catch (Exception e) {
-                                return new EvalError(e.getMessage());
-                            }
-                        }
-                    }.init(fn);
-                } catch (Exception e) {
-                    throw new ParsingException(e.getMessage());
-                }
-            }
-        }, "value");
-    }
-
     /**
      * languagePrefix will be stored in the meta model as an identifier. so be careful when change it as it will break
      * the backward compatibility for the old project
@@ -174,8 +109,10 @@ static public Evaluable parse(String s) throws ParsingException {
     }
 
     static protected Evaluable parseGREL(String s) throws ParsingException {
-        Parser parser = new Parser(s);
-
-        return parser.getExpression();
+        LanguageInfo info = s_languages.get("grel");
+        if (info == null) {
+            throw new ParsingException("Default language GREL is not available");
+        }
+        return info.parser.parse(s);
     }
 }

main/src/com/google/refine/expr/functions/Cross.java

@@ -35,6 +35,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 
 import java.util.Properties;
 
+import com.google.refine.LookupCacheManager;
 import com.google.refine.LookupCacheManager.ProjectLookup;
 import com.google.refine.ProjectManager;
 import com.google.refine.expr.EvalError;
@@ -49,7 +50,11 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 
 public class Cross implements Function {
 
-    public static final String INDEX_COLUMN_NAME = "_OpenRefine_Index_Column_Name_";
+    /**
+     * @deprecated use {@link LookupCacheManager#INDEX_COLUMN_NAME}.
+     */
+    @Deprecated
+    public static final String INDEX_COLUMN_NAME = LookupCacheManager.INDEX_COLUMN_NAME;
 
     @Override
     public Object call(Properties bindings, Object[] args) {
@@ -65,7 +70,7 @@ public Object call(Properties bindings, Object[] args) {
                 targetProjectName = args[1];
             }
             // if 3rd argument is omitted or set to "", use the index column
-            Object targetColumnName = args.length < 3 || "".equals(args[2]) ? INDEX_COLUMN_NAME : args[2];
+            Object targetColumnName = args.length < 3 || "".equals(args[2]) ? LookupCacheManager.INDEX_COLUMN_NAME : args[2];
 
             long targetProjectID;
             ProjectLookup lookup;

main/src/com/google/refine/grel/Parser.java

@@ -38,6 +38,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 import java.util.regex.Pattern;
 
 import com.google.refine.expr.Evaluable;
+import com.google.refine.expr.LanguageSpecificParser;
 import com.google.refine.expr.ParsingException;
 import com.google.refine.expr.functions.arrays.ArgsToArray;
 import com.google.refine.grel.Scanner.NumberToken;
@@ -53,6 +54,15 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 
 public class Parser {
 
+    static public LanguageSpecificParser grelParser = new LanguageSpecificParser() {
+
+        @Override
+        public Evaluable parse(String source) throws ParsingException {
+            Parser parser = new Parser(source);
+            return parser.getExpression();
+        }
+    };
+
     protected Scanner _scanner;
     protected Token _token;
     protected Evaluable _root;

main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java

@@ -56,7 +56,8 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
-import com.google.refine.expr.functions.ToDate;
+import com.google.refine.expr.util.CalendarParser;
+import com.google.refine.expr.util.CalendarParserException;
 import com.google.refine.model.ReconCandidate;
 import com.google.refine.model.ReconType;
 import com.google.refine.util.HttpClient;
@@ -250,11 +251,13 @@ protected ReconciledDataExtensionJob.DataExtension collectResult(
                     int v = val.get("int").asInt();
                     storeCell(rows, rowindex, colindex, v);
                 } else if (val.has("date")) {
-                    ToDate td = new ToDate();
-                    String[] args = new String[1];
-                    args[0] = val.get("date").asText();
-                    Object v = td.call(null, args);
-                    storeCell(rows, rowindex, colindex, v);
+                    Object date;
+                    try {
+                        date = CalendarParser.parseAsOffsetDateTime(val.get("date").asText());
+                    } catch (CalendarParserException e) {
+                        date = val.get("date").asText();
+                    }
+                    storeCell(rows, rowindex, colindex, date);
                 } else if (val.has("bool")) {
                     boolean v = val.get("bool").asBoolean();
                     storeCell(rows, rowindex, colindex, v);

main/tests/server/src/com/google/refine/RefineTest.java

@@ -68,6 +68,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 import com.google.refine.expr.ParsingException;
 import com.google.refine.grel.ControlFunctionRegistry;
 import com.google.refine.grel.Function;
+import com.google.refine.grel.Parser;
 import com.google.refine.importers.SeparatorBasedImporter;
 import com.google.refine.importing.ImportingJob;
 import com.google.refine.importing.ImportingManager;
@@ -120,6 +121,7 @@ public void init() {
         }
         // This just keeps track of any failed test, for cleanupWorkspace
         testFailed = false;
+        MetaParser.registerLanguageParser("grel", "GREL", Parser.grelParser, "value");
     }
 
     @BeforeMethod

main/tests/server/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperationTests.java

@@ -101,9 +101,8 @@ public class ColumnAdditionByFetchingURLsOperationTests extends RefineTest {
             "    \"status\" : \"pending\"\n" +
             " }";
 
-    @Override
     @BeforeTest
-    public void init() {
+    public void initOperation() {
         logger = LoggerFactory.getLogger(this.getClass());
         OperationRegistry.registerOperation(getCoreModule(), "column-addition-by-fetching-urls",
                 ColumnAdditionByFetchingURLsOperation.class);

main/webapp/modules/core/MOD-INF/controller.js

@@ -332,6 +332,12 @@ function registerExporters() {
    ER.registerExporter("sql", new Packages.com.google.refine.exporters.sql.SqlExporter());
 }
 
+function registerLanguages() {
+  var MP = Packages.com.google.refine.expr.MetaParser;
+  MP.registerLanguageParser("grel", "General Refine Expression Language (GREL)", Packages.com.google.refine.grel.Parser.grelParser, "value");
+  MP.registerLanguageParser("clojure", "Clojure", new Packages.com.google.refine.expr.ClojureParser(), "value");
+}
+
 function registerDistances() {
    var DF = Packages.com.google.refine.clustering.knn.DistanceFactory;
    var VicinoDistance = Packages.com.google.refine.clustering.knn.VicinoDistance;
@@ -359,6 +365,7 @@ function init() {
   registerOperations();
   registerImporting();
   registerExporters();
+  registerLanguages();
   registerDistances();
   registerKeyers();