diff --git a/main/src/com/google/refine/LookupCacheManager.java b/main/src/com/google/refine/LookupCacheManager.java index fe8ba805818c..2cd67f19055d 100644 --- a/main/src/com/google/refine/LookupCacheManager.java +++ b/main/src/com/google/refine/LookupCacheManager.java @@ -37,7 +37,6 @@ import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.HasFieldsListImpl; import com.google.refine.expr.WrappedRow; -import com.google.refine.expr.functions.Cross; import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.model.Row; @@ -50,6 +49,8 @@ */ public class LookupCacheManager { + public static final String INDEX_COLUMN_NAME = "_OpenRefine_Index_Column_Name_"; + protected final Map _lookups = new HashMap<>(); /** @@ -111,7 +112,7 @@ protected void computeLookup(ProjectLookup lookup) throws LookupException { } // if this is a lookup on the index column - if (lookup.targetColumnName.equals(Cross.INDEX_COLUMN_NAME)) { + if (INDEX_COLUMN_NAME.equals(lookup.targetColumnName)) { for (int r = 0; r < targetProject.rows.size(); r++) { lookup.valueToRowIndices.put(String.valueOf(r), Collections.singletonList(r)); } diff --git a/main/src/com/google/refine/browsing/facets/TextSearchFacet.java b/main/src/com/google/refine/browsing/facets/TextSearchFacet.java index f007c8302f9e..3069ce10be1b 100644 --- a/main/src/com/google/refine/browsing/facets/TextSearchFacet.java +++ b/main/src/com/google/refine/browsing/facets/TextSearchFacet.java @@ -33,6 +33,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT package com.google.refine.browsing.facets; +import java.util.Properties; import java.util.regex.Pattern; import com.fasterxml.jackson.annotation.JsonProperty; @@ -44,7 +45,6 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import com.google.refine.browsing.filters.AnyRowRecordFilter; import com.google.refine.browsing.filters.ExpressionStringComparisonRowFilter; import com.google.refine.expr.Evaluable; -import com.google.refine.grel.ast.VariableExpr; import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.util.PatternSyntaxExceptionParser; @@ -156,7 +156,14 @@ public RowFilter getRowFilter(Project project) { return null; } - Evaluable eval = new VariableExpr("value"); + Evaluable eval = new Evaluable() { + + @Override + public Object evaluate(Properties bindings) { + return bindings.get("value"); + } + + }; if ("regex".equals(_config._mode)) { return new ExpressionStringComparisonRowFilter(eval, _config._invert, _config._columnName, _cellIndex) { diff --git a/main/src/com/google/refine/expr/ClojureParser.java b/main/src/com/google/refine/expr/ClojureParser.java new file mode 100644 index 000000000000..d97f29f5905b --- /dev/null +++ b/main/src/com/google/refine/expr/ClojureParser.java @@ -0,0 +1,88 @@ +/* + +Copyright 2010,2011. Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.expr; + +import java.io.StringReader; +import java.util.Properties; + +import clojure.lang.IFn; +import clojure.lang.RT; + +/** + * A parser for expressions written in Clojure. + */ +public class ClojureParser implements LanguageSpecificParser { + + @Override + public Evaluable parse(String s) throws ParsingException { + try { +// RT.load("clojure/core"); // Make sure RT is initialized + Object foo = RT.CURRENT_NS; // Make sure RT is initialized + IFn fn = (IFn) clojure.lang.Compiler.load(new StringReader( + "(fn [value cell cells row rowIndex] " + s + ")")); + + // TODO: We should to switch from using Compiler.load + // because it's technically an internal interface +// Object code = CLOJURE_READ_STRING.invoke( +// "(fn [value cell cells row rowIndex] " + s + ")" +// ); + + return new Evaluable() { + + private IFn _fn; + + public Evaluable init(IFn fn) { + _fn = fn; + return this; + } + + @Override + public Object evaluate(Properties bindings) { + try { + return _fn.invoke( + bindings.get("value"), + bindings.get("cell"), + bindings.get("cells"), + bindings.get("row"), + bindings.get("rowIndex")); + } catch (Exception e) { + return new EvalError(e.getMessage()); + } + } + }.init(fn); + } catch (Exception e) { + throw new ParsingException(e.getMessage()); + } + } +} diff --git a/main/src/com/google/refine/expr/MetaParser.java b/main/src/com/google/refine/expr/MetaParser.java index c0987ecfeb6b..9840f67291e4 100644 --- a/main/src/com/google/refine/expr/MetaParser.java +++ b/main/src/com/google/refine/expr/MetaParser.java @@ -33,19 +33,13 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT package com.google.refine.expr; -import java.io.StringReader; import java.util.HashMap; import java.util.Map; -import java.util.Properties; import java.util.Set; -import clojure.lang.IFn; -import clojure.lang.RT; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.refine.grel.Parser; - abstract public class MetaParser { static public class LanguageInfo { @@ -66,65 +60,6 @@ static public class LanguageInfo { static final protected Map s_languages = new HashMap(); - // TODO: We should switch from using the internal compiler class -// final static private Var CLOJURE_READ_STRING = RT.var("clojure.core", "read-string"); -// final static private Var CLOJURE_EVAL = RT.var("clojure.core", "eval"); - - static { - registerLanguageParser("grel", "General Refine Expression Language (GREL)", new LanguageSpecificParser() { - - @Override - public Evaluable parse(String s) throws ParsingException { - return parseGREL(s); - } - }, "value"); - - registerLanguageParser("clojure", "Clojure", new LanguageSpecificParser() { - - @Override - public Evaluable parse(String s) throws ParsingException { - try { -// RT.load("clojure/core"); // Make sure RT is initialized - Object foo = RT.CURRENT_NS; // Make sure RT is initialized - IFn fn = (IFn) clojure.lang.Compiler.load(new StringReader( - "(fn [value cell cells row rowIndex] " + s + ")")); - - // TODO: We should to switch from using Compiler.load - // because it's technically an internal interface -// Object code = CLOJURE_READ_STRING.invoke( -// "(fn [value cell cells row rowIndex] " + s + ")" -// ); - - return new Evaluable() { - - private IFn _fn; - - public Evaluable init(IFn fn) { - _fn = fn; - return this; - } - - @Override - public Object evaluate(Properties bindings) { - try { - return _fn.invoke( - bindings.get("value"), - bindings.get("cell"), - bindings.get("cells"), - bindings.get("row"), - bindings.get("rowIndex")); - } catch (Exception e) { - return new EvalError(e.getMessage()); - } - } - }.init(fn); - } catch (Exception e) { - throw new ParsingException(e.getMessage()); - } - } - }, "value"); - } - /** * languagePrefix will be stored in the meta model as an identifier. so be careful when change it as it will break * the backward compatibility for the old project @@ -174,8 +109,10 @@ static public Evaluable parse(String s) throws ParsingException { } static protected Evaluable parseGREL(String s) throws ParsingException { - Parser parser = new Parser(s); - - return parser.getExpression(); + LanguageInfo info = s_languages.get("grel"); + if (info == null) { + throw new ParsingException("Default language GREL is not available"); + } + return info.parser.parse(s); } } diff --git a/main/src/com/google/refine/expr/functions/Cross.java b/main/src/com/google/refine/expr/functions/Cross.java index 6521536fbc39..6a58fb1cfbcb 100644 --- a/main/src/com/google/refine/expr/functions/Cross.java +++ b/main/src/com/google/refine/expr/functions/Cross.java @@ -35,6 +35,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import java.util.Properties; +import com.google.refine.LookupCacheManager; import com.google.refine.LookupCacheManager.ProjectLookup; import com.google.refine.ProjectManager; import com.google.refine.expr.EvalError; @@ -49,7 +50,11 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT public class Cross implements Function { - public static final String INDEX_COLUMN_NAME = "_OpenRefine_Index_Column_Name_"; + /** + * @deprecated use {@link LookupCacheManager#INDEX_COLUMN_NAME}. + */ + @Deprecated + public static final String INDEX_COLUMN_NAME = LookupCacheManager.INDEX_COLUMN_NAME; @Override public Object call(Properties bindings, Object[] args) { @@ -65,7 +70,7 @@ public Object call(Properties bindings, Object[] args) { targetProjectName = args[1]; } // if 3rd argument is omitted or set to "", use the index column - Object targetColumnName = args.length < 3 || "".equals(args[2]) ? INDEX_COLUMN_NAME : args[2]; + Object targetColumnName = args.length < 3 || "".equals(args[2]) ? LookupCacheManager.INDEX_COLUMN_NAME : args[2]; long targetProjectID; ProjectLookup lookup; diff --git a/main/src/com/google/refine/grel/Parser.java b/main/src/com/google/refine/grel/Parser.java index 51271d17b7d6..09c1db945b89 100644 --- a/main/src/com/google/refine/grel/Parser.java +++ b/main/src/com/google/refine/grel/Parser.java @@ -38,6 +38,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import java.util.regex.Pattern; import com.google.refine.expr.Evaluable; +import com.google.refine.expr.LanguageSpecificParser; import com.google.refine.expr.ParsingException; import com.google.refine.expr.functions.arrays.ArgsToArray; import com.google.refine.grel.Scanner.NumberToken; @@ -53,6 +54,15 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT public class Parser { + static public LanguageSpecificParser grelParser = new LanguageSpecificParser() { + + @Override + public Evaluable parse(String source) throws ParsingException { + Parser parser = new Parser(source); + return parser.getExpression(); + } + }; + protected Scanner _scanner; protected Token _token; protected Evaluable _root; diff --git a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java index 7dd311c57607..51b79113dffa 100644 --- a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java +++ b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java @@ -40,6 +40,8 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import java.io.IOException; import java.io.StringWriter; import java.io.Writer; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -56,7 +58,6 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.expr.functions.ToDate; import com.google.refine.model.ReconCandidate; import com.google.refine.model.ReconType; import com.google.refine.util.HttpClient; @@ -250,11 +251,13 @@ protected ReconciledDataExtensionJob.DataExtension collectResult( int v = val.get("int").asInt(); storeCell(rows, rowindex, colindex, v); } else if (val.has("date")) { - ToDate td = new ToDate(); - String[] args = new String[1]; - args[0] = val.get("date").asText(); - Object v = td.call(null, args); - storeCell(rows, rowindex, colindex, v); + Object date; + try { + date = OffsetDateTime.parse(val.get("date").asText()); + } catch (DateTimeParseException e) { + date = val.get("date").asText(); + } + storeCell(rows, rowindex, colindex, date); } else if (val.has("bool")) { boolean v = val.get("bool").asBoolean(); storeCell(rows, rowindex, colindex, v); diff --git a/main/tests/server/src/com/google/refine/RefineTest.java b/main/tests/server/src/com/google/refine/RefineTest.java index bd5d84d6f8a9..6e71efd52f99 100644 --- a/main/tests/server/src/com/google/refine/RefineTest.java +++ b/main/tests/server/src/com/google/refine/RefineTest.java @@ -67,6 +67,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import com.google.refine.expr.ParsingException; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; +import com.google.refine.grel.Parser; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingManager; import com.google.refine.io.FileProjectManager; @@ -118,6 +119,7 @@ public void init() { } // This just keeps track of any failed test, for cleanupWorkspace testFailed = false; + MetaParser.registerLanguageParser("grel", "GREL", Parser.grelParser, "value"); } @BeforeMethod diff --git a/main/tests/server/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperationTests.java b/main/tests/server/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperationTests.java index 265c9edc181e..4c10bc5a44a2 100644 --- a/main/tests/server/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperationTests.java +++ b/main/tests/server/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperationTests.java @@ -101,9 +101,8 @@ public class ColumnAdditionByFetchingURLsOperationTests extends RefineTest { " \"status\" : \"pending\"\n" + " }"; - @Override @BeforeTest - public void init() { + public void initOperation() { logger = LoggerFactory.getLogger(this.getClass()); OperationRegistry.registerOperation(getCoreModule(), "column-addition-by-fetching-urls", ColumnAdditionByFetchingURLsOperation.class); diff --git a/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java b/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java index 427360cb0b1a..9e15d7005f24 100644 --- a/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java +++ b/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java @@ -40,6 +40,7 @@ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT import java.io.StringWriter; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -303,6 +304,50 @@ public void testFetchStrings() throws Exception { } } + @Test + public void testFetchOtherDatatypes() throws Exception { + + DataExtensionConfig extension = DataExtensionConfig + .reconstruct("{\"properties\":[{\"id\":\"P123\",\"name\":\"dummy property\"}]}"); + + try (MockWebServer server = new MockWebServer()) { + server.start(); + server.setDispatcher(dispatcher); + + mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P123\"}]}", + "{" + + "\"rows\": {" + + " \"Q794\": {\"P123\": [{\"int\": 4}]}," + + " \"Q863\": {\"P123\": []}," + + " \"Q30\": {\"P123\": [{\"date\": \"2023-05-03T04:05:06Z\"}]}," + + " \"Q17\": {\"P123\": [{\"bool\": true}]}" + + "}," + + "\"meta\": [" + + " {\"name\": \"dummy property\", \"id\": \"P123\"}" + + "]}"); + + EngineDependentOperation op = new ExtendDataOperation(engine_config, + "country", + server.url("/reconcile").url().toString(), + RECON_IDENTIFIER_SPACE, + RECON_SCHEMA_SPACE, + extension, + 1); + + runOperation(op, project); + + Project expectedProject = createProject( + new String[] { "country", "dummy property" }, + new Serializable[][] { + { reconCell1, 4 }, + { reconCell2, true }, + { reconCell3, null }, + { reconCell4, OffsetDateTime.parse("2023-05-03T04:05:06Z") }, + }); + assertProjectEquals(project, expectedProject); + } + } + /** * Test to fetch counts of values */ diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index 344b1dfa4dd3..d037edba8ee2 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -332,6 +332,12 @@ function registerExporters() { ER.registerExporter("sql", new Packages.com.google.refine.exporters.sql.SqlExporter()); } +function registerLanguages() { + var MP = Packages.com.google.refine.expr.MetaParser; + MP.registerLanguageParser("grel", "General Refine Expression Language (GREL)", Packages.com.google.refine.grel.Parser.grelParser, "value"); + MP.registerLanguageParser("clojure", "Clojure", new Packages.com.google.refine.expr.ClojureParser(), "value"); +} + function registerDistances() { var DF = Packages.com.google.refine.clustering.knn.DistanceFactory; var VicinoDistance = Packages.com.google.refine.clustering.knn.VicinoDistance; @@ -359,6 +365,7 @@ function init() { registerOperations(); registerImporting(); registerExporters(); + registerLanguages(); registerDistances(); registerKeyers();