CNDB-9422: Add CQL function to get the tokens produced by a Lucene an…

…alyzer

src/java/org/apache/cassandra/cql3/functions/IndexFcts.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import com.google.common.base.Charsets;
+
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.analyzer.JSONAnalyzerParser;
+import org.apache.cassandra.index.sai.analyzer.LuceneAnalyzer;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.lucene.analysis.Analyzer;
+
+public abstract class IndexFcts
+{
+    public static void addFunctionsTo(NativeFunctions functions)
+    {
+        functions.add(new AnalyzeFunction());
+    }
+
+    /**
+     * CQL native function to get the tokens produced for given text value and the analyzer defined by the given JSON options.
+     */
+    private static class AnalyzeFunction extends NativeScalarFunction
+    {
+        private static final String NAME = "analyze";
+        private static final ListType<String> returnType = ListType.getInstance(UTF8Type.instance, false);
+
+        private AnalyzeFunction()
+        {
+            super(NAME, returnType, UTF8Type.instance, UTF8Type.instance);
+        }
+
+        @Override
+        public ByteBuffer execute(ProtocolVersion protocolVersion, List<ByteBuffer> parameters) throws InvalidRequestException
+        {
+            if (parameters.get(0) == null)
+                return null;
+            String text = UTF8Type.instance.compose(parameters.get(0));
+
+            if (parameters.get(1) == null)
+                throw new InvalidRequestException("Function " + name + " requires a non-null json_analyzer parameter (2nd argument)");
+            String json = UTF8Type.instance.compose(parameters.get(1));
+
+            LuceneAnalyzer luceneAnalyzer = null;
+            List<String> tokens = new ArrayList<>();
+            try (Analyzer analyzer = JSONAnalyzerParser.parse(json))
+            {
+                luceneAnalyzer = new LuceneAnalyzer(UTF8Type.instance, analyzer, new HashMap<>());
+
+                ByteBuffer toAnalyze = ByteBuffer.wrap(text.getBytes(Charsets.UTF_8));
+                luceneAnalyzer.reset(toAnalyze);
+                ByteBuffer analyzed;
+
+                while (luceneAnalyzer.hasNext())
+                {
+                    analyzed = luceneAnalyzer.next();
+                    tokens.add(ByteBufferUtil.string(analyzed, Charsets.UTF_8));
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidRequestException("Function " + name + " unable to analyze text=" + text + " json_analyzer=" + json, ex);
+            }
+            finally
+            {
+                if (luceneAnalyzer != null)
+                {
+                    luceneAnalyzer.end();
+                }
+            }
+
+            return returnType.decompose(tokens);
+        }
+    }
+}

src/java/org/apache/cassandra/cql3/functions/NativeFunctions.java

@@ -42,6 +42,7 @@ public class NativeFunctions
             AggregateFcts.addFunctionsTo(this);
             BytesConversionFcts.addFunctionsTo(this);
             VectorFcts.addFunctionsTo(this);
+            IndexFcts.addFunctionsTo(this);
         }
     };
 

src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java

@@ -22,7 +22,6 @@
 import com.google.common.collect.ImmutableList;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
-import org.apache.cassandra.index.sai.virtual.AnalyzerView;
 import org.apache.cassandra.index.sai.virtual.IndexesSystemView;
 import org.apache.cassandra.index.sai.virtual.SSTablesSystemView;
 import org.apache.cassandra.index.sai.virtual.SegmentsSystemView;
@@ -55,7 +54,6 @@ private static Collection<VirtualTable> buildTables()
                   .add(new InternodeInboundTable(VIRTUAL_VIEWS))
                   .add(new SSTablesSystemView(VIRTUAL_VIEWS))
                   .add(new SegmentsSystemView(VIRTUAL_VIEWS))
-                  .add(new AnalyzerView(VIRTUAL_VIEWS))
                   .addAll(TableMetricTables.getAll(VIRTUAL_VIEWS));
         if (CassandraRelevantProperties.SYSTEM_VIEWS_INCLUDE_ALL.getBoolean()
             || CassandraRelevantProperties.SYSTEM_VIEWS_INCLUDE_LOCAL_AND_PEERS.getBoolean())

test/unit/org/apache/cassandra/cql3/functions/IndexFctsTest.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import org.junit.Test;
+
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sai.SAITester;
+
+public class IndexFctsTest extends SAITester
+{
+    @Test
+    public void testAnalyzeFunction() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v text)");
+        execute("INSERT INTO %s (k, v) VALUES (1, 'johnny apples seedlings')");
+        execute("INSERT INTO %s (k, v) VALUES (2, null)");
+
+        assertRows(execute("SELECT k, analyze(v, ?) FROM %s",
+                           "{\n" +
+                           "\t\"tokenizer\":{\"name\":\"whitespace\"},\n" +
+                           "\t\"filters\":[{\"name\":\"porterstem\"}]\n" +
+                           '}'),
+                   row(1, list("johnni", "appl", "seedl")),
+                   row(2, null));
+
+        assertInvalidThrowMessage("Function system.analyze requires a non-null json_analyzer parameter (2nd argument)",
+                                  InvalidRequestException.class,
+                                  "SELECT analyze(v, null) FROM %s");
+
+        assertInvalidThrowMessage("Function system.analyze unable to analyze text=abc json_analyzer=def",
+                                  InvalidRequestException.class,
+                                  "SELECT analyze('abc', 'def') FROM %s");
+    }
+}